Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(egress): cover service degradation paths

Alfredo Lopezee3596867c16
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index b77601d73..ac616d14e 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -461,6 +461,9 @@ jobs:
- name: Test WRC internal dependency NetworkPolicy contracts
run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
+ - name: Test WRC external-egress degradation gate contract
+ run: bash scripts/tests/test-wrc-egress-degradation-gate.sh
+
- name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)
run: bash scripts/tests/test-lint-networkpolicies.sh
diff --git a/Makefile b/Makefile
index 487f3903f..c7d9fc1e7 100644
--- a/Makefile
+++ b/Makefile
@@ -1261,6 +1261,18 @@ test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC w
@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
+.PHONY: test-e2e-wrc-egress-degradation
+test-e2e-wrc-egress-degradation: ## Prove valid UI/workload services survive transient DNS while removed/raced egress is contracted
+ @echo "Running WRC external-egress degradation gate..."
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh
+
+.PHONY: test-e2e-pr567-egress-resilience
+test-e2e-pr567-egress-resilience: ## Run the exact-head HCC + WRC external-egress resilience journeys sequentially
+ @echo "Running PR #567 external-egress resilience gates..."
+ $(MAKE) test-e2e-hcc-mcp-context-readiness E2E_KUBECONTEXT=$(E2E_KUBECONTEXT) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"
+ $(MAKE) test-e2e-wrc-egress-degradation E2E_KUBECONTEXT=$(E2E_KUBECONTEXT) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"
+
.PHONY: test-e2e-hcc-readiness-bootstrap
test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active
@echo "Running HCC initial-fleet readiness gate..."
diff --git a/scripts/e2e/e2e-wrc-egress-degradation.sh b/scripts/e2e/e2e-wrc-egress-degradation.sh
new file mode 100755
index 000000000..bc1b3d66c
--- /dev/null
+++ b/scripts/e2e/e2e-wrc-egress-degradation.sh
@@ -0,0 +1,538 @@
+#!/usr/bin/env bash
+# Proves that a correctly configured, active WorkflowRecipe stays available
+# while WRC contracts removed UI/workload egress authorization across a real
+# transient DNS failure. The fixture also widens both live policies while DNS
+# is held and proves the post-DNS fresh-read contraction removes that drift
+# before the reconcile completes.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/e2e-lib.sh"
+# Reuse the exact-head/profile/fingerprint validator used by the disruptive HCC
+# gates. Despite its historical name, this helper performs no HCC mutation.
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+
+[ -n "$E2E_KUBECONTEXT" ] || {
+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch profile." >&2
+ exit 1
+}
+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
+ echo "Refusing WRC fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
+ exit 1
+}
+require_safe_kube_context
+[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" = 1 ] || {
+ echo "Set E2E_WRC_EGRESS_FAULT_INJECTION=1 to acknowledge reversible WRC DNS fault injection." >&2
+ exit 1
+}
+command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
+command -v node >/dev/null 2>&1 || { echo "node is required" >&2; exit 1; }
+
+WRC_NS="${WRC_NAMESPACE:-control-plane}"
+WRC_DEPLOY="${WRC_DEPLOYMENT:-workflow-recipes}"
+UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"
+RECIPE_NAMESPACE="${WORKFLOW_RECIPE_NS:-sandbox-recipes}"
+RUN_ID="$(date +%s)-$$"
+RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"
+SUITE_NAME=wrc-egress-degradation
+RECIPE_NAME="$(truncate_rfc1123 "e2e-wrc-egress-${RUN_ID}")"
+TARGET_DNS="$(truncate_rfc1123 "wrc-egress-${RUN_ID}").example.com"
+ANSWER_IP=93.184.216.34
+DNS_PROXY="$(truncate_rfc1123 "e2e-wrc-dns-${RUN_ID}")"
+DNS_PROXY_CONFIGMAP="$(truncate_rfc1123 "${DNS_PROXY}-script")"
+DNS_PROXY_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-policy")"
+WRC_DNS_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-from-wrc")"
+LOCK_NAME="e2e-wrc-egress-degradation-lock"
+UI_POLICY="ui-egress-${RECIPE_NAME}"
+WORKLOAD_POLICY="wl-egress-${RECIPE_NAME}-worker"
+DNS_PROXY_SCRIPT="${SCRIPT_DIR}/../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"
+
+LOCK_UID=""
+LOCK_ACQUIRED=0
+PROXY_CREATED=0
+WRC_MUTATED=0
+RECIPE_CREATED=0
+ORIGINAL_DNS_POLICY=""
+ORIGINAL_DNS_CONFIG=""
+ORIGINAL_REPLICAS=""
+WRC_IMAGE=""
+UPSTREAM_DNS_IP=""
+
+die() {
+ echo "[wrc-egress-degradation] ERROR: $*" >&2
+ if [ "$PROXY_CREATED" = 1 ]; then
+ kctl logs "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy --tail=120 >&2 || true
+ fi
+ exit 1
+}
+
+wait_until() {
+ local timeout=$1 description=$2
+ shift 2
+ local deadline=$(( $(date +%s) + timeout ))
+ while [ "$(date +%s)" -lt "$deadline" ]; do
+ "$@" && return 0
+ sleep 1
+ done
+ echo "Timed out after ${timeout}s waiting for ${description}" >&2
+ return 1
+}
+
+deployment_ready() {
+ local namespace=$1 name=$2
+ local deployment
+ deployment="$(kctl get deployment "$name" -n "$namespace" -o json 2>/dev/null)" || return 1
+ jq -e '
+ (.spec.replicas // 0) > 0 and
+ (.status.observedGeneration // 0) >= (.metadata.generation // 0) and
+ (.status.readyReplicas // 0) == (.spec.replicas // 0) and
+ (.status.availableReplicas // 0) == (.spec.replicas // 0)
+ ' <<<"$deployment" >/dev/null
+}
+
+recipe_active() {
+ [ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \
+ -o jsonpath='{.status.phase}' 2>/dev/null)" = active ]
+}
+
+policy_matches_ports() {
+ local namespace=$1 name=$2 expected_json=$3
+ local policy
+ policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null)" || return 1
+ jq -e --arg answer "${ANSWER_IP}/32" --argjson expected "$expected_json" '
+ .metadata.labels["clerum.io/managed-by"] == "workflow-recipes" and
+ ([.spec.egress[]?.ports[]?.port] | unique | sort) == ($expected | unique | sort) and
+ ([.spec.egress[]?.to[]?.ipBlock.cidr] | unique) == [$answer] and
+ ((.metadata.annotations["clerum.io/egress-fqdn-state"] | fromjson |
+ map(.port) | unique | sort) == ($expected | unique | sort))
+ ' <<<"$policy" >/dev/null
+}
+
+policy_has_port() {
+ local namespace=$1 name=$2 port=$3
+ kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |
+ jq -e --argjson port "$port" 'any(.spec.egress[]?.ports[]?; .port == $port)' >/dev/null
+}
+
+append_policy_port() {
+ local namespace=$1 name=$2 port=$3
+ local patch
+ patch="$(jq -cn --arg cidr "${ANSWER_IP}/32" --argjson port "$port" '
+ [{op:"add",path:"/spec/egress/-",value:{
+ to:[{ipBlock:{cidr:$cidr}}],ports:[{port:$port,protocol:"TCP"}]
+ }}]
+ ')"
+ kctl patch networkpolicy "$name" -n "$namespace" --type=json -p "$patch" >/dev/null
+}
+
+workload_health() {
+ local namespace=$1 workload=$2
+ local pod
+ pod="$(ready_pod_name "$namespace" \
+ "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${workload}" 2>/dev/null)" || return 1
+ kctl exec "$pod" -n "$namespace" -- node -e '
+ fetch("http://127.0.0.1:3001/")
+ .then(async response => {
+ const body = await response.json();
+ if (response.status !== 200 || body.status !== "ok") process.exit(2);
+ })
+ .catch(() => process.exit(3));
+ ' >/dev/null 2>&1
+}
+
+dns_state() {
+ kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- node -e '
+ const http = require("node:http");
+ http.get({host:"127.0.0.1",port:8090,path:"/state"}, response => {
+ let body=""; response.on("data", chunk => body += chunk);
+ response.on("end", () => { process.stdout.write(body); process.exit(response.statusCode === 200 ? 0 : 2); });
+ }).once("error", () => process.exit(3));
+ ' 2>/dev/null
+}
+
+dns_count() {
+ local key=$1
+ dns_state | jq -er --arg key "$key" '.counts[$key]'
+}
+
+set_dns_mode() {
+ local selected=$1
+ # The JavaScript reads DNS_MODE from the explicit `env` argument; shell
+ # expansion inside the program would be a quoting bug.
+ # shellcheck disable=SC2016
+ kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- \
+ env DNS_MODE="$selected" node -e '
+ const http = require("node:http");
+ const request = http.request(
+ {host:"127.0.0.1",port:8090,path:`/mode/${process.env.DNS_MODE}`,method:"POST"},
+ response => { response.resume(); response.on("end", () => process.exit(response.statusCode === 200 ? 0 : 2)); }
+ );
+ request.setTimeout(5000, () => request.destroy());
+ request.once("error", () => process.exit(3));
+ request.end();
+ '
+}
+
+dns_count_at_least() {
+ local key=$1 minimum=$2 current
+ current="$(dns_count "$key" 2>/dev/null)" || return 1
+ [ "$current" -ge "$minimum" ]
+}
+
+acquire_lock() {
+ local created existing
+ if ! created="$(kctl create configmap "$LOCK_NAME" -n "$WRC_NS" \
+ --from-literal="holder=${RUN_ID}" \
+ --from-literal="context=${E2E_KUBECONTEXT}" \
+ --from-literal="gitHead=$(git -C "${SCRIPT_DIR}/../.." rev-parse HEAD)" \
+ -o json 2>/dev/null)"; then
+ existing="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null || true)"
+ die "another WRC egress gate owns ${WRC_NS}/${LOCK_NAME}: holder=$(jq -r '.data.holder // "unknown"' <<<"${existing:-{}}"), head=$(jq -r '.data.gitHead // "unknown"' <<<"${existing:-{}}")"
+ fi
+ LOCK_UID="$(jq -r '.metadata.uid // ""' <<<"$created")"
+ [ -n "$LOCK_UID" ] || die "created gate lock has no UID"
+ LOCK_ACQUIRED=1
+}
+
+release_lock() {
+ local current
+ [ "$LOCK_ACQUIRED" = 1 ] || return 0
+ current="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null)" || return 1
+ [ "$(jq -r '.metadata.uid // ""' <<<"$current")" = "$LOCK_UID" ] || return 1
+ [ "$(jq -r '.data.holder // ""' <<<"$current")" = "$RUN_ID" ] || return 1
+ kctl delete configmap "$LOCK_NAME" -n "$WRC_NS" --wait=true --timeout=30s >/dev/null || return 1
+ LOCK_ACQUIRED=0
+}
+
+wrc_restore_verified() {
+ local deployment
+ deployment="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json 2>/dev/null)" || return 1
+ jq -e --arg policy "$ORIGINAL_DNS_POLICY" --argjson replicas "$ORIGINAL_REPLICAS" '
+ .spec.replicas == $replicas and
+ (.status.readyReplicas // 0) == $replicas and
+ (.status.availableReplicas // 0) == $replicas and
+ (.spec.template.spec.dnsPolicy // "ClusterFirst") == $policy and
+ (.spec.template.spec.dnsConfig // null) == null
+ ' <<<"$deployment" >/dev/null
+}
+
+delete_proxy_resources() {
+ local failed=0
+ kctl delete networkpolicy "$WRC_DNS_POLICY" "$DNS_PROXY_POLICY" -n "$WRC_NS" \
+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
+ kctl delete service "$DNS_PROXY" -n "$WRC_NS" \
+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
+ kctl delete deployment "$DNS_PROXY" -n "$WRC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \
+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1
+ return "$failed"
+}
+
+cleanup() {
+ local status=$?
+ trap - EXIT INT TERM
+ local cleanup_failed=0 restore_ok=1
+
+ if [ "$PROXY_CREATED" = 1 ]; then
+ set_dns_mode ok >/dev/null 2>&1 || cleanup_failed=1
+ fi
+ if [ "$RECIPE_CREATED" = 1 ]; then
+ kctl delete workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \
+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_failed=1
+ wait_until 90 "fixture WorkflowRecipe deletion" \
+ bash -c "! kubectl --context='${E2E_KUBECONTEXT}' get workflowrecipe '${RECIPE_NAME}' -n '${RECIPE_NAMESPACE}' >/dev/null 2>&1" || cleanup_failed=1
+ fi
+ if [ "$WRC_MUTATED" = 1 ]; then
+ local restore_patch
+ restore_patch="$(jq -cn --arg policy "$ORIGINAL_DNS_POLICY" '
+ {spec:{template:{spec:{dnsPolicy:$policy,dnsConfig:null}}}}
+ ')"
+ kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic \
+ -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0
+ kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" \
+ --timeout=180s >/dev/null 2>&1 || restore_ok=0
+ wrc_restore_verified || restore_ok=0
+ fi
+ if [ "$restore_ok" = 1 ]; then
+ delete_proxy_resources || cleanup_failed=1
+ else
+ cleanup_failed=1
+ fi
+ if [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
+ release_lock || cleanup_failed=1
+ fi
+ if [ "$cleanup_failed" != 0 ]; then
+ echo "WRC egress E2E cleanup/restore failed; lock retained at ${WRC_NS}/${LOCK_NAME}." >&2
+ echo "Inspect: kubectl --context=${E2E_KUBECONTEXT} -n ${WRC_NS} get deployment ${WRC_DEPLOY} -o yaml" >&2
+ status=1
+ fi
+ exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+require_branch_owned_hcc_gate "$WRC_NS"
+acquire_lock
+
+[ -r "$DNS_PROXY_SCRIPT" ] || die "DNS proxy fixture is missing at ${DNS_PROXY_SCRIPT}"
+deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC is not Ready before fault injection"
+deployment_ready "$WRC_NS" control-api || die "control-api is not Ready before fault injection"
+deployment_ready "$WRC_NS" host-context-controller || die "HCC is not Ready before fault injection"
+
+ORIGINAL_REPLICAS="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o jsonpath='{.spec.replicas}')"
+[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected one WRC replica, found ${ORIGINAL_REPLICAS:-unknown}"
+ORIGINAL_DNS_POLICY="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |
+ jq -r '.spec.template.spec.dnsPolicy // "ClusterFirst"')"
+[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] || die "WRC already has non-default dnsPolicy"
+ORIGINAL_DNS_CONFIG="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |
+ jq -c '.spec.template.spec.dnsConfig // null')"
+[ "$ORIGINAL_DNS_CONFIG" = null ] || die "WRC already has a dnsConfig; refusing non-restorable mutation"
+WRC_IMAGE="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="workflow-recipes")].image}')"
+[ -n "$WRC_IMAGE" ] || die "could not resolve WRC image"
+UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"
+[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-dns ClusterIP"
+
+kctl create configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \
+ --from-file="server.cjs=${DNS_PROXY_SCRIPT}" --dry-run=client -o yaml |
+ kctl apply -f - >/dev/null
+PROXY_CREATED=1
+
+kctl apply -f - >/dev/null <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ${DNS_PROXY}
+ namespace: ${WRC_NS}
+ labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
+spec:
+ replicas: 1
+ selector: {matchLabels: {app: ${DNS_PROXY}}}
+ template:
+ metadata:
+ labels: {app: ${DNS_PROXY}, e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
+ spec:
+ automountServiceAccountToken: false
+ terminationGracePeriodSeconds: 1
+ containers:
+ - name: dns-proxy
+ image: ${WRC_IMAGE}
+ imagePullPolicy: IfNotPresent
+ command: [node, /fixture/server.cjs]
+ env:
+ - {name: TARGET_DNS, value: "${TARGET_DNS}"}
+ - {name: ANSWER_IP, value: "${ANSWER_IP}"}
+ - {name: UPSTREAM_DNS, value: "${UPSTREAM_DNS_IP}"}
+ ports:
+ - {name: dns-udp, containerPort: 8053, protocol: UDP}
+ - {name: dns-tcp, containerPort: 8053, protocol: TCP}
+ readinessProbe: {tcpSocket: {port: 8053}, periodSeconds: 1, failureThreshold: 10}
+ volumeMounts:
+ - {name: fixture, mountPath: /fixture, readOnly: true}
+ resources:
+ requests: {cpu: 5m, memory: 20Mi}
+ limits: {cpu: 75m, memory: 64Mi}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities: {drop: [ALL]}
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ seccompProfile: {type: RuntimeDefault}
+ volumes:
+ - name: fixture
+ configMap: {name: ${DNS_PROXY_CONFIGMAP}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ${DNS_PROXY}
+ namespace: ${WRC_NS}
+spec:
+ selector: {app: ${DNS_PROXY}}
+ ports:
+ - {name: dns-udp, protocol: UDP, port: 53, targetPort: 8053}
+ - {name: dns-tcp, protocol: TCP, port: 53, targetPort: 8053}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${DNS_PROXY_POLICY}
+ namespace: ${WRC_NS}
+spec:
+ podSelector: {matchLabels: {app: ${DNS_PROXY}}}
+ policyTypes: [Ingress, Egress]
+ ingress:
+ - from: [{podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}}]
+ ports:
+ - {protocol: UDP, port: 53}
+ - {protocol: UDP, port: 8053}
+ - {protocol: TCP, port: 53}
+ - {protocol: TCP, port: 8053}
+ egress:
+ - to:
+ - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}
+ podSelector: {matchLabels: {k8s-app: kube-dns}}
+ ports:
+ - {protocol: UDP, port: 53}
+ - {protocol: TCP, port: 53}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${WRC_DNS_POLICY}
+ namespace: ${WRC_NS}
+spec:
+ podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}
+ policyTypes: [Egress]
+ egress:
+ - to: [{podSelector: {matchLabels: {app: ${DNS_PROXY}}}}]
+ ports:
+ - {protocol: UDP, port: 53}
+ - {protocol: UDP, port: 8053}
+ - {protocol: TCP, port: 53}
+ - {protocol: TCP, port: 8053}
+EOF
+
+kctl rollout status deployment "$DNS_PROXY" -n "$WRC_NS" --timeout=90s >/dev/null ||
+ die "DNS proxy did not become Ready"
+wait_until 30 "DNS proxy listener" \
+ bash -c "kubectl --context='${E2E_KUBECONTEXT}' logs deployment/'${DNS_PROXY}' -n '${WRC_NS}' -c dns-proxy | grep -Fq 'wrc egress DNS proxy ready'" ||
+ die "DNS proxy did not report readiness"
+DNS_PROXY_IP="$(kctl get service "$DNS_PROXY" -n "$WRC_NS" -o jsonpath='{.spec.clusterIP}')"
+[ -n "$DNS_PROXY_IP" ] || die "DNS proxy Service has no ClusterIP"
+
+WRC_MUTATED=1
+dns_patch="$(jq -cn --arg nameserver "$DNS_PROXY_IP" '
+ {spec:{template:{spec:{
+ dnsPolicy:"None",
+ dnsConfig:{nameservers:[$nameserver],options:[
+ {name:"ndots",value:"1"},{name:"timeout",value:"2"},{name:"attempts",value:"1"}
+ ]}
+ }}}}
+')"
+kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic -p "$dns_patch" >/dev/null
+kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" --timeout=180s >/dev/null ||
+ die "WRC did not become Ready with the isolated DNS proxy"
+deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC DNS setup"
+deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC DNS setup"
+ok "isolated WRC DNS proxy is Ready without affecting control-api or HCC"
+
+RECIPE_CREATED=1
+kctl apply -f - >/dev/null <<EOF
+apiVersion: clerum.io/v1alpha1
+kind: WorkflowRecipe
+metadata:
+ name: ${RECIPE_NAME}
+ namespace: ${RECIPE_NAMESPACE}
+ labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}
+spec:
+ description: WRC egress degradation E2E baseline
+ contextRef: context1
+ workloads:
+ - id: frontend
+ type: deployment
+ image: clerum/mock-mcp-server:test
+ port: 3001
+ healthCheck: {type: http, path: /, port: 3001}
+ - id: worker
+ type: deployment
+ image: clerum/mock-mcp-server:test
+ port: 3001
+ healthCheck: {type: http, path: /, port: 3001}
+ egressBindings:
+ - {dns: ${TARGET_DNS}, port: 443, protocol: TCP}
+ - {dns: ${TARGET_DNS}, port: 8443, protocol: TCP}
+ ui:
+ workloadRef: frontend
+ port: 3001
+ title: WRC egress degradation fixture
+ defaultPath: /
+ egress:
+ external:
+ - {fqdn: ${TARGET_DNS}, port: 443}
+ - {fqdn: ${TARGET_DNS}, port: 8443}
+ security: {isolationLevel: minimal}
+EOF
+
+wait_until 180 "valid fixture recipe to become active" recipe_active ||
+ die "fixture recipe did not become active"
+wait_until 120 "initial UI egress policy" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443,8443]' ||
+ die "initial UI policy did not enforce both declared ports"
+wait_until 120 "initial workload egress policy" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443,8443]' ||
+ die "initial workload policy did not enforce both declared ports"
+wait_until 120 "frontend business health" workload_health "$UI_NS" frontend ||
+ die "frontend health endpoint was unavailable"
+wait_until 120 "worker business health" workload_health "$RECIPE_NAMESPACE" worker ||
+ die "worker health endpoint was unavailable"
+ok "valid recipe is active, both workload health endpoints respond, and both policies enforce the declared set"
+
+hold_before="$(dns_count hold)"
+set_dns_mode hold || die "could not hold fixture DNS"
+reduced_patch="$(jq -cn --arg dns "$TARGET_DNS" '
+ [
+ {op:"replace",path:"/spec/ui/egress/external",value:[{fqdn:$dns,port:443}]},
+ {op:"replace",path:"/spec/workloads/1/egressBindings",value:[{dns:$dns,port:443,protocol:"TCP"}]}
+ ]
+')"
+kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=json \
+ -p "$reduced_patch" >/dev/null
+wait_until 60 "WRC DNS query to enter the deterministic hold" \
+ dns_count_at_least hold "$((hold_before + 1))" || die "WRC never reached the DNS hold"
+wait_until 60 "pre-DNS UI contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||
+ die "UI port 8443 was not revoked before DNS completed"
+wait_until 60 "pre-DNS workload contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
+ die "workload port 8443 was not revoked before DNS completed"
+recipe_active || die "correctly configured recipe left active before the held DNS completed"
+workload_health "$UI_NS" frontend || die "frontend stopped serving during held DNS"
+workload_health "$RECIPE_NAMESPACE" worker || die "worker stopped serving during held DNS"
+ok "removed UI/workload permissions contracted before DNS completed while both services stayed healthy"
+
+append_policy_port "$UI_NS" "$UI_POLICY" 9443
+append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443
+policy_has_port "$UI_NS" "$UI_POLICY" 9443 || die "could not inject UI policy race"
+policy_has_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443 || die "could not inject workload policy race"
+
+servfail_before="$(dns_count servfail)"
+set_dns_mode servfail || die "could not release held DNS as SERVFAIL"
+wait_until 90 "both WRC lanes to observe SERVFAIL" \
+ dns_count_at_least servfail "$((servfail_before + 2))" ||
+ die "UI and workload lanes did not both observe the transient DNS failure"
+wait_until 90 "post-DNS UI re-contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||
+ die "UI race widening survived transient DNS failure"
+wait_until 90 "post-DNS workload re-contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
+ die "workload race widening survived transient DNS failure"
+wait_until 60 "recipe to remain active after proven fail-static" recipe_active ||
+ die "correctly configured recipe was cut by transient DNS failure"
+workload_health "$UI_NS" frontend || die "frontend health failed after transient DNS failure"
+workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after transient DNS failure"
+deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC lost readiness during fault"
+deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC fault"
+deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC fault"
+ok "transient DNS failure removed concurrent widening, preserved proven port 443, and isolated blast radius"
+
+ok_before="$(dns_count ok)"
+set_dns_mode ok || die "could not restore successful DNS"
+recovery_patch="$(jq -cn --arg description "WRC egress recovery ${RUN_ID}" \
+ '{spec:{description:$description}}')"
+kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=merge \
+ -p "$recovery_patch" >/dev/null
+wait_until 90 "both WRC lanes to resolve after recovery" \
+ dns_count_at_least ok "$((ok_before + 2))" || die "DNS recovery was not observed by both lanes"
+wait_until 90 "recovered recipe active state" recipe_active || die "recipe did not recover active"
+policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' || die "UI policy drifted after recovery"
+policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||
+ die "workload policy drifted after recovery"
+workload_health "$UI_NS" frontend || die "frontend health failed after DNS recovery"
+workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after DNS recovery"
+ok "successful DNS recovery reconverged both lanes without restoring removed or raced permissions"
+
+echo "WRC_EGRESS_DEGRADATION_E2E_PASS"
+echo "recipe=${RECIPE_NAMESPACE}/${RECIPE_NAME}"
+echo "uiPolicy=${UI_NS}/${UI_POLICY}"
+echo "workloadPolicy=${RECIPE_NAMESPACE}/${WORKLOAD_POLICY}"
diff --git a/scripts/tests/test-wrc-egress-degradation-gate.sh b/scripts/tests/test-wrc-egress-degradation-gate.sh
new file mode 100755
index 000000000..5b4a8118e
--- /dev/null
+++ b/scripts/tests/test-wrc-egress-degradation-gate.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Literal source-contract patterns intentionally contain shell-looking text.
+# shellcheck disable=SC2016
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"
+FIXTURE="${ROOT}/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"
+MAKEFILE="${ROOT}/Makefile"
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+contains() {
+ local file=$1 pattern=$2 description=$3
+ grep -Fq -- "$pattern" "$file" || fail "$description"
+}
+
+bash -n "$GATE"
+node --check "$FIXTURE" >/dev/null
+
+contains "$GATE" 'is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"' \
+ 'gate must refuse non-branch Kubernetes contexts'
+contains "$GATE" 'E2E_WRC_EGRESS_FAULT_INJECTION' \
+ 'gate must require explicit fault-injection acknowledgement'
+contains "$GATE" 'require_branch_owned_hcc_gate "$WRC_NS"' \
+ 'gate must verify exact HEAD, worktree, profile, fingerprint, and pre-gate marker'
+contains "$GATE" 'acquire_lock' 'gate must acquire exclusive WRC fault-injection ownership'
+contains "$GATE" 'wrc_restore_verified' 'gate must prove WRC DNS configuration restoration'
+contains "$GATE" 'lock retained' 'failed restoration must retain the coordination lock'
+
+contains "$GATE" 'id: frontend' 'fixture must exercise the sandbox UI workload'
+contains "$GATE" 'id: worker' 'fixture must exercise a non-UI workload'
+contains "$GATE" 'port: 8443' 'fixture must begin with a permission that will be removed'
+contains "$GATE" 'set_dns_mode hold' 'gate must create a real DNS latency window'
+contains "$GATE" 'pre-DNS UI contraction' 'gate must assert UI contraction before DNS completes'
+contains "$GATE" 'pre-DNS workload contraction' \
+ 'gate must assert workload contraction before DNS completes'
+contains "$GATE" 'append_policy_port "$UI_NS" "$UI_POLICY" 9443' \
+ 'gate must inject live UI drift during DNS latency'
+contains "$GATE" 'append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443' \
+ 'gate must inject live workload drift during DNS latency'
+contains "$GATE" 'set_dns_mode servfail' 'gate must produce a real retryable resolver response'
+contains "$GATE" 'post-DNS UI re-contraction' \
+ 'gate must assert the UI fresh-read contraction after DNS failure'
+contains "$GATE" 'post-DNS workload re-contraction' \
+ 'gate must assert the workload fresh-read contraction after DNS failure'
+
+contains "$GATE" 'workload_health "$UI_NS" frontend' \
+ 'gate must prove the UI workload still serves its health endpoint'
+contains "$GATE" 'workload_health "$RECIPE_NAMESPACE" worker' \
+ 'gate must prove the workload still serves its health endpoint'
+contains "$GATE" 'deployment_ready "$WRC_NS" control-api' \
+ 'gate must assert an unrelated production service remains Ready'
+contains "$GATE" 'deployment_ready "$WRC_NS" host-context-controller' \
+ 'gate must assert HCC remains Ready'
+contains "$GATE" 'set_dns_mode ok' 'gate must restore successful DNS'
+contains "$GATE" 'WRC_EGRESS_DEGRADATION_E2E_PASS' \
+ 'gate must emit an unambiguous terminal success marker'
+
+contains "$FIXTURE" 'match(/^\/mode\/(ok|hold|servfail)$/)' \
+ 'DNS fixture must expose exactly the required deterministic modes'
+contains "$FIXTURE" '0x8180 | rcode' 'DNS fixture must emit an actual SERVFAIL response'
+contains "$FIXTURE" 'forwardUdp' 'DNS fixture must forward unrelated UDP queries'
+contains "$FIXTURE" 'forwardTcp' 'DNS fixture must forward unrelated TCP queries'
+
+contains "$MAKEFILE" '.PHONY: test-e2e-wrc-egress-degradation' \
+ 'Makefile must expose the WRC egress degradation gate'
+contains "$MAKEFILE" 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh' \
+ 'Make target must bind the mutation to the explicit profile/context'
+contains "$MAKEFILE" '.PHONY: test-e2e-pr567-egress-resilience' \
+ 'Makefile must expose the complete PR #567 resilience gate'
+contains "$MAKEFILE" '$(MAKE) test-e2e-hcc-mcp-context-readiness' \
+ 'complete PR gate must run the existing real HCC DNS/runtime journey'
+contains "$MAKEFILE" '$(MAKE) test-e2e-wrc-egress-degradation' \
+ 'complete PR gate must run the WRC UI/workload degradation journey'
+
+if grep -Eq 'kubectl[[:space:]]+(get|apply|patch|delete|exec|logs)' "$GATE"; then
+ fail 'raw kubectl operations must use kctl or an explicit --context cleanup probe'
+fi
+if grep -Fq -- '--force' "$GATE"; then
+ fail 'gate must not force Kubernetes mutations'
+fi
+
+echo 'PASS: WRC egress degradation E2E guard contract'
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index b27a139fb..47afd8db2 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.68",
+ "version": "1.0.69",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.68",
+ "version": "1.0.69",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index 8b093d1d3..c71bf0258 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.68",
+ "version": "1.0.69",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
new file mode 100644
index 000000000..1bbd136e3
--- /dev/null
+++ b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs
@@ -0,0 +1,183 @@
+'use strict'
+
+const dgram = require('node:dgram')
+const http = require('node:http')
+const net = require('node:net')
+
+const target = String(process.env.TARGET_DNS || '').toLowerCase()
+const upstream = process.env.UPSTREAM_DNS
+const answerIp = String(process.env.ANSWER_IP || '93.184.216.34')
+const answerOctets = answerIp.split('.').map(Number)
+
+if (!target || !upstream || answerOctets.length !== 4 || answerOctets.some(Number.isNaN)) {
+ throw new Error('TARGET_DNS, UPSTREAM_DNS, and a valid IPv4 ANSWER_IP are required')
+}
+
+let mode = 'ok'
+const held = []
+const counts = { ok: 0, hold: 0, servfail: 0, forwarded: 0 }
+const udp = dgram.createSocket('udp4')
+
+function question(message) {
+ let offset = 12
+ const labels = []
+ while (offset < message.length) {
+ const length = message[offset++]
+ if (length === 0) break
+ if (offset + length > message.length) return null
+ labels.push(message.subarray(offset, offset + length).toString('ascii'))
+ offset += length
+ }
+ if (offset + 4 > message.length) return null
+ return {
+ name: labels.join('.').toLowerCase(),
+ type: message.readUInt16BE(offset),
+ end: offset + 4,
+ }
+}
+
+function dnsResponse(message, parsed, rcode) {
+ const header = Buffer.from(message.subarray(0, 12))
+ header.writeUInt16BE(0x8180 | rcode, 2)
+ header.writeUInt16BE(rcode === 0 ? 1 : 0, 6)
+ header.writeUInt16BE(0, 8)
+ header.writeUInt16BE(0, 10)
+ const questionBytes = message.subarray(12, parsed.end)
+ if (rcode !== 0) return Buffer.concat([header, questionBytes])
+ const record = Buffer.from([
+ 0xc0,
+ 0x0c,
+ 0x00,
+ 0x01,
+ 0x00,
+ 0x01,
+ 0x00,
+ 0x00,
+ 0x01,
+ 0x2c,
+ 0x00,
+ 0x04,
+ ...answerOctets,
+ ])
+ return Buffer.concat([header, questionBytes, record])
+}
+
+function tcpFrame(message) {
+ const frame = Buffer.alloc(message.length + 2)
+ frame.writeUInt16BE(message.length, 0)
+ message.copy(frame, 2)
+ return frame
+}
+
+function send(entry, selectedMode) {
+ const response = dnsResponse(entry.message, entry.parsed, selectedMode === 'ok' ? 0 : 2)
+ counts[selectedMode]++
+ if (entry.transport === 'udp') {
+ udp.send(response, entry.rinfo.port, entry.rinfo.address)
+ } else if (!entry.socket.destroyed) {
+ entry.socket.write(tcpFrame(response))
+ }
+ console.log(`${selectedMode} DNS A ${target} via ${entry.transport.toUpperCase()}`)
+}
+
+function forwardUdp(message, rinfo) {
+ counts.forwarded++
+ const relay = dgram.createSocket('udp4')
+ const timer = setTimeout(() => relay.close(), 5000)
+ relay.once('message', response => {
+ clearTimeout(timer)
+ udp.send(response, rinfo.port, rinfo.address)
+ relay.close()
+ })
+ relay.once('error', () => {
+ clearTimeout(timer)
+ relay.close()
+ })
+ relay.send(message, 53, upstream)
+}
+
+function forwardTcp(message, socket) {
+ counts.forwarded++
+ const relay = net.createConnection({ host: upstream, port: 53 }, () =>
+ relay.write(tcpFrame(message))
+ )
+ let pending = Buffer.alloc(0)
+ relay.setTimeout(5000, () => relay.destroy())
+ relay.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ if (pending.length < 2) return
+ const length = pending.readUInt16BE(0)
+ if (pending.length < length + 2) return
+ if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))
+ relay.destroy()
+ })
+ relay.once('error', () => relay.destroy())
+}
+
+function handle(message, entry) {
+ const parsed = question(message)
+ if (parsed && parsed.name === target && parsed.type === 1) {
+ entry.message = message
+ entry.parsed = parsed
+ if (mode === 'hold') {
+ counts.hold++
+ held.push(entry)
+ console.log(`hold DNS A ${target} via ${entry.transport.toUpperCase()}`)
+ } else {
+ send(entry, mode)
+ }
+ return
+ }
+ if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)
+ else forwardTcp(message, entry.socket)
+}
+
+function setMode(nextMode) {
+ mode = nextMode
+ if (mode !== 'hold') {
+ while (held.length > 0) send(held.shift(), mode)
+ }
+}
+
+let listeners = 0
+function listening() {
+ listeners++
+ if (listeners === 2) console.log(`wrc egress DNS proxy ready for ${target}`)
+}
+
+udp.on('message', (message, rinfo) => handle(message, { transport: 'udp', rinfo }))
+udp.bind(8053, '0.0.0.0', listening)
+
+const tcp = net.createServer(socket => {
+ let pending = Buffer.alloc(0)
+ socket.on('data', chunk => {
+ pending = Buffer.concat([pending, chunk])
+ while (pending.length >= 2) {
+ const length = pending.readUInt16BE(0)
+ if (pending.length < length + 2) return
+ const message = pending.subarray(2, length + 2)
+ pending = pending.subarray(length + 2)
+ handle(message, { transport: 'tcp', socket })
+ }
+ })
+})
+tcp.listen(8053, '0.0.0.0', listening)
+
+http
+ .createServer((request, response) => {
+ const nextMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]
+ if (request.method === 'POST' && nextMode) {
+ setMode(nextMode)
+ response.writeHead(200, { 'content-type': 'application/json' })
+ response.end(JSON.stringify({ mode, held: held.length, counts }))
+ return
+ }
+ if (request.method === 'GET' && request.url === '/state') {
+ response.writeHead(200, { 'content-type': 'application/json' })
+ response.end(JSON.stringify({ mode, held: held.length, counts }))
+ return
+ }
+ response.writeHead(404)
+ response.end()
+ })
+ .listen(8090, '127.0.0.1')
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index e63ee916f..543d4a0a6 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.343",
+ "version": "0.1.344",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.343",
+ "version": "0.1.344",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 18fc8a122..e2b3242cc 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.343",
+ "version": "0.1.344",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index c22ac5028..5891f489c 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -2927,6 +2927,127 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ it.each([
+ {
+ resolverLabel: 'controller fault',
+ resolverResult: 'throw' as const,
+ label: 'retryable API outage',
+ contractionError: () =>
+ Object.assign(new Error('workload re-contraction unavailable'), { code: 503 }),
+ expectedPhase: 'active' as const,
+ retryable: true,
+ },
+ {
+ resolverLabel: 'controller fault',
+ resolverResult: 'throw' as const,
+ label: 'non-retryable identity conflict',
+ contractionError: () =>
+ Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),
+ expectedPhase: 'failed' as const,
+ retryable: false,
+ },
+ {
+ resolverLabel: 'transient DNS result',
+ resolverResult: 'transient' as const,
+ label: 'retryable API outage',
+ contractionError: () =>
+ Object.assign(new Error('workload re-contraction unavailable'), { code: 503 }),
+ expectedPhase: 'active' as const,
+ retryable: true,
+ },
+ {
+ resolverLabel: 'transient DNS result',
+ resolverResult: 'transient' as const,
+ label: 'non-retryable identity conflict',
+ contractionError: () =>
+ Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),
+ expectedPhase: 'failed' as const,
+ retryable: false,
+ },
+ ])(
+ '#567 remediation: workload $resolverLabel plus $label preserves fail-closed service semantics',
+ async ({ contractionError, expectedPhase, resolverResult, retryable }) => {
+ const policyName = 'wl-egress-test-recipe-worker'
+ let resolverFault = false
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ if (resolverFault) {
+ if (resolverResult === 'transient') {
+ return { kind: 'error', error: 'ESERVFAIL', retryable: true }
+ }
+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
+ }
+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }
+ },
+ })
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:1',
+ port: 8080,
+ egressBindings: [{ dns: 'current.example.com', port: 443 }],
+ },
+ ],
+ },
+ })
+ await rec.reconcile(recipe)
+ const written = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => call[0]?.body as k8s.V1NetworkPolicy)
+ .find(body => body.metadata?.name === policyName)
+ expect(written).toBeDefined()
+ const live: k8s.V1NetworkPolicy = {
+ ...structuredClone(written!),
+ metadata: {
+ ...written!.metadata,
+ name: policyName,
+ namespace: 'sandbox-recipes',
+ uid: 'uid-workload-double-fault',
+ resourceVersion: '70',
+ },
+ }
+ let targetReads = 0
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ if (name !== policyName) return Promise.reject({ code: 404 })
+ targetReads++
+ return targetReads === 1 ? Promise.resolve(live) : Promise.reject(contractionError())
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockClear()
+ resolverFault = true
+
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+
+ expect(targetReads).toBe(2)
+ expect(result.phase).toBe(expectedPhase)
+ if (retryable) {
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ } else {
+ expect(result.skipStatusPatch).not.toBe(true)
+ expect(result.message).toContain('workload re-contraction conflict')
+ }
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ metadata: expect.objectContaining({ name: policyName }),
+ }),
+ })
+ )
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ }
+ )
+
it('returns failed phase when a workload egress binding uses wildcard DNS', async () => {
const recipe = makeRecipe({
spec: {
@@ -4665,6 +4786,111 @@ describe('WorkflowRecipeReconciler', () => {
}
)
+ it.each([
+ {
+ resolverLabel: 'controller fault',
+ resolverResult: 'throw' as const,
+ label: 'retryable API outage',
+ contractionError: () =>
+ Object.assign(new Error('UI re-contraction unavailable'), { code: 503 }),
+ expectedPhase: 'active' as const,
+ retryable: true,
+ },
+ {
+ resolverLabel: 'controller fault',
+ resolverResult: 'throw' as const,
+ label: 'non-retryable identity conflict',
+ contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),
+ expectedPhase: 'failed' as const,
+ retryable: false,
+ },
+ {
+ resolverLabel: 'transient DNS result',
+ resolverResult: 'transient' as const,
+ label: 'retryable API outage',
+ contractionError: () =>
+ Object.assign(new Error('UI re-contraction unavailable'), { code: 503 }),
+ expectedPhase: 'active' as const,
+ retryable: true,
+ },
+ {
+ resolverLabel: 'transient DNS result',
+ resolverResult: 'transient' as const,
+ label: 'non-retryable identity conflict',
+ contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),
+ expectedPhase: 'failed' as const,
+ retryable: false,
+ },
+ ])(
+ '#567 remediation: UI $resolverLabel plus $label preserves fail-closed service semantics',
+ async ({ contractionError, expectedPhase, resolverResult, retryable }) => {
+ const policyName = 'ui-egress-test-recipe'
+ let resolverFault = false
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => {
+ if (resolverFault) {
+ if (resolverResult === 'transient') {
+ return { kind: 'error', error: 'ESERVFAIL', retryable: true }
+ }
+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })
+ }
+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }
+ },
+ })
+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])
+ await rec.reconcile(recipe)
+ const written = createdPolicy(policyName)
+ expect(written).toBeDefined()
+ const live: k8s.V1NetworkPolicy = {
+ ...structuredClone(written!),
+ metadata: {
+ ...written!.metadata,
+ name: policyName,
+ namespace: 'sandbox-ui',
+ uid: 'uid-ui-double-fault',
+ resourceVersion: '80',
+ },
+ }
+ let targetReads = 0
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ if (name !== policyName) return Promise.reject({ code: 404 })
+ targetReads++
+ return targetReads === 1 ? Promise.resolve(live) : Promise.reject(contractionError())
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockClear()
+ resolverFault = true
+
+ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })
+
+ expect(targetReads).toBe(2)
+ expect(result.phase).toBe(expectedPhase)
+ if (retryable) {
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ } else {
+ expect(result.skipStatusPatch).not.toBe(true)
+ expect(result.message).toContain('UI re-contraction conflict')
+ }
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ metadata: expect.objectContaining({ name: policyName }),
+ }),
+ })
+ )
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: policyName })
+ )
+ }
+ )
+
it('#567 remediation: persists the prior resolved-at when a permanent failure narrows the policy', async () => {
let permanentFailure = false
let live: k8s.V1NetworkPolicy | undefined