Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(egress): cover service degradation paths
Alfredo Lopezee3596867c16
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex b77601d73..ac616d14e 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -461,6 +461,9 @@ jobs:- name: Test WRC internal dependency NetworkPolicy contractsrun: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh+ - name: Test WRC external-egress degradation gate contract+ run: bash scripts/tests/test-wrc-egress-degradation-gate.sh+- name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)run: bash scripts/tests/test-lint-networkpolicies.shdiff --git a/Makefile b/Makefileindex 487f3903f..c7d9fc1e7 100644--- a/Makefile+++ b/Makefile@@ -1261,6 +1261,18 @@ test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC w@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh+.PHONY: test-e2e-wrc-egress-degradation+test-e2e-wrc-egress-degradation: ## Prove valid UI/workload services survive transient DNS while removed/raced egress is contracted+ @echo "Running WRC external-egress degradation gate..."+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_WRC_EGRESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh++.PHONY: test-e2e-pr567-egress-resilience+test-e2e-pr567-egress-resilience: ## Run the exact-head HCC + WRC external-egress resilience journeys sequentially+ @echo "Running PR #567 external-egress resilience gates..."+ $(MAKE) test-e2e-hcc-mcp-context-readiness E2E_KUBECONTEXT=$(E2E_KUBECONTEXT) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"+ $(MAKE) test-e2e-wrc-egress-degradation E2E_KUBECONTEXT=$(E2E_KUBECONTEXT) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"+.PHONY: test-e2e-hcc-readiness-bootstraptest-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active@echo "Running HCC initial-fleet readiness gate..."diff --git a/scripts/e2e/e2e-wrc-egress-degradation.sh b/scripts/e2e/e2e-wrc-egress-degradation.shnew file mode 100755index 000000000..bc1b3d66c--- /dev/null+++ b/scripts/e2e/e2e-wrc-egress-degradation.sh@@ -0,0 +1,538 @@+#!/usr/bin/env bash+# Proves that a correctly configured, active WorkflowRecipe stays available+# while WRC contracts removed UI/workload egress authorization across a real+# transient DNS failure. The fixture also widens both live policies while DNS+# is held and proves the post-DNS fresh-read contraction removes that drift+# before the reconcile completes.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+# shellcheck source=scripts/e2e/e2e-lib.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/e2e-lib.sh"+# Reuse the exact-head/profile/fingerprint validator used by the disruptive HCC+# gates. Despite its historical name, this helper performs no HCC mutation.+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"++[ -n "$E2E_KUBECONTEXT" ] || {+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch profile." >&2+ exit 1+}+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {+ echo "Refusing WRC fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2+ exit 1+}+require_safe_kube_context+[ "${E2E_WRC_EGRESS_FAULT_INJECTION:-0}" = 1 ] || {+ echo "Set E2E_WRC_EGRESS_FAULT_INJECTION=1 to acknowledge reversible WRC DNS fault injection." >&2+ exit 1+}+command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }+command -v node >/dev/null 2>&1 || { echo "node is required" >&2; exit 1; }++WRC_NS="${WRC_NAMESPACE:-control-plane}"+WRC_DEPLOY="${WRC_DEPLOYMENT:-workflow-recipes}"+UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"+RECIPE_NAMESPACE="${WORKFLOW_RECIPE_NS:-sandbox-recipes}"+RUN_ID="$(date +%s)-$$"+RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"+SUITE_NAME=wrc-egress-degradation+RECIPE_NAME="$(truncate_rfc1123 "e2e-wrc-egress-${RUN_ID}")"+TARGET_DNS="$(truncate_rfc1123 "wrc-egress-${RUN_ID}").example.com"+ANSWER_IP=93.184.216.34+DNS_PROXY="$(truncate_rfc1123 "e2e-wrc-dns-${RUN_ID}")"+DNS_PROXY_CONFIGMAP="$(truncate_rfc1123 "${DNS_PROXY}-script")"+DNS_PROXY_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-policy")"+WRC_DNS_POLICY="$(truncate_rfc1123 "${DNS_PROXY}-from-wrc")"+LOCK_NAME="e2e-wrc-egress-degradation-lock"+UI_POLICY="ui-egress-${RECIPE_NAME}"+WORKLOAD_POLICY="wl-egress-${RECIPE_NAME}-worker"+DNS_PROXY_SCRIPT="${SCRIPT_DIR}/../../tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"++LOCK_UID=""+LOCK_ACQUIRED=0+PROXY_CREATED=0+WRC_MUTATED=0+RECIPE_CREATED=0+ORIGINAL_DNS_POLICY=""+ORIGINAL_DNS_CONFIG=""+ORIGINAL_REPLICAS=""+WRC_IMAGE=""+UPSTREAM_DNS_IP=""++die() {+ echo "[wrc-egress-degradation] ERROR: $*" >&2+ if [ "$PROXY_CREATED" = 1 ]; then+ kctl logs "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy --tail=120 >&2 || true+ fi+ exit 1+}++wait_until() {+ local timeout=$1 description=$2+ shift 2+ local deadline=$(( $(date +%s) + timeout ))+ while [ "$(date +%s)" -lt "$deadline" ]; do+ "$@" && return 0+ sleep 1+ done+ echo "Timed out after ${timeout}s waiting for ${description}" >&2+ return 1+}++deployment_ready() {+ local namespace=$1 name=$2+ local deployment+ deployment="$(kctl get deployment "$name" -n "$namespace" -o json 2>/dev/null)" || return 1+ jq -e '+ (.spec.replicas // 0) > 0 and+ (.status.observedGeneration // 0) >= (.metadata.generation // 0) and+ (.status.readyReplicas // 0) == (.spec.replicas // 0) and+ (.status.availableReplicas // 0) == (.spec.replicas // 0)+ ' <<<"$deployment" >/dev/null+}++recipe_active() {+ [ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \+ -o jsonpath='{.status.phase}' 2>/dev/null)" = active ]+}++policy_matches_ports() {+ local namespace=$1 name=$2 expected_json=$3+ local policy+ policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null)" || return 1+ jq -e --arg answer "${ANSWER_IP}/32" --argjson expected "$expected_json" '+ .metadata.labels["clerum.io/managed-by"] == "workflow-recipes" and+ ([.spec.egress[]?.ports[]?.port] | unique | sort) == ($expected | unique | sort) and+ ([.spec.egress[]?.to[]?.ipBlock.cidr] | unique) == [$answer] and+ ((.metadata.annotations["clerum.io/egress-fqdn-state"] | fromjson |+ map(.port) | unique | sort) == ($expected | unique | sort))+ ' <<<"$policy" >/dev/null+}++policy_has_port() {+ local namespace=$1 name=$2 port=$3+ kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |+ jq -e --argjson port "$port" 'any(.spec.egress[]?.ports[]?; .port == $port)' >/dev/null+}++append_policy_port() {+ local namespace=$1 name=$2 port=$3+ local patch+ patch="$(jq -cn --arg cidr "${ANSWER_IP}/32" --argjson port "$port" '+ [{op:"add",path:"/spec/egress/-",value:{+ to:[{ipBlock:{cidr:$cidr}}],ports:[{port:$port,protocol:"TCP"}]+ }}]+ ')"+ kctl patch networkpolicy "$name" -n "$namespace" --type=json -p "$patch" >/dev/null+}++workload_health() {+ local namespace=$1 workload=$2+ local pod+ pod="$(ready_pod_name "$namespace" \+ "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${workload}" 2>/dev/null)" || return 1+ kctl exec "$pod" -n "$namespace" -- node -e '+ fetch("http://127.0.0.1:3001/")+ .then(async response => {+ const body = await response.json();+ if (response.status !== 200 || body.status !== "ok") process.exit(2);+ })+ .catch(() => process.exit(3));+ ' >/dev/null 2>&1+}++dns_state() {+ kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- node -e '+ const http = require("node:http");+ http.get({host:"127.0.0.1",port:8090,path:"/state"}, response => {+ let body=""; response.on("data", chunk => body += chunk);+ response.on("end", () => { process.stdout.write(body); process.exit(response.statusCode === 200 ? 0 : 2); });+ }).once("error", () => process.exit(3));+ ' 2>/dev/null+}++dns_count() {+ local key=$1+ dns_state | jq -er --arg key "$key" '.counts[$key]'+}++set_dns_mode() {+ local selected=$1+ # The JavaScript reads DNS_MODE from the explicit `env` argument; shell+ # expansion inside the program would be a quoting bug.+ # shellcheck disable=SC2016+ kctl exec "deployment/${DNS_PROXY}" -n "$WRC_NS" -c dns-proxy -- \+ env DNS_MODE="$selected" node -e '+ const http = require("node:http");+ const request = http.request(+ {host:"127.0.0.1",port:8090,path:`/mode/${process.env.DNS_MODE}`,method:"POST"},+ response => { response.resume(); response.on("end", () => process.exit(response.statusCode === 200 ? 0 : 2)); }+ );+ request.setTimeout(5000, () => request.destroy());+ request.once("error", () => process.exit(3));+ request.end();+ '+}++dns_count_at_least() {+ local key=$1 minimum=$2 current+ current="$(dns_count "$key" 2>/dev/null)" || return 1+ [ "$current" -ge "$minimum" ]+}++acquire_lock() {+ local created existing+ if ! created="$(kctl create configmap "$LOCK_NAME" -n "$WRC_NS" \+ --from-literal="holder=${RUN_ID}" \+ --from-literal="context=${E2E_KUBECONTEXT}" \+ --from-literal="gitHead=$(git -C "${SCRIPT_DIR}/../.." rev-parse HEAD)" \+ -o json 2>/dev/null)"; then+ existing="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null || true)"+ die "another WRC egress gate owns ${WRC_NS}/${LOCK_NAME}: holder=$(jq -r '.data.holder // "unknown"' <<<"${existing:-{}}"), head=$(jq -r '.data.gitHead // "unknown"' <<<"${existing:-{}}")"+ fi+ LOCK_UID="$(jq -r '.metadata.uid // ""' <<<"$created")"+ [ -n "$LOCK_UID" ] || die "created gate lock has no UID"+ LOCK_ACQUIRED=1+}++release_lock() {+ local current+ [ "$LOCK_ACQUIRED" = 1 ] || return 0+ current="$(kctl get configmap "$LOCK_NAME" -n "$WRC_NS" -o json 2>/dev/null)" || return 1+ [ "$(jq -r '.metadata.uid // ""' <<<"$current")" = "$LOCK_UID" ] || return 1+ [ "$(jq -r '.data.holder // ""' <<<"$current")" = "$RUN_ID" ] || return 1+ kctl delete configmap "$LOCK_NAME" -n "$WRC_NS" --wait=true --timeout=30s >/dev/null || return 1+ LOCK_ACQUIRED=0+}++wrc_restore_verified() {+ local deployment+ deployment="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json 2>/dev/null)" || return 1+ jq -e --arg policy "$ORIGINAL_DNS_POLICY" --argjson replicas "$ORIGINAL_REPLICAS" '+ .spec.replicas == $replicas and+ (.status.readyReplicas // 0) == $replicas and+ (.status.availableReplicas // 0) == $replicas and+ (.spec.template.spec.dnsPolicy // "ClusterFirst") == $policy and+ (.spec.template.spec.dnsConfig // null) == null+ ' <<<"$deployment" >/dev/null+}++delete_proxy_resources() {+ local failed=0+ kctl delete networkpolicy "$WRC_DNS_POLICY" "$DNS_PROXY_POLICY" -n "$WRC_NS" \+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1+ kctl delete service "$DNS_PROXY" -n "$WRC_NS" \+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1+ kctl delete deployment "$DNS_PROXY" -n "$WRC_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ kctl delete configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \+ --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || failed=1+ return "$failed"+}++cleanup() {+ local status=$?+ trap - EXIT INT TERM+ local cleanup_failed=0 restore_ok=1++ if [ "$PROXY_CREATED" = 1 ]; then+ set_dns_mode ok >/dev/null 2>&1 || cleanup_failed=1+ fi+ if [ "$RECIPE_CREATED" = 1 ]; then+ kctl delete workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" \+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_failed=1+ wait_until 90 "fixture WorkflowRecipe deletion" \+ bash -c "! kubectl --context='${E2E_KUBECONTEXT}' get workflowrecipe '${RECIPE_NAME}' -n '${RECIPE_NAMESPACE}' >/dev/null 2>&1" || cleanup_failed=1+ fi+ if [ "$WRC_MUTATED" = 1 ]; then+ local restore_patch+ restore_patch="$(jq -cn --arg policy "$ORIGINAL_DNS_POLICY" '+ {spec:{template:{spec:{dnsPolicy:$policy,dnsConfig:null}}}}+ ')"+ kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic \+ -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0+ kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" \+ --timeout=180s >/dev/null 2>&1 || restore_ok=0+ wrc_restore_verified || restore_ok=0+ fi+ if [ "$restore_ok" = 1 ]; then+ delete_proxy_resources || cleanup_failed=1+ else+ cleanup_failed=1+ fi+ if [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then+ release_lock || cleanup_failed=1+ fi+ if [ "$cleanup_failed" != 0 ]; then+ echo "WRC egress E2E cleanup/restore failed; lock retained at ${WRC_NS}/${LOCK_NAME}." >&2+ echo "Inspect: kubectl --context=${E2E_KUBECONTEXT} -n ${WRC_NS} get deployment ${WRC_DEPLOY} -o yaml" >&2+ status=1+ fi+ exit "$status"+}+trap cleanup EXIT INT TERM++require_branch_owned_hcc_gate "$WRC_NS"+acquire_lock++[ -r "$DNS_PROXY_SCRIPT" ] || die "DNS proxy fixture is missing at ${DNS_PROXY_SCRIPT}"+deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC is not Ready before fault injection"+deployment_ready "$WRC_NS" control-api || die "control-api is not Ready before fault injection"+deployment_ready "$WRC_NS" host-context-controller || die "HCC is not Ready before fault injection"++ORIGINAL_REPLICAS="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o jsonpath='{.spec.replicas}')"+[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected one WRC replica, found ${ORIGINAL_REPLICAS:-unknown}"+ORIGINAL_DNS_POLICY="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |+ jq -r '.spec.template.spec.dnsPolicy // "ClusterFirst"')"+[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] || die "WRC already has non-default dnsPolicy"+ORIGINAL_DNS_CONFIG="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" -o json |+ jq -c '.spec.template.spec.dnsConfig // null')"+[ "$ORIGINAL_DNS_CONFIG" = null ] || die "WRC already has a dnsConfig; refusing non-restorable mutation"+WRC_IMAGE="$(kctl get deployment "$WRC_DEPLOY" -n "$WRC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="workflow-recipes")].image}')"+[ -n "$WRC_IMAGE" ] || die "could not resolve WRC image"+UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"+[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-dns ClusterIP"++kctl create configmap "$DNS_PROXY_CONFIGMAP" -n "$WRC_NS" \+ --from-file="server.cjs=${DNS_PROXY_SCRIPT}" --dry-run=client -o yaml |+ kctl apply -f - >/dev/null+PROXY_CREATED=1++kctl apply -f - >/dev/null <<EOF+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ${DNS_PROXY}+ namespace: ${WRC_NS}+ labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}+spec:+ replicas: 1+ selector: {matchLabels: {app: ${DNS_PROXY}}}+ template:+ metadata:+ labels: {app: ${DNS_PROXY}, e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}+ spec:+ automountServiceAccountToken: false+ terminationGracePeriodSeconds: 1+ containers:+ - name: dns-proxy+ image: ${WRC_IMAGE}+ imagePullPolicy: IfNotPresent+ command: [node, /fixture/server.cjs]+ env:+ - {name: TARGET_DNS, value: "${TARGET_DNS}"}+ - {name: ANSWER_IP, value: "${ANSWER_IP}"}+ - {name: UPSTREAM_DNS, value: "${UPSTREAM_DNS_IP}"}+ ports:+ - {name: dns-udp, containerPort: 8053, protocol: UDP}+ - {name: dns-tcp, containerPort: 8053, protocol: TCP}+ readinessProbe: {tcpSocket: {port: 8053}, periodSeconds: 1, failureThreshold: 10}+ volumeMounts:+ - {name: fixture, mountPath: /fixture, readOnly: true}+ resources:+ requests: {cpu: 5m, memory: 20Mi}+ limits: {cpu: 75m, memory: 64Mi}+ securityContext:+ allowPrivilegeEscalation: false+ capabilities: {drop: [ALL]}+ readOnlyRootFilesystem: true+ runAsNonRoot: true+ runAsUser: 1000+ runAsGroup: 1000+ seccompProfile: {type: RuntimeDefault}+ volumes:+ - name: fixture+ configMap: {name: ${DNS_PROXY_CONFIGMAP}}+---+apiVersion: v1+kind: Service+metadata:+ name: ${DNS_PROXY}+ namespace: ${WRC_NS}+spec:+ selector: {app: ${DNS_PROXY}}+ ports:+ - {name: dns-udp, protocol: UDP, port: 53, targetPort: 8053}+ - {name: dns-tcp, protocol: TCP, port: 53, targetPort: 8053}+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${DNS_PROXY_POLICY}+ namespace: ${WRC_NS}+spec:+ podSelector: {matchLabels: {app: ${DNS_PROXY}}}+ policyTypes: [Ingress, Egress]+ ingress:+ - from: [{podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}}]+ ports:+ - {protocol: UDP, port: 53}+ - {protocol: UDP, port: 8053}+ - {protocol: TCP, port: 53}+ - {protocol: TCP, port: 8053}+ egress:+ - to:+ - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: kube-system}}+ podSelector: {matchLabels: {k8s-app: kube-dns}}+ ports:+ - {protocol: UDP, port: 53}+ - {protocol: TCP, port: 53}+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${WRC_DNS_POLICY}+ namespace: ${WRC_NS}+spec:+ podSelector: {matchLabels: {app: ${WRC_DEPLOY}}}+ policyTypes: [Egress]+ egress:+ - to: [{podSelector: {matchLabels: {app: ${DNS_PROXY}}}}]+ ports:+ - {protocol: UDP, port: 53}+ - {protocol: UDP, port: 8053}+ - {protocol: TCP, port: 53}+ - {protocol: TCP, port: 8053}+EOF++kctl rollout status deployment "$DNS_PROXY" -n "$WRC_NS" --timeout=90s >/dev/null ||+ die "DNS proxy did not become Ready"+wait_until 30 "DNS proxy listener" \+ bash -c "kubectl --context='${E2E_KUBECONTEXT}' logs deployment/'${DNS_PROXY}' -n '${WRC_NS}' -c dns-proxy | grep -Fq 'wrc egress DNS proxy ready'" ||+ die "DNS proxy did not report readiness"+DNS_PROXY_IP="$(kctl get service "$DNS_PROXY" -n "$WRC_NS" -o jsonpath='{.spec.clusterIP}')"+[ -n "$DNS_PROXY_IP" ] || die "DNS proxy Service has no ClusterIP"++WRC_MUTATED=1+dns_patch="$(jq -cn --arg nameserver "$DNS_PROXY_IP" '+ {spec:{template:{spec:{+ dnsPolicy:"None",+ dnsConfig:{nameservers:[$nameserver],options:[+ {name:"ndots",value:"1"},{name:"timeout",value:"2"},{name:"attempts",value:"1"}+ ]}+ }}}}+')"+kctl patch deployment "$WRC_DEPLOY" -n "$WRC_NS" --type=strategic -p "$dns_patch" >/dev/null+kctl rollout status deployment "$WRC_DEPLOY" -n "$WRC_NS" --timeout=180s >/dev/null ||+ die "WRC did not become Ready with the isolated DNS proxy"+deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC DNS setup"+deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC DNS setup"+ok "isolated WRC DNS proxy is Ready without affecting control-api or HCC"++RECIPE_CREATED=1+kctl apply -f - >/dev/null <<EOF+apiVersion: clerum.io/v1alpha1+kind: WorkflowRecipe+metadata:+ name: ${RECIPE_NAME}+ namespace: ${RECIPE_NAMESPACE}+ labels: {e2e.clerum.io/suite: ${SUITE_NAME}, e2e.clerum.io/run: "${RUN_LABEL}"}+spec:+ description: WRC egress degradation E2E baseline+ contextRef: context1+ workloads:+ - id: frontend+ type: deployment+ image: clerum/mock-mcp-server:test+ port: 3001+ healthCheck: {type: http, path: /, port: 3001}+ - id: worker+ type: deployment+ image: clerum/mock-mcp-server:test+ port: 3001+ healthCheck: {type: http, path: /, port: 3001}+ egressBindings:+ - {dns: ${TARGET_DNS}, port: 443, protocol: TCP}+ - {dns: ${TARGET_DNS}, port: 8443, protocol: TCP}+ ui:+ workloadRef: frontend+ port: 3001+ title: WRC egress degradation fixture+ defaultPath: /+ egress:+ external:+ - {fqdn: ${TARGET_DNS}, port: 443}+ - {fqdn: ${TARGET_DNS}, port: 8443}+ security: {isolationLevel: minimal}+EOF++wait_until 180 "valid fixture recipe to become active" recipe_active ||+ die "fixture recipe did not become active"+wait_until 120 "initial UI egress policy" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443,8443]' ||+ die "initial UI policy did not enforce both declared ports"+wait_until 120 "initial workload egress policy" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443,8443]' ||+ die "initial workload policy did not enforce both declared ports"+wait_until 120 "frontend business health" workload_health "$UI_NS" frontend ||+ die "frontend health endpoint was unavailable"+wait_until 120 "worker business health" workload_health "$RECIPE_NAMESPACE" worker ||+ die "worker health endpoint was unavailable"+ok "valid recipe is active, both workload health endpoints respond, and both policies enforce the declared set"++hold_before="$(dns_count hold)"+set_dns_mode hold || die "could not hold fixture DNS"+reduced_patch="$(jq -cn --arg dns "$TARGET_DNS" '+ [+ {op:"replace",path:"/spec/ui/egress/external",value:[{fqdn:$dns,port:443}]},+ {op:"replace",path:"/spec/workloads/1/egressBindings",value:[{dns:$dns,port:443,protocol:"TCP"}]}+ ]+')"+kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=json \+ -p "$reduced_patch" >/dev/null+wait_until 60 "WRC DNS query to enter the deterministic hold" \+ dns_count_at_least hold "$((hold_before + 1))" || die "WRC never reached the DNS hold"+wait_until 60 "pre-DNS UI contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||+ die "UI port 8443 was not revoked before DNS completed"+wait_until 60 "pre-DNS workload contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||+ die "workload port 8443 was not revoked before DNS completed"+recipe_active || die "correctly configured recipe left active before the held DNS completed"+workload_health "$UI_NS" frontend || die "frontend stopped serving during held DNS"+workload_health "$RECIPE_NAMESPACE" worker || die "worker stopped serving during held DNS"+ok "removed UI/workload permissions contracted before DNS completed while both services stayed healthy"++append_policy_port "$UI_NS" "$UI_POLICY" 9443+append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443+policy_has_port "$UI_NS" "$UI_POLICY" 9443 || die "could not inject UI policy race"+policy_has_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443 || die "could not inject workload policy race"++servfail_before="$(dns_count servfail)"+set_dns_mode servfail || die "could not release held DNS as SERVFAIL"+wait_until 90 "both WRC lanes to observe SERVFAIL" \+ dns_count_at_least servfail "$((servfail_before + 2))" ||+ die "UI and workload lanes did not both observe the transient DNS failure"+wait_until 90 "post-DNS UI re-contraction" policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' ||+ die "UI race widening survived transient DNS failure"+wait_until 90 "post-DNS workload re-contraction" policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||+ die "workload race widening survived transient DNS failure"+wait_until 60 "recipe to remain active after proven fail-static" recipe_active ||+ die "correctly configured recipe was cut by transient DNS failure"+workload_health "$UI_NS" frontend || die "frontend health failed after transient DNS failure"+workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after transient DNS failure"+deployment_ready "$WRC_NS" "$WRC_DEPLOY" || die "WRC lost readiness during fault"+deployment_ready "$WRC_NS" control-api || die "control-api lost readiness during WRC fault"+deployment_ready "$WRC_NS" host-context-controller || die "HCC lost readiness during WRC fault"+ok "transient DNS failure removed concurrent widening, preserved proven port 443, and isolated blast radius"++ok_before="$(dns_count ok)"+set_dns_mode ok || die "could not restore successful DNS"+recovery_patch="$(jq -cn --arg description "WRC egress recovery ${RUN_ID}" \+ '{spec:{description:$description}}')"+kctl patch workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NAMESPACE" --type=merge \+ -p "$recovery_patch" >/dev/null+wait_until 90 "both WRC lanes to resolve after recovery" \+ dns_count_at_least ok "$((ok_before + 2))" || die "DNS recovery was not observed by both lanes"+wait_until 90 "recovered recipe active state" recipe_active || die "recipe did not recover active"+policy_matches_ports "$UI_NS" "$UI_POLICY" '[443]' || die "UI policy drifted after recovery"+policy_matches_ports "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" '[443]' ||+ die "workload policy drifted after recovery"+workload_health "$UI_NS" frontend || die "frontend health failed after DNS recovery"+workload_health "$RECIPE_NAMESPACE" worker || die "worker health failed after DNS recovery"+ok "successful DNS recovery reconverged both lanes without restoring removed or raced permissions"++echo "WRC_EGRESS_DEGRADATION_E2E_PASS"+echo "recipe=${RECIPE_NAMESPACE}/${RECIPE_NAME}"+echo "uiPolicy=${UI_NS}/${UI_POLICY}"+echo "workloadPolicy=${RECIPE_NAMESPACE}/${WORKLOAD_POLICY}"diff --git a/scripts/tests/test-wrc-egress-degradation-gate.sh b/scripts/tests/test-wrc-egress-degradation-gate.shnew file mode 100755index 000000000..5b4a8118e--- /dev/null+++ b/scripts/tests/test-wrc-egress-degradation-gate.sh@@ -0,0 +1,83 @@+#!/usr/bin/env bash+# Literal source-contract patterns intentionally contain shell-looking text.+# shellcheck disable=SC2016+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-wrc-egress-degradation.sh"+FIXTURE="${ROOT}/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs"+MAKEFILE="${ROOT}/Makefile"++fail() { echo "FAIL: $*" >&2; exit 1; }+contains() {+ local file=$1 pattern=$2 description=$3+ grep -Fq -- "$pattern" "$file" || fail "$description"+}++bash -n "$GATE"+node --check "$FIXTURE" >/dev/null++contains "$GATE" 'is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"' \+ 'gate must refuse non-branch Kubernetes contexts'+contains "$GATE" 'E2E_WRC_EGRESS_FAULT_INJECTION' \+ 'gate must require explicit fault-injection acknowledgement'+contains "$GATE" 'require_branch_owned_hcc_gate "$WRC_NS"' \+ 'gate must verify exact HEAD, worktree, profile, fingerprint, and pre-gate marker'+contains "$GATE" 'acquire_lock' 'gate must acquire exclusive WRC fault-injection ownership'+contains "$GATE" 'wrc_restore_verified' 'gate must prove WRC DNS configuration restoration'+contains "$GATE" 'lock retained' 'failed restoration must retain the coordination lock'++contains "$GATE" 'id: frontend' 'fixture must exercise the sandbox UI workload'+contains "$GATE" 'id: worker' 'fixture must exercise a non-UI workload'+contains "$GATE" 'port: 8443' 'fixture must begin with a permission that will be removed'+contains "$GATE" 'set_dns_mode hold' 'gate must create a real DNS latency window'+contains "$GATE" 'pre-DNS UI contraction' 'gate must assert UI contraction before DNS completes'+contains "$GATE" 'pre-DNS workload contraction' \+ 'gate must assert workload contraction before DNS completes'+contains "$GATE" 'append_policy_port "$UI_NS" "$UI_POLICY" 9443' \+ 'gate must inject live UI drift during DNS latency'+contains "$GATE" 'append_policy_port "$RECIPE_NAMESPACE" "$WORKLOAD_POLICY" 9443' \+ 'gate must inject live workload drift during DNS latency'+contains "$GATE" 'set_dns_mode servfail' 'gate must produce a real retryable resolver response'+contains "$GATE" 'post-DNS UI re-contraction' \+ 'gate must assert the UI fresh-read contraction after DNS failure'+contains "$GATE" 'post-DNS workload re-contraction' \+ 'gate must assert the workload fresh-read contraction after DNS failure'++contains "$GATE" 'workload_health "$UI_NS" frontend' \+ 'gate must prove the UI workload still serves its health endpoint'+contains "$GATE" 'workload_health "$RECIPE_NAMESPACE" worker' \+ 'gate must prove the workload still serves its health endpoint'+contains "$GATE" 'deployment_ready "$WRC_NS" control-api' \+ 'gate must assert an unrelated production service remains Ready'+contains "$GATE" 'deployment_ready "$WRC_NS" host-context-controller' \+ 'gate must assert HCC remains Ready'+contains "$GATE" 'set_dns_mode ok' 'gate must restore successful DNS'+contains "$GATE" 'WRC_EGRESS_DEGRADATION_E2E_PASS' \+ 'gate must emit an unambiguous terminal success marker'++contains "$FIXTURE" 'match(/^\/mode\/(ok|hold|servfail)$/)' \+ 'DNS fixture must expose exactly the required deterministic modes'+contains "$FIXTURE" '0x8180 | rcode' 'DNS fixture must emit an actual SERVFAIL response'+contains "$FIXTURE" 'forwardUdp' 'DNS fixture must forward unrelated UDP queries'+contains "$FIXTURE" 'forwardTcp' 'DNS fixture must forward unrelated TCP queries'++contains "$MAKEFILE" '.PHONY: test-e2e-wrc-egress-degradation' \+ 'Makefile must expose the WRC egress degradation gate'+contains "$MAKEFILE" 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-egress-degradation.sh' \+ 'Make target must bind the mutation to the explicit profile/context'+contains "$MAKEFILE" '.PHONY: test-e2e-pr567-egress-resilience' \+ 'Makefile must expose the complete PR #567 resilience gate'+contains "$MAKEFILE" '$(MAKE) test-e2e-hcc-mcp-context-readiness' \+ 'complete PR gate must run the existing real HCC DNS/runtime journey'+contains "$MAKEFILE" '$(MAKE) test-e2e-wrc-egress-degradation' \+ 'complete PR gate must run the WRC UI/workload degradation journey'++if grep -Eq 'kubectl[[:space:]]+(get|apply|patch|delete|exec|logs)' "$GATE"; then+ fail 'raw kubectl operations must use kctl or an explicit --context cleanup probe'+fi+if grep -Fq -- '--force' "$GATE"; then+ fail 'gate must not force Kubernetes mutations'+fi++echo 'PASS: WRC egress degradation E2E guard contract'diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex b27a139fb..47afd8db2 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.68",+ "version": "1.0.69","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.68",+ "version": "1.0.69","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex 8b093d1d3..c71bf0258 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.68",+ "version": "1.0.69","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",diff --git a/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjsnew file mode 100644index 000000000..1bbd136e3--- /dev/null+++ b/tests/e2e/fixtures/wrc-egress-dns-proxy/server.cjs@@ -0,0 +1,183 @@+'use strict'++const dgram = require('node:dgram')+const http = require('node:http')+const net = require('node:net')++const target = String(process.env.TARGET_DNS || '').toLowerCase()+const upstream = process.env.UPSTREAM_DNS+const answerIp = String(process.env.ANSWER_IP || '93.184.216.34')+const answerOctets = answerIp.split('.').map(Number)++if (!target || !upstream || answerOctets.length !== 4 || answerOctets.some(Number.isNaN)) {+ throw new Error('TARGET_DNS, UPSTREAM_DNS, and a valid IPv4 ANSWER_IP are required')+}++let mode = 'ok'+const held = []+const counts = { ok: 0, hold: 0, servfail: 0, forwarded: 0 }+const udp = dgram.createSocket('udp4')++function question(message) {+ let offset = 12+ const labels = []+ while (offset < message.length) {+ const length = message[offset++]+ if (length === 0) break+ if (offset + length > message.length) return null+ labels.push(message.subarray(offset, offset + length).toString('ascii'))+ offset += length+ }+ if (offset + 4 > message.length) return null+ return {+ name: labels.join('.').toLowerCase(),+ type: message.readUInt16BE(offset),+ end: offset + 4,+ }+}++function dnsResponse(message, parsed, rcode) {+ const header = Buffer.from(message.subarray(0, 12))+ header.writeUInt16BE(0x8180 | rcode, 2)+ header.writeUInt16BE(rcode === 0 ? 1 : 0, 6)+ header.writeUInt16BE(0, 8)+ header.writeUInt16BE(0, 10)+ const questionBytes = message.subarray(12, parsed.end)+ if (rcode !== 0) return Buffer.concat([header, questionBytes])+ const record = Buffer.from([+ 0xc0,+ 0x0c,+ 0x00,+ 0x01,+ 0x00,+ 0x01,+ 0x00,+ 0x00,+ 0x01,+ 0x2c,+ 0x00,+ 0x04,+ ...answerOctets,+ ])+ return Buffer.concat([header, questionBytes, record])+}++function tcpFrame(message) {+ const frame = Buffer.alloc(message.length + 2)+ frame.writeUInt16BE(message.length, 0)+ message.copy(frame, 2)+ return frame+}++function send(entry, selectedMode) {+ const response = dnsResponse(entry.message, entry.parsed, selectedMode === 'ok' ? 0 : 2)+ counts[selectedMode]+++ if (entry.transport === 'udp') {+ udp.send(response, entry.rinfo.port, entry.rinfo.address)+ } else if (!entry.socket.destroyed) {+ entry.socket.write(tcpFrame(response))+ }+ console.log(`${selectedMode} DNS A ${target} via ${entry.transport.toUpperCase()}`)+}++function forwardUdp(message, rinfo) {+ counts.forwarded+++ const relay = dgram.createSocket('udp4')+ const timer = setTimeout(() => relay.close(), 5000)+ relay.once('message', response => {+ clearTimeout(timer)+ udp.send(response, rinfo.port, rinfo.address)+ relay.close()+ })+ relay.once('error', () => {+ clearTimeout(timer)+ relay.close()+ })+ relay.send(message, 53, upstream)+}++function forwardTcp(message, socket) {+ counts.forwarded+++ const relay = net.createConnection({ host: upstream, port: 53 }, () =>+ relay.write(tcpFrame(message))+ )+ let pending = Buffer.alloc(0)+ relay.setTimeout(5000, () => relay.destroy())+ relay.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ if (pending.length < 2) return+ const length = pending.readUInt16BE(0)+ if (pending.length < length + 2) return+ if (!socket.destroyed) socket.write(pending.subarray(0, length + 2))+ relay.destroy()+ })+ relay.once('error', () => relay.destroy())+}++function handle(message, entry) {+ const parsed = question(message)+ if (parsed && parsed.name === target && parsed.type === 1) {+ entry.message = message+ entry.parsed = parsed+ if (mode === 'hold') {+ counts.hold+++ held.push(entry)+ console.log(`hold DNS A ${target} via ${entry.transport.toUpperCase()}`)+ } else {+ send(entry, mode)+ }+ return+ }+ if (entry.transport === 'udp') forwardUdp(message, entry.rinfo)+ else forwardTcp(message, entry.socket)+}++function setMode(nextMode) {+ mode = nextMode+ if (mode !== 'hold') {+ while (held.length > 0) send(held.shift(), mode)+ }+}++let listeners = 0+function listening() {+ listeners+++ if (listeners === 2) console.log(`wrc egress DNS proxy ready for ${target}`)+}++udp.on('message', (message, rinfo) => handle(message, { transport: 'udp', rinfo }))+udp.bind(8053, '0.0.0.0', listening)++const tcp = net.createServer(socket => {+ let pending = Buffer.alloc(0)+ socket.on('data', chunk => {+ pending = Buffer.concat([pending, chunk])+ while (pending.length >= 2) {+ const length = pending.readUInt16BE(0)+ if (pending.length < length + 2) return+ const message = pending.subarray(2, length + 2)+ pending = pending.subarray(length + 2)+ handle(message, { transport: 'tcp', socket })+ }+ })+})+tcp.listen(8053, '0.0.0.0', listening)++http+ .createServer((request, response) => {+ const nextMode = request.url?.match(/^\/mode\/(ok|hold|servfail)$/)?.[1]+ if (request.method === 'POST' && nextMode) {+ setMode(nextMode)+ response.writeHead(200, { 'content-type': 'application/json' })+ response.end(JSON.stringify({ mode, held: held.length, counts }))+ return+ }+ if (request.method === 'GET' && request.url === '/state') {+ response.writeHead(200, { 'content-type': 'application/json' })+ response.end(JSON.stringify({ mode, held: held.length, counts }))+ return+ }+ response.writeHead(404)+ response.end()+ })+ .listen(8090, '127.0.0.1')diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex e63ee916f..543d4a0a6 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.343",+ "version": "0.1.344","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.343",+ "version": "0.1.344","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 18fc8a122..e2b3242cc 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.343",+ "version": "0.1.344","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex c22ac5028..5891f489c 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -2927,6 +2927,127 @@ describe('WorkflowRecipeReconciler', () => {})})+ it.each([+ {+ resolverLabel: 'controller fault',+ resolverResult: 'throw' as const,+ label: 'retryable API outage',+ contractionError: () =>+ Object.assign(new Error('workload re-contraction unavailable'), { code: 503 }),+ expectedPhase: 'active' as const,+ retryable: true,+ },+ {+ resolverLabel: 'controller fault',+ resolverResult: 'throw' as const,+ label: 'non-retryable identity conflict',+ contractionError: () =>+ Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),+ expectedPhase: 'failed' as const,+ retryable: false,+ },+ {+ resolverLabel: 'transient DNS result',+ resolverResult: 'transient' as const,+ label: 'retryable API outage',+ contractionError: () =>+ Object.assign(new Error('workload re-contraction unavailable'), { code: 503 }),+ expectedPhase: 'active' as const,+ retryable: true,+ },+ {+ resolverLabel: 'transient DNS result',+ resolverResult: 'transient' as const,+ label: 'non-retryable identity conflict',+ contractionError: () =>+ Object.assign(new Error('workload re-contraction conflict'), { code: 409 }),+ expectedPhase: 'failed' as const,+ retryable: false,+ },+ ])(+ '#567 remediation: workload $resolverLabel plus $label preserves fail-closed service semantics',+ async ({ contractionError, expectedPhase, resolverResult, retryable }) => {+ const policyName = 'wl-egress-test-recipe-worker'+ let resolverFault = false+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ if (resolverFault) {+ if (resolverResult === 'transient') {+ return { kind: 'error', error: 'ESERVFAIL', retryable: true }+ }+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ }+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }+ },+ })+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:1',+ port: 8080,+ egressBindings: [{ dns: 'current.example.com', port: 443 }],+ },+ ],+ },+ })+ await rec.reconcile(recipe)+ const written = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls+ .map(call => call[0]?.body as k8s.V1NetworkPolicy)+ .find(body => body.metadata?.name === policyName)+ expect(written).toBeDefined()+ const live: k8s.V1NetworkPolicy = {+ ...structuredClone(written!),+ metadata: {+ ...written!.metadata,+ name: policyName,+ namespace: 'sandbox-recipes',+ uid: 'uid-workload-double-fault',+ resourceVersion: '70',+ },+ }+ let targetReads = 0+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ if (name !== policyName) return Promise.reject({ code: 404 })+ targetReads+++ return targetReads === 1 ? Promise.resolve(live) : Promise.reject(contractionError())+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockClear()+ resolverFault = true++ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })++ expect(targetReads).toBe(2)+ expect(result.phase).toBe(expectedPhase)+ if (retryable) {+ expect(result.skipStatusPatch).toBe(true)+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ } else {+ expect(result.skipStatusPatch).not.toBe(true)+ expect(result.message).toContain('workload re-contraction conflict')+ }+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({+ metadata: expect.objectContaining({ name: policyName }),+ }),+ })+ )+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ }+ )+it('returns failed phase when a workload egress binding uses wildcard DNS', async () => {const recipe = makeRecipe({spec: {@@ -4665,6 +4786,111 @@ describe('WorkflowRecipeReconciler', () => {})+ it.each([+ {+ resolverLabel: 'controller fault',+ resolverResult: 'throw' as const,+ label: 'retryable API outage',+ contractionError: () =>+ Object.assign(new Error('UI re-contraction unavailable'), { code: 503 }),+ expectedPhase: 'active' as const,+ retryable: true,+ },+ {+ resolverLabel: 'controller fault',+ resolverResult: 'throw' as const,+ label: 'non-retryable identity conflict',+ contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),+ expectedPhase: 'failed' as const,+ retryable: false,+ },+ {+ resolverLabel: 'transient DNS result',+ resolverResult: 'transient' as const,+ label: 'retryable API outage',+ contractionError: () =>+ Object.assign(new Error('UI re-contraction unavailable'), { code: 503 }),+ expectedPhase: 'active' as const,+ retryable: true,+ },+ {+ resolverLabel: 'transient DNS result',+ resolverResult: 'transient' as const,+ label: 'non-retryable identity conflict',+ contractionError: () => Object.assign(new Error('UI re-contraction conflict'), { code: 409 }),+ expectedPhase: 'failed' as const,+ retryable: false,+ },+ ])(+ '#567 remediation: UI $resolverLabel plus $label preserves fail-closed service semantics',+ async ({ contractionError, expectedPhase, resolverResult, retryable }) => {+ const policyName = 'ui-egress-test-recipe'+ let resolverFault = false+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => {+ if (resolverFault) {+ if (resolverResult === 'transient') {+ return { kind: 'error', error: 'ESERVFAIL', retryable: true }+ }+ throw Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ }+ return { kind: 'ok', ipv4: ['93.184.216.20'], ipv6: [], ttlSeconds: 300 }+ },+ })+ const recipe = uiRecipeWithExternals([{ fqdn: 'current.example.com', port: 443 }])+ await rec.reconcile(recipe)+ const written = createdPolicy(policyName)+ expect(written).toBeDefined()+ const live: k8s.V1NetworkPolicy = {+ ...structuredClone(written!),+ metadata: {+ ...written!.metadata,+ name: policyName,+ namespace: 'sandbox-ui',+ uid: 'uid-ui-double-fault',+ resourceVersion: '80',+ },+ }+ let targetReads = 0+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ if (name !== policyName) return Promise.reject({ code: 404 })+ targetReads+++ return targetReads === 1 ? Promise.resolve(live) : Promise.reject(contractionError())+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.deleteNamespacedNetworkPolicy.mockClear()+ resolverFault = true++ const result = await rec.reconcile({ ...recipe, status: { phase: 'active' } })++ expect(targetReads).toBe(2)+ expect(result.phase).toBe(expectedPhase)+ if (retryable) {+ expect(result.skipStatusPatch).toBe(true)+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ } else {+ expect(result.skipStatusPatch).not.toBe(true)+ expect(result.message).toContain('UI re-contraction conflict')+ }+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({+ metadata: expect.objectContaining({ name: policyName }),+ }),+ })+ )+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )+ }+ )+it('#567 remediation: persists the prior resolved-at when a permanent failure narrows the policy', async () => {let permanentFailure = falselet live: k8s.V1NetworkPolicy | undefined