Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): close WRC HCC runtime coverage
Alfredo Lopez161d2dce85de
diff --git a/scripts/e2e/e2e-wrc-hcc-context-noop-resync.sh b/scripts/e2e/e2e-wrc-hcc-context-noop-resync.shindex 25c231a16..4c75d702c 100755--- a/scripts/e2e/e2e-wrc-hcc-context-noop-resync.sh+++ b/scripts/e2e/e2e-wrc-hcc-context-noop-resync.sh@@ -53,20 +53,28 @@ SUITE_NAME="wrc-hcc-context-noop-resync"RECIPE_NAME="$(truncate_rfc1123 "e2e-pr568-${RUN_LABEL}")"CONTEXT_NAME="$(truncate_rfc1123 "e2e-pr568-context-${RUN_LABEL}")"MCP_NAME="$(truncate_rfc1123 "${RECIPE_NAME}-mock-tools")"+SECOND_MCP_NAME="$(truncate_rfc1123 "${RECIPE_NAME}-mock-tools-2")"PROBE_NAME="$(truncate_rfc1123 "e2e-pr568-probe-${RUN_LABEL}")"+HOST_NAME="$(truncate_rfc1123 "e2e-pr568-host-${RUN_LABEL}")"+SOURCE_HOST_NAME="${E2E_WRC_HCC_SOURCE_HOST:-chatllm}"CONTEXT_POLICY="ctx-${CONTEXT_NAME}-${MCP_NAME}"CONTEXT_EGRESS_POLICY="${CONTEXT_POLICY}-egress"RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_NAME}-${MCP_NAME}"+SECOND_CONTEXT_POLICY="ctx-${CONTEXT_NAME}-${SECOND_MCP_NAME}"+SECOND_CONTEXT_EGRESS_POLICY="${SECOND_CONTEXT_POLICY}-egress"+SECOND_RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_NAME}-${SECOND_MCP_NAME}"WORKLOAD_SELECTOR="clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=mock-tools"+SECOND_WORKLOAD_SELECTOR="clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=mock-tools-2"+WRC_CLEANUP_KINDS="deployment,statefulset,daemonset,job,cronjob,replicaset,pod,service,serviceaccount,role,rolebinding,configmap,secret,persistentvolumeclaim,networkpolicy"RESYNC_SECONDS="${E2E_WRC_HCC_NETPOL_RESYNC_SEC:-120}"RESYNC_TIMEOUT_SECONDS="${E2E_WRC_HCC_RESYNC_TIMEOUT_SEC:-360}"-[[ "$RESYNC_SECONDS" =~ ^[0-9]+$ ]] && [ "$RESYNC_SECONDS" -ge 5 ] &&+[[ "$RESYNC_SECONDS" =~ ^[1-9][0-9]*$ ]] && [ "$RESYNC_SECONDS" -ge 30 ] &&[ "$RESYNC_SECONDS" -le 120 ] || {- echo "E2E_WRC_HCC_NETPOL_RESYNC_SEC must be an integer from 5 through 120." >&2+ echo "E2E_WRC_HCC_NETPOL_RESYNC_SEC must be an integer from 30 through 120." >&2exit 1}-[[ "$RESYNC_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] &&+[[ "$RESYNC_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] &&[ "$RESYNC_TIMEOUT_SECONDS" -gt "$RESYNC_SECONDS" ] || {echo "E2E_WRC_HCC_RESYNC_TIMEOUT_SEC must be an integer greater than the resync interval." >&2exit 1@@ -83,9 +91,14 @@ HCC_ENV_MUTATED=0RECIPE_CREATED=0CONTEXT_CREATED=0PROBE_CREATED=0+HOST_CREATED=0ORIGINAL_CONTEXT_RV=""ORIGINAL_CONTEXT_GENERATION=""ORIGINAL_MCP_GENERATION=""+ORIGINAL_SECOND_MCP_GENERATION=""+PROBE_BASELINE_IDENTITY=""+PRIMARY_MCP_BASELINE_IDENTITY=""+SECOND_MCP_BASELINE_IDENTITY=""ORIGINAL_POLICY_UID=""ORIGINAL_POLICY_SNAPSHOT=""FINAL_SUMMARY=""@@ -119,9 +132,15 @@ wait_until() {return 1}+utc_now() {+ node -e 'process.stdout.write(new Date().toISOString())'+}+resource_absent() {- local kind=$1 name=$2 namespace=$3- ! kctl get "$kind" "$name" -n "$namespace" >/dev/null 2>&1+ local kind=$1 name=$2 namespace=$3 found+ found="$(kctl get "$kind" "$name" -n "$namespace" --ignore-not-found -o name 2>/dev/null)" ||+ return 1+ [ -z "$found" ]}resource_collection_absent() {@@ -171,14 +190,16 @@ probe_hcc_ready() {}hcc_log_contains() {- local marker=$1 pod+ local marker=$1 pod logspod="$(running_hcc_pod)" || return 1- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller --since=10m 2>/dev/null |- grep -Fq "$marker"+ logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \+ --since=10m 2>/dev/null)" || return 1+ grep -Fq "$marker" <<<"$logs"}-mcpserver_current_ready() {- kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json 2>/dev/null |+mcpserver_named_current_ready() {+ local server=$1+ kctl get mcpserver "$server" -n "$MCP_NS" -o json 2>/dev/null |jq -e '.metadata.generation as $generation |any(.status.conditions[]?;@@ -187,30 +208,53 @@ mcpserver_current_ready() {' >/dev/null}-mcp_workload_deployment_json() {- local deployments- deployments="$(kctl get deployments -n "$MCP_NS" -l "$WORKLOAD_SELECTOR" -o json 2>/dev/null)" ||+mcp_workload_deployment_named_json() {+ local selector=$1 deployments+ deployments="$(kctl get deployments -n "$MCP_NS" -l "$selector" -o json 2>/dev/null)" ||return 1jq -ce 'if (.items | length) == 1 then .items[0] else empty end' <<<"$deployments"}-mcp_runtime_still_ready() {- local deployment deployment_ready endpoint_addresses- mcpserver_current_ready || return 1- deployment="$(mcp_workload_deployment_json)" || return 1+mcp_named_runtime_still_ready() {+ local server=$1 selector=$2 deployment deployment_ready endpoint_addresses+ mcpserver_named_current_ready "$server" || return 1+ deployment="$(mcp_workload_deployment_named_json "$selector")" || return 1deployment_ready="$(jq -r '.status.readyReplicas // 0' <<<"$deployment")" || return 1[ "${deployment_ready:-0}" -ge 1 ] || return 1- endpoint_addresses="$(kctl get endpoints "$MCP_NAME" -n "$MCP_NS" \+ endpoint_addresses="$(kctl get endpoints "$server" -n "$MCP_NS" \-o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null)" || return 1[ -n "$endpoint_addresses" ]}+mcp_runtime_still_ready() {+ mcp_named_runtime_still_ready "$MCP_NAME" "$WORKLOAD_SELECTOR"+}++selector_runtime_identity() {+ local namespace=$1 selector=$2 deployments deployment pods+ deployments="$(kctl get deployments -n "$namespace" -l "$selector" -o json 2>/dev/null)" ||+ return 1+ deployment="$(jq -ce '+ if (.items | length) == 1 then+ .items[0] | [.metadata.uid, .metadata.generation, .spec.replicas, .spec.selector, .spec.template]+ else empty end+ ' <<<"$deployments")" || return 1+ pods="$(kctl get pods -n "$namespace" -l "$selector" -o json 2>/dev/null |+ jq -ceS '[.items[] |+ select(.status.phase == "Running" and .metadata.deletionTimestamp == null) |+ select(any(.status.conditions[]?; .type == "Ready" and .status == "True")) |+ {uid:.metadata.uid,containers:([.status.containerStatuses[]? |+ {name,restartCount}] | sort_by(.name))}] | sort_by(.uid)')" || return 1+ [ "$(jq -r 'length' <<<"$pods")" -ge 1 ] || return 1+ printf '%s | %s\n' "$deployment" "$pods"+}+policy_has_identity() {- local namespace=$1 name=$2 policy_type=$3+ local namespace=$1 name=$2 policy_type=$3 server=$4kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |jq -e \--arg context "$CONTEXT_NAME" \- --arg server "$MCP_NAME" \+ --arg server "$server" \--arg policyType "$policy_type" '.metadata.labels["clerum.io/managed-by"] == "host-context-controller" and.metadata.labels["clerum.io/policy-type"] == $policyType and@@ -226,18 +270,47 @@ context_projection_converged() {.spec.contextId == .metadata.name and.spec.mcpServers == [$server] and.metadata.labels["clerum.io/managed-by"] == "wrc" and- (.metadata.labels["clerum.io/recipe"] // "") == ""+ (.metadata.labels["clerum.io/recipe"] // "") == "" and+ ((.metadata.ownerReferences // []) | length) == 0' <<<"$context" >/dev/null || return 1kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json 2>/dev/null |jq -e --arg context "$CONTEXT_NAME" '.spec.contextRef == $context' >/dev/null || return 1- mcpserver_current_ready || return 1- deployment="$(mcp_workload_deployment_json)" || return 1+ mcpserver_named_current_ready "$MCP_NAME" || return 1+ deployment="$(mcp_workload_deployment_named_json "$WORKLOAD_SELECTOR")" || return 1deployment_ready="$(jq -r '.status.readyReplicas // 0' <<<"$deployment")" || return 1[ "${deployment_ready:-0}" -ge 1 ] || return 1kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1- policy_has_identity "$MCP_NS" "$CONTEXT_POLICY" context-allow || return 1- policy_has_identity "$HOST_NS" "$CONTEXT_EGRESS_POLICY" context-allow || return 1- policy_has_identity "$RPC_PROXY_NS" "$RPC_EGRESS_POLICY" rpc-proxy-egress+ policy_has_identity "$MCP_NS" "$CONTEXT_POLICY" context-allow "$MCP_NAME" || return 1+ policy_has_identity "$HOST_NS" "$CONTEXT_EGRESS_POLICY" context-allow "$MCP_NAME" || return 1+ policy_has_identity "$RPC_PROXY_NS" "$RPC_EGRESS_POLICY" rpc-proxy-egress "$MCP_NAME"+}++expanded_context_projection_converged() {+ local context+ context="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" -o json 2>/dev/null)" || return 1+ jq -e --arg first "$MCP_NAME" --arg second "$SECOND_MCP_NAME" '+ .spec.contextId == .metadata.name and+ ((.spec.mcpServers | sort) == ([$first, $second] | sort)) and+ .metadata.labels["clerum.io/managed-by"] == "wrc" and+ (.metadata.labels["clerum.io/recipe"] // "") == "" and+ ((.metadata.ownerReferences // []) | length) == 0+ ' <<<"$context" >/dev/null || return 1+ kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json 2>/dev/null |+ jq -e --arg context "$CONTEXT_NAME" '.spec.contextRef == $context' >/dev/null || return 1+ kctl get mcpserver "$SECOND_MCP_NAME" -n "$MCP_NS" -o json 2>/dev/null |+ jq -e --arg context "$CONTEXT_NAME" '.spec.contextRef == $context' >/dev/null || return 1+ mcp_named_runtime_still_ready "$MCP_NAME" "$WORKLOAD_SELECTOR" || return 1+ mcp_named_runtime_still_ready "$SECOND_MCP_NAME" "$SECOND_WORKLOAD_SELECTOR" || return 1+ policy_has_identity "$MCP_NS" "$CONTEXT_POLICY" context-allow "$MCP_NAME" || return 1+ policy_has_identity "$HOST_NS" "$CONTEXT_EGRESS_POLICY" context-allow "$MCP_NAME" || return 1+ policy_has_identity "$RPC_PROXY_NS" "$RPC_EGRESS_POLICY" rpc-proxy-egress "$MCP_NAME" ||+ return 1+ policy_has_identity "$MCP_NS" "$SECOND_CONTEXT_POLICY" context-allow "$SECOND_MCP_NAME" ||+ return 1+ policy_has_identity "$HOST_NS" "$SECOND_CONTEXT_EGRESS_POLICY" context-allow \+ "$SECOND_MCP_NAME" || return 1+ policy_has_identity "$RPC_PROXY_NS" "$SECOND_RPC_EGRESS_POLICY" rpc-proxy-egress \+ "$SECOND_MCP_NAME"}wrc_reconcile_count() {@@ -252,6 +325,264 @@ wrc_reconciled_after() {[ "$current" -gt "$before" ]}+host_runtime_ready() {+ local deployment ready desired pod_rows ready_pods endpoint_addresses+ deployment="$(kctl get deployment "$HOST_NAME" -n "$HOST_NS" -o json 2>/dev/null)" ||+ return 1+ jq -e --arg host "$HOST_NAME" --arg context "$CONTEXT_NAME" '+ .metadata.labels["clerum.io/managed-by"] == "host-context-controller" and+ .metadata.labels["clerum.io/host"] == $host and+ .spec.template.metadata.labels["clerum.io/context"] == $context+ ' <<<"$deployment" >/dev/null || return 1+ ready="$(jq -r '.status.readyReplicas // 0' <<<"$deployment")"+ desired="$(jq -r '.spec.replicas // 0' <<<"$deployment")"+ [ "$desired" = 1 ] && [ "$ready" = 1 ] || return 1+ pod_rows="$(kctl get pods -n "$HOST_NS" -l "clerum.io/host=${HOST_NAME}" \+ --field-selector=status.phase=Running \+ -o jsonpath='{range .items[*]}{.metadata.uid}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \+ 2>/dev/null)" || return 1+ ready_pods="$(awk -F '\t' '$1 != "" && $2 == "True" && $3 == "" { count++ } END { print count + 0 }' \+ <<<"$pod_rows")"+ [ "$ready_pods" = 1 ] || return 1+ endpoint_addresses="$(kctl get endpoints "$HOST_NAME" -n "$HOST_NS" \+ -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null)" || return 1+ [ -n "$endpoint_addresses" ]+}++host_deployment_identity() {+ selector_runtime_identity "$HOST_NS" \+ "clerum.io/managed-by=host-context-controller,clerum.io/host=${HOST_NAME}"+}++probe_host_runtime_health() {+ local port+ port="$(kctl get deployment "$HOST_NAME" -n "$HOST_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].ports[?(@.name=="http")].containerPort}' \+ 2>/dev/null)" || return 1+ [[ "$port" =~ ^[0-9]+$ ]] || return 1+ # shellcheck disable=SC2016 # JavaScript is intentionally literal.+ kctl exec deployment/"$HOST_NAME" -n "$HOST_NS" -c mcp-host -- \+ env "MCP_HOST_E2E_PORT=${port}" node -e '+ const http = require("node:http");+ const request = http.get(+ {host:"127.0.0.1",port:Number(process.env.MCP_HOST_E2E_PORT),path:"/v1/runtime/health"},+ response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ try {+ const parsed = JSON.parse(body);+ process.exit(response.statusCode === 200 && parsed?.status === "ok" ? 0 : 2);+ } catch { process.exit(3); }+ });+ }+ );+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));+ request.once("error", () => process.exit(4));+ ' >/dev/null 2>&1+}++host_reconcile_witness_count() {+ local logs+ logs="$(kctl logs deployment/"$HCC_DEPLOY" -n "$HCC_NS" --since=15m 2>/dev/null)" ||+ return 1+ jq -Rr --arg host "$HOST_NAME" '+ fromjson? |+ select(.host == $host) |+ (.msg // .message // "") |+ select(test("mcp-host-runtime-token Secret")) |+ 1+ ' <<<"$logs" | awk 'NF { count++ } END { print count + 0 }'+}++host_reconcile_witness_count_since() {+ local since_time=$1 host=${2:-} logs+ logs="$(kctl logs deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ --since-time="$since_time" 2>/dev/null)" || return 1+ jq -Rr --arg host "$host" '+ fromjson? |+ select($host == "" or .host == $host) |+ (.msg // .message // "") |+ select(test("mcp-host-runtime-token Secret")) |+ 1+ ' <<<"$logs" | awk 'NF { count++ } END { print count + 0 }'+}++host_reconcile_witness_exists() {+ local current+ current="$(host_reconcile_witness_count)" || return 1+ [ "$current" -gt 0 ]+}++host_fleet_reconcile_count_since() {+ local since_time=$1 logs+ logs="$(kctl logs deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ --since-time="$since_time" 2>/dev/null)" || return 1+ grep -Ec '\[K8s\] Reconciling [0-9]+ Host\(s\)' <<<"$logs" || true+}++context_netpol_failure_count_since() {+ local since_time=$1 logs+ logs="$(kctl logs deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ --since-time="$since_time" 2>/dev/null)" || return 1+ grep -Fc "[K8s] NetworkPolicy reconciliation failed for context ${CONTEXT_NAME}:" \+ <<<"$logs" || true+}++hcc_metric_value() {+ local metric=$1 required_labels=${2:-} pod+ pod="$(running_hcc_pod)" || return 1+ # shellcheck disable=SC2016 # JavaScript template interpolation is intentional.+ kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \+ env "HCC_E2E_PORT=${HCC_PORT}" "HCC_E2E_METRIC=${metric}" \+ "HCC_E2E_REQUIRED_LABELS=${required_labels}" node -e '+ const http = require("node:http");+ const request = http.get({host:"127.0.0.1",port:Number(process.env.HCC_E2E_PORT),path:"/metrics"}, response => {+ let body = "";+ response.setEncoding("utf8");+ response.on("data", chunk => { body += chunk; });+ response.on("end", () => {+ if (response.statusCode !== 200) process.exit(4);+ const metric = process.env.HCC_E2E_METRIC;+ const labels = (process.env.HCC_E2E_REQUIRED_LABELS || "")+ .split("|").filter(Boolean);+ const line = body.split("\n").find(value =>+ (value.startsWith(`${metric} `) || value.startsWith(`${metric}{`)) &&+ labels.every(label => value.includes(label)));+ if (!line) process.exit(2);+ process.stdout.write(line.trim().split(/\s+/).at(-1));+ });+ });+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));+ request.once("error", () => process.exit(3));+ ' 2>/dev/null+}++hcc_metadata_only_event_count() {+ hcc_metric_value clerum_hcc_context_metadata_only_events_total+}++hcc_metadata_only_event_after() {+ local before=$1 current+ current="$(hcc_metadata_only_event_count)" || return 1+ awk -v current="$current" -v before="$before" 'BEGIN { exit(current >= before + 1 ? 0 : 1) }'+}++hcc_netpol_certified_count() {+ hcc_metric_value clerum_hcc_initial_convergence_pass_results_total \+ 'lane="NetworkPolicy"|result="certified"'+}++hcc_netpol_certified_after() {+ local before=$1 current+ current="$(hcc_netpol_certified_count)" || return 1+ awk -v current="$current" -v before="$before" 'BEGIN { exit(current > before ? 0 : 1) }'+}++hcc_netpol_last_success_epoch() {+ hcc_metric_value clerum_hcc_initial_convergence_last_success_timestamp_seconds \+ 'lane="NetworkPolicy"'+}++hcc_netpol_timer_arm_epoch() {+ local pod logs timestamp+ pod="$(running_hcc_pod)" || return 1+ logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \+ --since=10m --timestamps=true 2>/dev/null)" || return 1+ timestamp="$(awk -v marker="NetworkPolicy periodic resync enabled (every ${RESYNC_SECONDS}s)" \+ 'index($0, marker) { value=$1 } END { print value }' <<<"$logs")"+ [ -n "$timestamp" ] || return 1+ node -e '+ const value = Date.parse(process.argv[1]);+ if (!Number.isFinite(value)) process.exit(2);+ process.stdout.write(String(Math.floor(value / 1000)));+ ' "$timestamp"+}++netpol_desired_inventory_snapshot() {+ local contexts servers+ contexts="$(kctl get context -A -o json 2>/dev/null)" || return 1+ servers="$(kctl get mcpserver -A -o json 2>/dev/null)" || return 1+ jq -ceS -n --argjson contexts "$contexts" --argjson servers "$servers" '+ {contexts:([$contexts.items[] |+ {namespace:.metadata.namespace,name:.metadata.name,uid:.metadata.uid,+ generation:.metadata.generation,spec:.spec}] | sort_by(.namespace,.name)),+ servers:([$servers.items[] |+ {namespace:.metadata.namespace,name:.metadata.name,uid:.metadata.uid,+ generation:.metadata.generation,spec:.spec,+ labels:(.metadata.labels // {}),+ annotations:((.metadata.annotations // {}) | del(."clerum.io/network-ready"))}] |+ sort_by(.namespace,.name))}+ '+}++netpol_watch_recovery_count_since() {+ local since_time=$1 logs+ logs="$(kctl logs deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ --since-time="$since_time" 2>/dev/null)" || return 1+ grep -Ec '\[K8s\] (Context|McpServer) watch ended; recovering authoritative inventory' \+ <<<"$logs" || true+}++hcc_host_urgent_success_count() {+ hcc_metric_value clerum_hcc_host_reconcile_duration_seconds_count \+ 'source="urgent"|outcome="success"'+}++wait_for_positive_host_fanout() {+ local timeout=$1 since_time=$2 urgent_before=$3 deadline+ local target_witnesses all_witnesses urgent fleet failures urgent_delta+ deadline=$(( $(date +%s) + timeout ))+ while [ "$(date +%s)" -lt "$deadline" ]; do+ target_witnesses="$(host_reconcile_witness_count_since "$since_time" "$HOST_NAME")" ||+ return 1+ all_witnesses="$(host_reconcile_witness_count_since "$since_time")" || return 1+ urgent="$(hcc_host_urgent_success_count)" || return 1+ fleet="$(host_fleet_reconcile_count_since "$since_time")" || return 1+ failures="$(context_netpol_failure_count_since "$since_time")" || return 1+ urgent_delta="$(awk -v current="$urgent" -v before="$urgent_before" \+ 'BEGIN { print current - before }')"+ if [ "$target_witnesses" -gt 1 ] || [ "$all_witnesses" -gt 1 ] ||+ [ "$all_witnesses" -gt "$target_witnesses" ] || [ "$fleet" -gt 0 ] ||+ [ "$failures" -gt 0 ] || awk -v delta="$urgent_delta" \+ -v target="$target_witnesses" \+ 'BEGIN { exit(delta > 1 || (delta > 0 && target == 0) ? 0 : 1) }'; then+ echo "Host activity outside the fixture made fan-out attribution inconclusive" >&2+ return 2+ fi+ if [ "$target_witnesses" = 1 ] && [ "$all_witnesses" = 1 ] &&+ awk -v delta="$urgent_delta" 'BEGIN { exit(delta == 1 ? 0 : 1) }'; then+ return 0+ fi+ sleep 1+ done+ echo "Timed out after ${timeout}s waiting for one isolated Host fan-out" >&2+ return 1+}++host_activity_stays() {+ local expected_witnesses=$1 expected_urgent=$2 seconds=$3 deadline witnesses urgent+ deadline=$(( $(date +%s) + seconds ))+ while [ "$(date +%s)" -lt "$deadline" ]; do+ witnesses="$(host_reconcile_witness_count)" || return 1+ urgent="$(hcc_host_urgent_success_count)" || return 1+ [ "$witnesses" = "$expected_witnesses" ] || return 1+ [ "$urgent" = "$expected_urgent" ] || return 1+ sleep 1+ done+}++target_host_activity_stays_absent() {+ local since_time=$1 seconds=$2 deadline witnesses+ deadline=$(( $(date +%s) + seconds ))+ while [ "$(date +%s)" -lt "$deadline" ]; do+ witnesses="$(host_reconcile_witness_count_since "$since_time" "$HOST_NAME")" || return 1+ [ "$witnesses" = 0 ] || return 1+ sleep 1+ done+}+context_identity_is_original() {local identityidentity="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \@@ -261,11 +592,14 @@ context_identity_is_original() {mcp_generation_is_original() {[ "$(kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" \- -o jsonpath='{.metadata.generation}' 2>/dev/null)" = "$ORIGINAL_MCP_GENERATION" ]+ -o jsonpath='{.metadata.generation}' 2>/dev/null)" = "$ORIGINAL_MCP_GENERATION" ] &&+ [ "$(kctl get mcpserver "$SECOND_MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.generation}' 2>/dev/null)" = "$ORIGINAL_SECOND_MCP_GENERATION" ]}-context_policy_snapshot() {- kctl get networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" -o json 2>/dev/null |+policy_snapshot() {+ local namespace=$1 name=$2+ kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null |jq -ceS '{labels:{managedBy:.metadata.labels["clerum.io/managed-by"],policyType:.metadata.labels["clerum.io/policy-type"],@@ -273,17 +607,26 @@ context_policy_snapshot() {mcpserver:.metadata.labels["clerum.io/mcpserver"]},spec}'}-context_policy_recreated_from_snapshot() {- local live uid snapshot- live="$(kctl get networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" -o json 2>/dev/null)" || return 1+policy_recreated_from_snapshot() {+ local namespace=$1 name=$2 original_uid=$3 original_snapshot=$4 live uid snapshot+ live="$(kctl get networkpolicy "$name" -n "$namespace" -o json 2>/dev/null)" || return 1uid="$(jq -r '.metadata.uid // ""' <<<"$live")"- [ -n "$uid" ] && [ "$uid" != "$ORIGINAL_POLICY_UID" ] || return 1+ [ -n "$uid" ] && [ "$uid" != "$original_uid" ] || return 1snapshot="$(jq -ceS '{labels:{managedBy:.metadata.labels["clerum.io/managed-by"],policyType:.metadata.labels["clerum.io/policy-type"],context:.metadata.labels["clerum.io/context"],mcpserver:.metadata.labels["clerum.io/mcpserver"]},spec}' <<<"$live")" || return 1- [ "$snapshot" = "$ORIGINAL_POLICY_SNAPSHOT" ]+ [ "$snapshot" = "$original_snapshot" ]+}++context_policy_snapshot() {+ policy_snapshot "$MCP_NS" "$CONTEXT_POLICY"+}++context_policy_recreated_from_snapshot() {+ policy_recreated_from_snapshot \+ "$MCP_NS" "$CONTEXT_POLICY" "$ORIGINAL_POLICY_UID" "$ORIGINAL_POLICY_SNAPSHOT"}probe_mcp_tcp_connectivity() {@@ -298,14 +641,32 @@ probe_mcp_tcp_connectivity() {}mcp_data_plane_is_policy_blocked() {- resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&- mcp_runtime_still_ready &&- ! probe_mcp_tcp_connectivity+ local connectivity_rc+ resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" || return 1+ mcp_runtime_still_ready || return 1+ [ "$(selector_runtime_identity "$HOST_NS" "app=${PROBE_NAME}")" = \+ "$PROBE_BASELINE_IDENTITY" ] || return 1+ [ "$(selector_runtime_identity "$MCP_NS" "$WORKLOAD_SELECTOR")" = \+ "$PRIMARY_MCP_BASELINE_IDENTITY" ] || return 1+ [ "$(selector_runtime_identity "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR")" = \+ "$SECOND_MCP_BASELINE_IDENTITY" ] || return 1+ if probe_mcp_tcp_connectivity; then+ return 1+ else+ connectivity_rc=$?+ fi+ # The remote Node probe reserves 42 for a TCP timeout. DNS, connection+ # refusal, kubectl/API failures, or an unhealthy probe return a different code.+ [ "$connectivity_rc" = 42 ] || return 1+ # Same pod, DNS, namespace, and CNI path; only the deleted primary policy+ # differs. A successful exact tool call rules out a cluster-wide timeout.+ probe_mcp_business_signal "$SECOND_MCP_NAME"}-probe_mcp_business_signal() {+probe_mcp_business_signal_from() {+ local target=$1 namespace=$2 container=$3 server=$4# shellcheck disable=SC2016 # JavaScript is intentionally literal.- kctl exec deployment/"$PROBE_NAME" -n "$HOST_NS" -c probe -- \+ kctl exec "$target" -n "$namespace" -c "$container" -- \node -e 'function parse(text) {try { return JSON.parse(text); } catch {@@ -335,13 +696,23 @@ probe_mcp_business_signal() {if (called.parsed?.error || called.parsed?.result?.isError === true) {throw new Error(`tool call failed: ${called.text}`);}- if (!JSON.stringify(called.parsed?.result).includes("42")) {- throw new Error(`add result did not contain 42: ${called.text}`);+ const content = called.parsed?.result?.content;+ if (!Array.isArray(content) || content.length !== 1 ||+ content[0]?.type !== "text" || content[0]?.text !== "42") {+ throw new Error(`add result was not exactly one text value 42: ${called.text}`);}})().then(() => process.exit(0)).catch(error => {console.error(error.message); process.exit(1);});- ' "http://${MCP_NAME}.${MCP_NS}.svc.cluster.local:3000/mcp" >/dev/null+ ' "http://${server}.${MCP_NS}.svc.cluster.local:3000/mcp" >/dev/null+}++probe_mcp_business_signal() {+ probe_mcp_business_signal_from deployment/"$PROBE_NAME" "$HOST_NS" probe "${1:-$MCP_NAME}"+}++probe_host_mcp_business_signal() {+ probe_mcp_business_signal_from deployment/"$HOST_NAME" "$HOST_NS" mcp-host "${1:-$MCP_NAME}"}hcc_resync_config_is_expected() {@@ -388,13 +759,44 @@ restore_hcc_resync_config() {fixture_resources_absent() {resource_absent workflowrecipe "$RECIPE_NAME" "$WORKFLOW_NS" &&resource_absent context "$CONTEXT_NAME" "$MCP_NS" &&+ resource_absent host "$HOST_NAME" "$HOST_NS" &&resource_absent mcpserver "$MCP_NAME" "$MCP_NS" &&+ resource_absent mcpserver "$SECOND_MCP_NAME" "$MCP_NS" &&resource_collection_absent deployment "$MCP_NS" "$WORKLOAD_SELECTOR" &&+ resource_collection_absent replicaset "$MCP_NS" "$WORKLOAD_SELECTOR" &&+ resource_collection_absent pod "$MCP_NS" "$WORKLOAD_SELECTOR" &&+ resource_collection_absent deployment "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR" &&+ resource_collection_absent replicaset "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR" &&+ resource_collection_absent pod "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR" &&resource_absent service "$MCP_NAME" "$MCP_NS" &&+ resource_absent service "$SECOND_MCP_NAME" "$MCP_NS" &&resource_absent deployment "$PROBE_NAME" "$HOST_NS" &&+ resource_collection_absent replicaset "$HOST_NS" "app=${PROBE_NAME}" &&+ resource_collection_absent pod "$HOST_NS" "app=${PROBE_NAME}" &&resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&resource_absent networkpolicy "$CONTEXT_EGRESS_POLICY" "$HOST_NS" &&- resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS"+ resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS" &&+ resource_absent networkpolicy "$SECOND_CONTEXT_POLICY" "$MCP_NS" &&+ resource_absent networkpolicy "$SECOND_CONTEXT_EGRESS_POLICY" "$HOST_NS" &&+ resource_absent networkpolicy "$SECOND_RPC_EGRESS_POLICY" "$RPC_PROXY_NS" &&+ wrc_managed_resources_absent &&+ host_managed_resources_absent+}++wrc_managed_resources_absent() {+ local resources+ resources="$(kctl get "$WRC_CLEANUP_KINDS" -A \+ -l "clerum.io/recipe=${RECIPE_NAME}" -o name 2>/dev/null)" || return 1+ [ -z "$resources" ]+}++host_managed_resources_absent() {+ local resources+ resources="$(kctl get \+ deployment,replicaset,pod,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${HOST_NAME}" \+ -o name 2>/dev/null)" || return 1+ [ -z "$resources" ]}print_repair_instructions() {@@ -402,7 +804,7 @@ print_repair_instructions() {WRC/HCC Context no-op resync gate cleanup could not restore a verified clean state.Context: ${E2E_KUBECONTEXT}HCC: ${HCC_NS}/${HCC_DEPLOY}-Fixture: ${WORKFLOW_NS}/${RECIPE_NAME}, ${MCP_NS}/${CONTEXT_NAME}, ${MCP_NS}/${MCP_NAME}+Fixture: ${WORKFLOW_NS}/${RECIPE_NAME}, ${MCP_NS}/${CONTEXT_NAME}, ${MCP_NS}/${MCP_NAME}, ${HOST_NS}/${HOST_NAME}Inspect before changing anything:kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml@@ -422,6 +824,14 @@ cleanup() {kctl delete deployment "$PROBE_NAME" -n "$HOST_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1fi+ if [ "$HOST_CREATED" = 1 ]; then+ kctl delete host "$HOST_NAME" -n "$HOST_NS" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete \+ deployment,replicaset,pod,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${HOST_NAME}" \+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || cleanup_failed=1+ fiif [ "$RECIPE_CREATED" = 1 ]; thenkctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_NS" --ignore-not-found \--wait=false >/dev/null 2>&1 || cleanup_failed=1@@ -435,16 +845,30 @@ cleanup() {if [ "$RECIPE_CREATED" = 1 ] || [ "$CONTEXT_CREATED" = 1 ]; thenkctl delete mcpserver "$MCP_NAME" -n "$MCP_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete mcpserver "$SECOND_MCP_NAME" -n "$MCP_NS" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1kctl delete deployment -n "$MCP_NS" -l "$WORKLOAD_SELECTOR" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete deployment -n "$MCP_NS" -l "$SECOND_WORKLOAD_SELECTOR" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1kctl delete service "$MCP_NAME" -n "$MCP_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete service "$SECOND_MCP_NAME" -n "$MCP_NS" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1kctl delete networkpolicy "$CONTEXT_EGRESS_POLICY" -n "$HOST_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1kctl delete networkpolicy "$RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" --ignore-not-found \--wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete networkpolicy "$SECOND_CONTEXT_POLICY" -n "$MCP_NS" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete networkpolicy "$SECOND_CONTEXT_EGRESS_POLICY" -n "$HOST_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete networkpolicy "$SECOND_RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_failed=1+ kctl delete "$WRC_CLEANUP_KINDS" -A -l "clerum.io/recipe=${RECIPE_NAME}" \+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || cleanup_failed=1wait_until 90 "all WRC/HCC Context no-op fixtures to disappear" fixture_resources_absent \>/dev/null 2>&1 || cleanup_failed=1fi@@ -516,19 +940,6 @@ CONTEXT_CREATED=1RECIPE_CREATED=1kctl apply -f - >/dev/null <<EOFapiVersion: clerum.io/v1alpha1-kind: Context-metadata:- name: ${CONTEXT_NAME}- namespace: ${MCP_NS}- labels:- e2e.clerum.io/suite: ${SUITE_NAME}- e2e.clerum.io/run: "${RUN_LABEL}"-spec:- contextId: ${CONTEXT_NAME}- description: Shared Context entry point for the PR 568 cross-controller gate.- mcpServers: []-----apiVersion: clerum.io/v1alpha1kind: WorkflowRecipemetadata:name: ${RECIPE_NAME}@@ -562,6 +973,46 @@ wait_until 300 "WRC projection, MCP runtime, status, and all three HCC policies"context_projection_converged ||die "WorkflowRecipe did not converge through WRC, Context watch, HCC policies, and MCP runtime"ok "real WorkflowRecipe converged through WRC, apiserver, Context watch, HCC, and MCP runtime"+ok "WRC created the absent shared Context without a recipe label or cross-namespace ownerReference"++source_host="$(kctl get host "$SOURCE_HOST_NAME" -n "$HOST_NS" -o json 2>/dev/null)" ||+ die "source Host ${HOST_NS}/${SOURCE_HOST_NAME} is unavailable"+source_secret="$(jq -r '.spec.secretRef // ""' <<<"$source_host")"+source_provider="$(jq -r '.spec.model.provider // ""' <<<"$source_host")"+source_model="$(jq -r '.spec.model.name // ""' <<<"$source_host")"+[ -n "$source_secret" ] && [ -n "$source_provider" ] && [ -n "$source_model" ] ||+ die "source Host does not expose a reusable secretRef/provider/model contract"+kctl get secret "$source_secret" -n "$HOST_NS" >/dev/null 2>&1 ||+ die "source Host Secret ${HOST_NS}/${source_secret} is unavailable"+ok "source Host and its existing Secret/provider/model contract passed preflight without reading Secret data"++HOST_CREATED=1+jq -n --arg name "$HOST_NAME" --arg namespace "$HOST_NS" --arg context "$CONTEXT_NAME" \+ --arg secret "$source_secret" --arg provider "$source_provider" --arg model "$source_model" \+ --arg suite "$SUITE_NAME" --arg run "$RUN_LABEL" '+ {apiVersion:"clerum.io/v1alpha1",kind:"Host",+ metadata:{name:$name,namespace:$namespace,+ labels:{"e2e.clerum.io/suite":$suite,"e2e.clerum.io/run":$run}},+ spec:{host:$name,contextRef:$context,secretRef:$secret,+ lifecycle:{stateless:false},model:{provider:$provider,name:$model}}}+ ' | kctl apply -f - >/dev/null+wait_until 240 "gate-owned Host runtime to become Ready" host_runtime_ready ||+ die "gate-owned Host did not converge from the real Host CRD"+wait_until 60 "initial Host reconcile witness" host_reconcile_witness_exists ||+ die "gate-owned Host reached Ready without a host-specific reconcile witness"+wait_until 60 "gate-owned Host runtime health" probe_host_runtime_health ||+ die "gate-owned Host pod was Ready but its runtime health endpoint was not"+HOST_BASELINE_IDENTITY="$(host_deployment_identity)" ||+ die "could not capture the gate-owned Host deployment identity"+initial_host_reconciles="$(host_reconcile_witness_count)" ||+ die "could not capture the initial host-specific reconcile count"+initial_host_urgent="$(hcc_host_urgent_success_count)" ||+ die "could not capture the initial urgent Host reconcile metric"+host_activity_stays "$initial_host_reconciles" "$initial_host_urgent" 5 ||+ die "gate-owned Host did not reach a quiet baseline before the Context fan-out proof"+wait_until 60 "primary MCP signal from the real Host" probe_host_mcp_business_signal "$MCP_NAME" ||+ die "the correctly configured Host could not invoke the primary MCP tool"+ok "real gate-owned Host is Ready on the WRC-created shared Context"PROBE_CREATED=1kctl apply -f - >/dev/null <<EOF@@ -611,12 +1062,68 @@ wait_until 60 "baseline MCP add(19,23) business signal" probe_mcp_business_signadie "baseline MCP business signal did not return 42 through Context policies"ok "baseline mcp-host-labelled probe invoked add(19,23) and received 42"+read -r context_before_real_rv context_before_real_generation <<<"$(+ kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.resourceVersion}{" "}{.metadata.generation}'+)"+host_urgent_before="$(hcc_host_urgent_success_count)" ||+ die "could not establish the urgent Host reconcile metric baseline"+fanout_window_start="$(utc_now)" || die "could not timestamp the positive Host fan-out window"+reconciles_before="$(wrc_reconcile_count)" || die "could not establish WRC reconcile log baseline"+second_workload_patch="$(jq -cn '[{op:"add",path:"/spec/workloads/-",value:{+ id:"mock-tools-2",type:"deployment",image:"clerum/mock-mcp-server:test",+ imagePullPolicy:"IfNotPresent",port:3000,+ transport:{type:"streamableHttp",path:"/mcp"},+ healthCheck:{type:"tcp",port:3001},+ resources:{requests:{cpu:"20m",memory:"48Mi"},limits:{cpu:"150m",memory:"128Mi"}}+}}]')"+kctl patch workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_NS" --type=json \+ -p "$second_workload_patch" >/dev/null+wait_until 90 "WRC to process the real second-workload change" \+ wrc_reconciled_after "$reconciles_before" ||+ die "WRC did not visibly reconcile the real recipe change"+wait_until 300 "the two-server WRC/Context/HCC projection" expanded_context_projection_converged ||+ die "the real recipe change did not converge both MCP runtimes and six Context policies"+read -r context_after_real_rv context_after_real_generation <<<"$(+ kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.resourceVersion}{" "}{.metadata.generation}'+)"+[ "$context_after_real_rv" != "$context_before_real_rv" ] ||+ die "the real recipe change did not move Context resourceVersion"+if ! [[ "$context_before_real_generation" =~ ^[0-9]+$ ]] ||+ ! [[ "$context_after_real_generation" =~ ^[0-9]+$ ]] ||+ [ "$context_after_real_generation" -ne $((context_before_real_generation + 1)) ]; then+ die "the real recipe change did not produce exactly one Context spec transition"+fi+wait_for_positive_host_fanout 90 "$fanout_window_start" "$host_urgent_before" ||+ die "the Context change lacked one isolated, successful Host fan-out after NetworkPolicy convergence"+host_reconciles_after_real_change="$(+ host_reconcile_witness_count_since "$fanout_window_start" "$HOST_NAME"+)" ||+ die "could not capture the post-change Host reconcile witness"+host_urgent_after_real_change="$(hcc_host_urgent_success_count)" ||+ die "could not capture the post-change urgent Host reconcile metric"+host_runtime_ready || die "the bound Host degraded after a real Context change"+[ "$(host_deployment_identity)" = "$HOST_BASELINE_IDENTITY" ] ||+ die "a no-mount Context server change unnecessarily restarted or rewrote the bound Host"+probe_host_runtime_health || die "the Host runtime health signal failed after the real Context change"+wait_until 60 "primary MCP add(19,23) signal from the real Host after the change" \+ probe_host_mcp_business_signal "$MCP_NAME" ||+ die "the primary MCP signal from the real Host regressed after the Context change"+wait_until 60 "second MCP add(19,23) signal from the real Host after the change" \+ probe_host_mcp_business_signal "$SECOND_MCP_NAME" ||+ die "the real Host could not invoke the second WRC-projected MCP tool"+probe_hcc_ready || die "HCC lost readiness after the real Context change"+ok "real WRC Context change fanned out to a stable, Ready Host and both MCP tools returned 42"+ORIGINAL_CONTEXT_RV="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \-o jsonpath='{.metadata.resourceVersion}')"ORIGINAL_CONTEXT_GENERATION="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \-o jsonpath='{.metadata.generation}')"ORIGINAL_MCP_GENERATION="$(kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" \-o jsonpath='{.metadata.generation}')"+ORIGINAL_SECOND_MCP_GENERATION="$(kctl get mcpserver "$SECOND_MCP_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.generation}')"reconciles_before="$(wrc_reconcile_count)" || die "could not establish WRC reconcile log baseline"description_patch="$(jq -cn --arg description \'PR 568 no-op projection after a real recipe-only change.' '{spec:{description:$description}}')"@@ -630,7 +1137,74 @@ context_identity_is_original ||mcp_generation_is_original ||die "a recipe-only change unexpectedly changed McpServer desired generation"hcc_identity_is_stable || die "HCC restarted during the WRC no-op projection proof"-ok "WRC processed a real recipe change without moving Context resourceVersion"+host_runtime_ready || die "the bound Host degraded during the WRC no-op projection proof"+ok "WRC processed a recipe-only change without moving Context resourceVersion"++metadata_events_before="$(hcc_metadata_only_event_count)" ||+ die "could not establish the HCC metadata-only counter baseline"+host_identity_before_metadata="$(host_deployment_identity)" ||+ die "could not capture Host identity before metadata-only MODIFIED"+metadata_context_before_rv="$ORIGINAL_CONTEXT_RV"+metadata_context_before_generation="$ORIGINAL_CONTEXT_GENERATION"+metadata_policy_uid="$(kctl get networkpolicy "$SECOND_RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \+ -o jsonpath='{.metadata.uid}')"+metadata_policy_snapshot="$(policy_snapshot "$RPC_PROXY_NS" "$SECOND_RPC_EGRESS_POLICY")" ||+ die "could not capture the metadata-only scoped NetworkPolicy fixture"+[ -n "$metadata_policy_uid" ] && [ -n "$metadata_policy_snapshot" ] ||+ die "metadata-only scoped NetworkPolicy fixture was empty"+kctl delete networkpolicy "$SECOND_RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \+ --wait=true --timeout=60s >/dev/null+resource_absent networkpolicy "$SECOND_RPC_EGRESS_POLICY" "$RPC_PROXY_NS" ||+ die "metadata-only scoped NetworkPolicy fault was not observable"+metadata_window_start="$(utc_now)" || die "could not timestamp the metadata-only event window"+kctl annotate context "$CONTEXT_NAME" -n "$MCP_NS" \+ "e2e.clerum.io/metadata-touch=${RUN_LABEL}" --overwrite >/dev/null+wait_until 60 "HCC metadata-only Context counter to advance" \+ hcc_metadata_only_event_after "$metadata_events_before" ||+ die "HCC did not classify the annotation-only Context MODIFIED as metadata-only"+metadata_events_after="$(hcc_metadata_only_event_count)" ||+ die "could not capture the post-event HCC metadata-only counter"+wait_until 90 "metadata-only scoped NetworkPolicy repair" \+ policy_recreated_from_snapshot "$RPC_PROXY_NS" "$SECOND_RPC_EGRESS_POLICY" \+ "$metadata_policy_uid" "$metadata_policy_snapshot" ||+ die "metadata-only Context MODIFIED did not complete its scoped NetworkPolicy repair"+read -r metadata_context_after_rv metadata_context_after_generation <<<"$(+ kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.resourceVersion}{" "}{.metadata.generation}'+)"+[ "$metadata_context_after_rv" != "$metadata_context_before_rv" ] ||+ die "the annotation fault did not emit a real Context MODIFIED resourceVersion"+[ "$metadata_context_after_generation" = "$metadata_context_before_generation" ] ||+ die "the metadata-only Context event unexpectedly changed desired generation"+target_host_activity_stays_absent "$metadata_window_start" 30 ||+ die "metadata-only Context MODIFIED incorrectly fanned out to the bound Host"+[ "$(context_netpol_failure_count_since "$metadata_window_start")" = 0 ] ||+ die "metadata-only NetworkPolicy convergence failed and made the Host skip inconclusive"+[ "$(host_deployment_identity)" = "$host_identity_before_metadata" ] ||+ die "metadata-only Context MODIFIED rewrote or restarted the bound Host"+host_runtime_ready || die "the bound Host degraded after metadata-only Context MODIFIED"+probe_host_runtime_health || die "the Host runtime health signal failed after metadata-only MODIFIED"+expanded_context_projection_converged ||+ die "metadata-only Context MODIFIED disturbed MCP or NetworkPolicy desired state"+if ! probe_mcp_business_signal "$MCP_NAME" || ! probe_mcp_business_signal "$SECOND_MCP_NAME"; then+ die "metadata-only Context MODIFIED interrupted an MCP business signal"+fi+if ! probe_host_mcp_business_signal "$MCP_NAME" ||+ ! probe_host_mcp_business_signal "$SECOND_MCP_NAME"; then+ die "metadata-only Context MODIFIED interrupted the real Host-to-MCP path"+fi+hcc_identity_is_stable || die "HCC restarted during metadata-only Context handling"+probe_hcc_ready || die "HCC lost readiness during metadata-only Context handling"+[ "$(host_reconcile_witness_count_since "$metadata_window_start" "$HOST_NAME")" = 0 ] ||+ die "the bound Host reconciled after the metadata-only stability window"+ok "metadata-only Context MODIFIED kept the real Host stable while scoped NetworkPolicy and both MCP tools stayed live"++# The annotation-only event legitimately moves resourceVersion. Freeze the new+# identity for the periodic-policy recovery phase that follows.+ORIGINAL_CONTEXT_RV="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.resourceVersion}')"+ORIGINAL_CONTEXT_GENERATION="$(kctl get context "$CONTEXT_NAME" -n "$MCP_NS" \+ -o jsonpath='{.metadata.generation}')"# Arm the timer only after the WRC no-op proof. A final rollout to the requested# interval creates a fresh, bounded observation window for the policy-delete@@ -651,6 +1225,8 @@ wait_until 60 "HCC resync config and readiness" hcc_resync_config_is_expected ||die "HCC did not expose the requested NetworkPolicy resync interval"wait_until 60 "HCC /ready after arming periodic convergence" probe_hcc_ready ||die "HCC /ready did not recover after arming periodic convergence"+timer_arm_epoch="$(hcc_netpol_timer_arm_epoch)" ||+ die "could not timestamp the freshly armed NetworkPolicy periodic timer"read -r HCC_UID HCC_RESTARTS <<<"$(kctl get pod "$(running_hcc_pod)" -n "$HCC_NS" \-o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}'@@ -658,14 +1234,19 @@ read -r HCC_UID HCC_RESTARTS <<<"$([ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] || die "could not capture armed HCC pod identity"context_identity_is_original || die "arming HCC changed Context desired identity"mcp_generation_is_original || die "arming HCC changed McpServer desired generation"-wait_until 120 "Context policies after the HCC timer rollout" context_projection_converged ||- die "Context policies did not remain converged after the HCC timer rollout"-wait_until 60 "MCP business signal after the HCC timer rollout" probe_mcp_business_signal ||- die "MCP business signal failed before policy fault injection"+wait_until 120 "Context policies after the HCC timer rollout" \+ expanded_context_projection_converged ||+ die "two-server Context policies did not remain converged after the HCC timer rollout"+wait_until 60 "primary MCP business signal after the HCC timer rollout" \+ probe_mcp_business_signal "$MCP_NAME" ||+ die "primary MCP business signal failed before policy fault injection"+wait_until 60 "second MCP business signal after the HCC timer rollout" \+ probe_mcp_business_signal "$SECOND_MCP_NAME" ||+ die "second MCP business signal failed before policy fault injection"+host_runtime_ready || die "the real Host degraded during the HCC timer rollout"wait_until 180 "the armed HCC startup NetworkPolicy pass to complete" \hcc_log_contains "[NetPol] Full reconciliation complete" ||die "HCC startup NetworkPolicy pass did not complete before policy fault injection"-ok "HCC periodic NetworkPolicy resync is freshly armed at ${RESYNC_SECONDS}s on a stable pod"ORIGINAL_POLICY_UID="$(kctl get networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" \-o jsonpath='{.metadata.uid}')"@@ -673,10 +1254,50 @@ ORIGINAL_POLICY_SNAPSHOT="$(context_policy_snapshot)" ||die "could not capture the canonical Context ingress policy"[ -n "$ORIGINAL_POLICY_UID" ] && [ -n "$ORIGINAL_POLICY_SNAPSHOT" ] ||die "Context ingress policy baseline was empty"+PROBE_BASELINE_IDENTITY="$(selector_runtime_identity "$HOST_NS" "app=${PROBE_NAME}")" ||+ die "could not freeze the probe runtime identity before policy fault injection"+PRIMARY_MCP_BASELINE_IDENTITY="$(selector_runtime_identity "$MCP_NS" "$WORKLOAD_SELECTOR")" ||+ die "could not freeze the primary MCP runtime identity before policy fault injection"+SECOND_MCP_BASELINE_IDENTITY="$(selector_runtime_identity "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR")" ||+ die "could not freeze the second MCP runtime identity before policy fault injection"+probe_host_mcp_business_signal "$MCP_NAME" ||+ die "the real Host-to-MCP path was unavailable immediately before fault injection"++timer_now_epoch="$(date +%s)"+[ "$timer_arm_epoch" -le "$timer_now_epoch" ] || die "NetworkPolicy timer timestamp is in the future"+timer_tick_due_epoch=$((+ timer_arm_epoch + (((timer_now_epoch - timer_arm_epoch) / RESYNC_SECONDS) + 1) * RESYNC_SECONDS+))+if [ $((timer_tick_due_epoch - timer_now_epoch)) -lt 30 ]; then+ timer_window_passes="$(hcc_netpol_certified_count)" ||+ die "could not capture a NetworkPolicy pass before opening a fresh timer window"+ wait_until "$RESYNC_TIMEOUT_SECONDS" "the imminent timer pass to finish before fault injection" \+ hcc_netpol_certified_after "$timer_window_passes" ||+ die "could not establish a fresh post-pass timer window"+ timer_now_epoch="$(date +%s)"+ timer_tick_due_epoch=$((+ timer_arm_epoch + (((timer_now_epoch - timer_arm_epoch) / RESYNC_SECONDS) + 1) * RESYNC_SECONDS+ ))+fi+[ $((timer_tick_due_epoch - timer_now_epoch)) -ge 30 ] ||+ die "NetworkPolicy timer did not leave a 30-second fault-injection window"+netpol_certified_before_fault="$(hcc_netpol_certified_count)" ||+ die "could not capture the certified NetworkPolicy pass baseline"+netpol_last_success_before_fault="$(hcc_netpol_last_success_epoch)" ||+ die "could not capture the last NetworkPolicy success timestamp"+netpol_desired_before_fault="$(netpol_desired_inventory_snapshot)" ||+ die "could not freeze the NetworkPolicy desired inventory"+metadata_events_before_fault="$(hcc_metadata_only_event_count)" ||+ die "could not freeze the metadata-only event counter before the timer fault"+fault_window_start="$(utc_now)" || die "could not timestamp the timer fault window"+ok "HCC periodic NetworkPolicy resync is freshly armed at ${RESYNC_SECONDS}s on a stable pod"kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" --wait=true --timeout=60s >/dev/nullresource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" ||die "fault injection did not remove the Context ingress policy"+fault_confirmed_epoch="$(date +%s)"+[ "$timer_tick_due_epoch" -gt "$fault_confirmed_epoch" ] ||+ die "the policy fault was not confirmed before the next periodic timer deadline"wait_until 30 "Calico to enforce the missing Context ingress policy while MCP stays Ready" \mcp_data_plane_is_policy_blocked ||die "MCP did not become policy-isolated while its workload and Service endpoints stayed Ready"@@ -686,26 +1307,84 @@ wait_until "$RESYNC_TIMEOUT_SECONDS" \"HCC periodic convergence to recreate the canonical Context policy" \context_policy_recreated_from_snapshot ||die "HCC periodic convergence did not recreate the deleted Context policy"+policy_recovery_epoch="$(date +%s)"+[ "$policy_recovery_epoch" -ge $((timer_tick_due_epoch - 2)) ] ||+ die "the policy recovered before the armed periodic tick and cannot be attributed to the timer"+wait_until "$RESYNC_TIMEOUT_SECONDS" "a post-fault certified NetworkPolicy pass" \+ hcc_netpol_certified_after "$netpol_certified_before_fault" ||+ die "policy recreation was not backed by a certified post-fault NetworkPolicy pass"+netpol_certified_after_fault="$(hcc_netpol_certified_count)" ||+ die "could not read the post-fault certified NetworkPolicy pass count"+awk -v after="$netpol_certified_after_fault" -v before="$netpol_certified_before_fault" \+ 'BEGIN { exit(after == before + 1 ? 0 : 1) }' ||+ die "more than one NetworkPolicy pass ran in the timer attribution window"+netpol_last_success_after_fault="$(hcc_netpol_last_success_epoch)" ||+ die "could not read the post-fault NetworkPolicy success timestamp"+awk -v after="$netpol_last_success_after_fault" -v before="$netpol_last_success_before_fault" \+ -v fault="$fault_confirmed_epoch" -v due="$timer_tick_due_epoch" \+ 'BEGIN { exit(after > before && after >= fault && after >= due - 2 ? 0 : 1) }' ||+ die "the certified NetworkPolicy pass timestamp does not belong to the post-fault timer window"+[ "$(netpol_desired_inventory_snapshot)" = "$netpol_desired_before_fault" ] ||+ die "Context/McpServer desired inventory changed during timer attribution"+[ "$(hcc_metadata_only_event_count)" = "$metadata_events_before_fault" ] ||+ die "a metadata-only Context event made timer attribution inconclusive"+[ "$(netpol_watch_recovery_count_since "$fault_window_start")" = 0 ] ||+ die "Context/McpServer watch recovery made timer attribution inconclusive"+policy_recovery_elapsed=$((policy_recovery_epoch - fault_confirmed_epoch))+[ "$policy_recovery_elapsed" -le "$RESYNC_TIMEOUT_SECONDS" ] ||+ die "periodic policy recovery exceeded the configured test timeout"context_identity_is_original ||die "Context desired state changed while waiting for periodic policy recovery"mcp_generation_is_original ||die "McpServer desired state changed while waiting for periodic policy recovery"hcc_identity_is_stable ||die "HCC restarted instead of healing the deleted policy in process"-wait_until 60 "MCP business signal after periodic policy recovery" probe_mcp_business_signal ||+[ "$(selector_runtime_identity "$HOST_NS" "app=${PROBE_NAME}")" = \+ "$PROBE_BASELINE_IDENTITY" ] || die "the probe changed identity during policy recovery"+[ "$(selector_runtime_identity "$MCP_NS" "$WORKLOAD_SELECTOR")" = \+ "$PRIMARY_MCP_BASELINE_IDENTITY" ] || die "the MCP runtime changed during policy recovery"+[ "$(selector_runtime_identity "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR")" = \+ "$SECOND_MCP_BASELINE_IDENTITY" ] || die "the unaffected MCP runtime changed during recovery"+wait_until 60 "MCP business signal after periodic policy recovery" \+ probe_mcp_business_signal "$MCP_NAME" ||die "recreated policy did not restore the MCP add(19,23) business signal"+probe_mcp_business_signal "$SECOND_MCP_NAME" ||+ die "periodic recovery disturbed the unaffected second MCP business signal"+host_runtime_ready || die "the real Host degraded after periodic policy recovery"+probe_host_mcp_business_signal "$MCP_NAME" ||+ die "periodic recovery did not restore the real Host-to-primary-MCP business path"+probe_host_mcp_business_signal "$SECOND_MCP_NAME" ||+ die "periodic recovery disturbed the real Host-to-second-MCP business path"probe_hcc_ready || die "HCC lost readiness after periodic policy recovery"ok "HCC recreated the policy without a Context/McpServer change or process restart"ok "the restored data plane again returned 42 from the MCP add tool"FINAL_SUMMARY="$(jq -cn \--arg recipe "$RECIPE_NAME" --arg context "$CONTEXT_NAME" --arg server "$MCP_NAME" \+ --arg secondServer "$SECOND_MCP_NAME" --arg host "$HOST_NAME" \--arg oldPolicyUid "$ORIGINAL_POLICY_UID" \--arg newPolicyUid "$(kctl get networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" -o jsonpath='{.metadata.uid}')" \--arg contextResourceVersion "$ORIGINAL_CONTEXT_RV" \--arg hccPodUid "$HCC_UID" --argjson resyncSeconds "$RESYNC_SECONDS" \- '{recipe:$recipe,context:$context,server:$server,resyncSeconds:$resyncSeconds,+ --argjson metadataOnlyBefore "$metadata_events_before" \+ --argjson metadataOnlyAfter "$metadata_events_after" \+ --argjson hostReconcilesAfterRealChange "$host_reconciles_after_real_change" \+ --argjson hostUrgentBeforeRealChange "$host_urgent_before" \+ --argjson hostUrgentAfterRealChange "$host_urgent_after_real_change" \+ --argjson timerTickDueEpoch "$timer_tick_due_epoch" \+ --argjson policyRecoveryElapsedSeconds "$policy_recovery_elapsed" \+ --argjson netpolCertifiedBeforeFault "$netpol_certified_before_fault" \+ --argjson netpolCertifiedAfterFault "$netpol_certified_after_fault" \+ '{recipe:$recipe,context:$context,servers:[$server,$secondServer],host:$host,+ resyncSeconds:$resyncSeconds,metadataOnlyBefore:$metadataOnlyBefore,+ metadataOnlyAfter:$metadataOnlyAfter,hostReconcilesAfterRealChange:$hostReconcilesAfterRealChange,+ hostUrgentBeforeRealChange:$hostUrgentBeforeRealChange,+ hostUrgentAfterRealChange:$hostUrgentAfterRealChange,timerTickDueEpoch:$timerTickDueEpoch,+ policyRecoveryElapsedSeconds:$policyRecoveryElapsedSeconds,+ netpolCertifiedBeforeFault:$netpolCertifiedBeforeFault,+ netpolCertifiedAfterFault:$netpolCertifiedAfterFault,contextResourceVersion:$contextResourceVersion,hccPodUid:$hccPodUid,- oldPolicyUid:$oldPolicyUid,newPolicyUid:$newPolicyUid,businessSignal:"add(19,23)=42"}')"+ oldPolicyUid:$oldPolicyUid,newPolicyUid:$newPolicyUid,+ businessSignal:"real-host-and-probe:add(19,23)=42"}')"header "WRC/HCC Context assertions complete; restoring branch-owned runtime"diff --git a/scripts/tests/test-wrc-hcc-contracts-gate.sh b/scripts/tests/test-wrc-hcc-contracts-gate.shindex b01ccb524..9e3bdae3a 100755--- a/scripts/tests/test-wrc-hcc-contracts-gate.sh+++ b/scripts/tests/test-wrc-hcc-contracts-gate.sh@@ -70,11 +70,45 @@ if grep -Fq 'kind: WorkflowRecipe' "$RUNTIME_GATE" &&grep -Fq 'transport:' "$RUNTIME_GATE" &&grep -Fq 'context_projection_converged' "$RUNTIME_GATE" &&grep -Fq 'WORKLOAD_SELECTOR="clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=mock-tools"' "$RUNTIME_GATE" &&- grep -Fq 'mcp_workload_deployment_json' "$RUNTIME_GATE" &&+ grep -Fq 'mcp_workload_deployment_named_json' "$RUNTIME_GATE" &&+ ! grep -Fq 'kind: Context' "$RUNTIME_GATE" &&! grep -Fq 'kctl patch context' "$RUNTIME_GATE"; then- pass "WRC-HCC runtime gate follows the real WorkflowRecipe projection and its assigned workload identity"+ pass "WRC-HCC runtime gate makes WRC create the shared Context and follows its assigned workload identity"else- fail "WRC-HCC runtime gate bypasses WRC or invents the assigned workload identity"+ fail "WRC-HCC runtime gate pre-creates/bypasses Context or invents the assigned workload identity"+fi++if grep -Fq 'kind:"Host"' "$RUNTIME_GATE" &&+ grep -Fq 'SOURCE_HOST_NAME="${E2E_WRC_HCC_SOURCE_HOST:-chatllm}"' "$RUNTIME_GATE" &&+ grep -Fq 'second_workload_patch=' "$RUNTIME_GATE" &&+ grep -Fq 'expanded_context_projection_converged' "$RUNTIME_GATE" &&+ grep -Fq 'context_after_real_generation" -ne $((context_before_real_generation + 1))' "$RUNTIME_GATE" &&+ grep -Fq 'wait_for_positive_host_fanout 90 "$fanout_window_start" "$host_urgent_before"' "$RUNTIME_GATE" &&+ grep -Fq 'target_witnesses" = 1' "$RUNTIME_GATE" &&+ grep -Fq 'all_witnesses" = 1' "$RUNTIME_GATE" &&+ grep -Fq 'fleet" -gt 0' "$RUNTIME_GATE" &&+ grep -Fq 'failures" -gt 0' "$RUNTIME_GATE" &&+ grep -Fq 'context_netpol_failure_count' "$RUNTIME_GATE" &&+ grep -Fq 'probe_host_mcp_business_signal "$SECOND_MCP_NAME"' "$RUNTIME_GATE" &&+ grep -Fq 'real WRC Context change fanned out to a stable, Ready Host' "$RUNTIME_GATE"; then+ pass "WRC-HCC runtime gate witnesses a real WRC Context change in a correctly configured Host"+else+ fail "WRC-HCC runtime gate lacks a real Host, urgent completion, or positive business witness"+fi++if grep -Fq 'kctl annotate context "$CONTEXT_NAME"' "$RUNTIME_GATE" &&+ grep -Fq 'hcc_metadata_only_event_after "$metadata_events_before"' "$RUNTIME_GATE" &&+ grep -Fq 'kctl delete networkpolicy "$SECOND_RPC_EGRESS_POLICY"' "$RUNTIME_GATE" &&+ grep -Fq 'policy_recreated_from_snapshot "$RPC_PROXY_NS" "$SECOND_RPC_EGRESS_POLICY"' "$RUNTIME_GATE" &&+ grep -Fq 'target_host_activity_stays_absent "$metadata_window_start" 30' "$RUNTIME_GATE" &&+ grep -Fq 'host_reconcile_witness_count_since "$metadata_window_start" "$HOST_NAME"' "$RUNTIME_GATE" &&+ grep -Fq 'metadata_context_after_generation" = "$metadata_context_before_generation' "$RUNTIME_GATE" &&+ grep -Fq 'metadata-only NetworkPolicy convergence failed and made the Host skip inconclusive' "$RUNTIME_GATE" &&+ grep -Fq 'metadata-only Context MODIFIED interrupted the real Host-to-MCP path' "$RUNTIME_GATE" &&+ grep -Fq 'metadata-only Context MODIFIED kept the real Host stable' "$RUNTIME_GATE"; then+ pass "WRC-HCC runtime gate proves metadata-only MODIFIED runs scoped NP without Host fan-out"+else+ fail "WRC-HCC runtime gate can miss or misattribute the metadata-only Host skip boundary"fiif grep -Fq 'wrc_reconciled_after "$reconciles_before"' "$RUNTIME_GATE" &&@@ -88,23 +122,72 @@ fiif grep -Fq 'CONTEXT_MAPPER_NETPOL_RESYNC_SEC=${RESYNC_SECONDS}' "$RUNTIME_GATE" &&grep -Fq 'kctl delete networkpolicy "$CONTEXT_POLICY"' "$RUNTIME_GATE" &&grep -Fq 'probe_mcp_tcp_connectivity' "$RUNTIME_GATE" &&+ grep -Fq '[ "$connectivity_rc" = 42 ]' "$RUNTIME_GATE" &&+ grep -Fq 'probe_mcp_business_signal "$SECOND_MCP_NAME"' "$RUNTIME_GATE" &&grep -Fq 'mcp_runtime_still_ready' "$RUNTIME_GATE" &&+ grep -Fq 'PROBE_BASELINE_IDENTITY="$(selector_runtime_identity' "$RUNTIME_GATE" &&+ grep -Fq 'PRIMARY_MCP_BASELINE_IDENTITY="$(selector_runtime_identity' "$RUNTIME_GATE" &&+ grep -Fq 'SECOND_MCP_BASELINE_IDENTITY="$(selector_runtime_identity' "$RUNTIME_GATE" &&grep -Fq 'mcp_data_plane_is_policy_blocked' "$RUNTIME_GATE" &&- grep -Fq 'limits: {cpu: 50m, memory: 128Mi}' "$RUNTIME_GATE" &&+ grep -Fq 'timer_tick_due_epoch=' "$RUNTIME_GATE" &&+ grep -Fq 'timer_tick_due_epoch - timer_now_epoch)) -lt 30' "$RUNTIME_GATE" &&grep -Fq 'context_policy_recreated_from_snapshot' "$RUNTIME_GATE" &&+ grep -Fq 'hcc_netpol_certified_after "$netpol_certified_before_fault"' "$RUNTIME_GATE" &&+ grep -Fq 'netpol_desired_inventory_snapshot' "$RUNTIME_GATE" &&+ grep -Fq 'del(."clerum.io/network-ready")' "$RUNTIME_GATE" &&+ grep -Fq 'netpol_watch_recovery_count_since "$fault_window_start"' "$RUNTIME_GATE" &&+ grep -Fq 'more than one NetworkPolicy pass ran in the timer attribution window' "$RUNTIME_GATE" &&+ grep -Fq 'policy recovered before the armed periodic tick' "$RUNTIME_GATE" &&grep -Fq 'hcc_identity_is_stable' "$RUNTIME_GATE" &&grep -Fq 'restore_hcc_resync_config' "$RUNTIME_GATE"; then- pass "WRC-HCC runtime gate attributes real policy loss and proves in-process periodic recovery with exact restoration"+ pass "WRC-HCC runtime gate attributes policy loss to a post-fault timer pass with stable runtimes"else- fail "WRC-HCC runtime gate can pass without attributed policy loss, stable HCC identity, or config restoration"+ fail "WRC-HCC runtime gate can pass without timer attribution, stable identities, or config restoration"fiif grep -Fq 'name:"add",arguments:{a:19,b:23}' "$RUNTIME_GATE" &&- grep -Fq 'add result did not contain 42' "$RUNTIME_GATE" &&- grep -Fq 'businessSignal:"add(19,23)=42"' "$RUNTIME_GATE"; then- pass "WRC-HCC runtime gate carries a non-vacuous MCP business signal across the fault"+ grep -Fq 'content[0]?.text !== "42"' "$RUNTIME_GATE" &&+ grep -Fq 'probe_host_mcp_business_signal "$MCP_NAME"' "$RUNTIME_GATE" &&+ grep -Fq 'businessSignal:"real-host-and-probe:add(19,23)=42"' "$RUNTIME_GATE"; then+ pass "WRC-HCC runtime gate carries an exact MCP signal through the real Host and fault probe"+else+ fail "WRC-HCC runtime gate lacks an exact real-Host MCP business signal"+fi++real_change_line="$(grep -nF 'kctl patch workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_NS" --type=json' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+positive_host_line="$(grep -nF 'wait_for_positive_host_fanout 90 "$fanout_window_start" "$host_urgent_before"' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+metadata_event_line="$(grep -nF 'kctl annotate context "$CONTEXT_NAME"' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+metadata_guard_line="$(grep -nF 'target_host_activity_stays_absent "$metadata_window_start" 30' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+fault_line="$(grep -nF 'kctl delete networkpolicy "$CONTEXT_POLICY"' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+blocked_line="$(grep -nF 'mcp_data_plane_is_policy_blocked ||' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+recovery_line="$(grep -nF 'context_policy_recreated_from_snapshot ||' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+certified_line="$(grep -nF 'hcc_netpol_certified_after "$netpol_certified_before_fault"' "$RUNTIME_GATE" | tail -1 | cut -d: -f1)"+if [ -n "$real_change_line" ] && [ -n "$positive_host_line" ] &&+ [ -n "$metadata_event_line" ] && [ -n "$metadata_guard_line" ] &&+ [ -n "$fault_line" ] && [ -n "$blocked_line" ] &&+ [ -n "$recovery_line" ] && [ -n "$certified_line" ] &&+ [ "$real_change_line" -lt "$positive_host_line" ] &&+ [ "$positive_host_line" -lt "$metadata_event_line" ] &&+ [ "$metadata_event_line" -lt "$metadata_guard_line" ] &&+ [ "$metadata_guard_line" -lt "$fault_line" ] &&+ [ "$fault_line" -lt "$blocked_line" ] &&+ [ "$blocked_line" -lt "$recovery_line" ] &&+ [ "$recovery_line" -lt "$certified_line" ]; then+ pass "WRC-HCC runtime phases are ordered from real change through negative guard and timed recovery"+else+ fail "WRC-HCC runtime assertions can execute outside their causal phase ordering"+fi++if grep -Fq -- '--ignore-not-found -o name' "$RUNTIME_GATE" &&+ grep -Fq 'resource_collection_absent replicaset "$MCP_NS" "$WORKLOAD_SELECTOR"' "$RUNTIME_GATE" &&+ grep -Fq 'resource_collection_absent replicaset "$MCP_NS" "$SECOND_WORKLOAD_SELECTOR"' "$RUNTIME_GATE" &&+ grep -Fq 'resource_collection_absent pod "$HOST_NS" "app=${PROBE_NAME}"' "$RUNTIME_GATE" &&+ grep -Fq 'wrc_managed_resources_absent' "$RUNTIME_GATE" &&+ grep -Fq 'clerum.io/recipe=${RECIPE_NAME}' "$RUNTIME_GATE" &&+ grep -Fq 'deployment,replicaset,pod,service,serviceaccount' "$RUNTIME_GATE"; then+ pass "WRC-HCC runtime absence and cleanup checks fail closed across dependent workloads"else- fail "WRC-HCC runtime gate checks only policy presence and lacks a business signal"+ fail "WRC-HCC runtime cleanup can hide API errors or leave dependent Pods/ReplicaSets"fiif grep -Fq '.PHONY: test-e2e-wrc-hcc-context-noop-resync' "$MAKEFILE" &&