Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
32.9 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E Shared Library — Common functions for composite WorkflowRecipe tests
# ═══════════════════════════════════════════════════════════════════════
#
# Source this from any E2E script:
#   SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
#   source "${SCRIPT_DIR}/e2e-lib.sh"

# ─── Colors ──────────────────────────────────────────────────────────
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

# ─── Counters ────────────────────────────────────────────────────────
e2e_pass=0
e2e_fail=0
e2e_total=0

# ─── Configurable defaults ───────────────────────────────────────────
RECIPE_NS="${RECIPE_NS:-mcp-server}"
WORKFLOW_RECIPE_NS="${WORKFLOW_RECIPE_NS:-sandbox-recipes}"
SANDBOX_NS="${SANDBOX_NS:-sandbox-recipes}"
MCP_HOST_NS="${MCP_HOST_NS:-mcp-host}"
CONTROL_NS="${CONTROL_NS:-control-plane}"
TIMEOUT_POD="${TIMEOUT_POD:-120}"
TIMEOUT_DELETE="${TIMEOUT_DELETE:-90}"
TIMEOUT_DISCOVERY="${TIMEOUT_DISCOVERY:-60}"
POLL_INTERVAL="${POLL_INTERVAL:-3}"
CONTEXT_NAME="${CONTEXT_NAME:-context1}"
E2E_KUBECONTEXT="${KUBECONTEXT:-${E2E_K8S_CONTEXT:-}}"
E2E_ALLOWED_CONTEXTS="${E2E_ALLOWED_CONTEXTS:-minikube,clerum-test}"
KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"

is_branch_scoped_e2e_context() {
  local context=$1
  case "$context" in
    *prod*|*production*) return 1 ;;
  esac
  printf "%s\n" "$context" | grep -Eq '^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$'
}

# A branch-scoped context is only valid with the port-forward map generated by
# scripts/minikube-profiles/branch.mk for that exact context. A
# non-empty URL is not sufficient: another profile can be healthy on the same
# machine and would otherwise make a Desktop run appear to validate the wrong
# cluster.
require_branch_profile_urls() {
  local context="$1"
  local ports_env="${2:-${HOME}/.cache/clerum/minikube-profiles/${context}/ports.env}"
  if ! is_branch_scoped_e2e_context "$context"; then
    return 0
  fi
  if [[ ! -f "$ports_env" ]]; then
    echo "Refusing E2E for branch context ${context}: profile ports.env is missing at ${ports_env}" >&2
    return 1
  fi

  profile_url() {
    local key="$1"
    awk -F= -v expected_key="$key" '$1 == expected_key { sub(/^[^=]*=/, ""); print; exit }' "$ports_env"
  }
  normalize_url() {
    local value="$1"
    printf '%s' "${value%/}"
  }

  local pair name expected_key actual expected
  for pair in \
    'CONTROL_UI_BASE_URL CONTROL_UI_URL' \
    'CONTROL_API_BASE_URL CONTROL_API_URL' \
    'EXTERNAL_REST_API_BASE_URL EXTERNAL_REST_API_URL' \
    'RPC_PROXY_BASE_URL RPC_PROXY_URL'; do
    name="${pair%% *}"
    expected_key="${pair#* }"
    actual="${!name:-}"
    expected="$(profile_url "$expected_key")"
    if [[ -z "$expected" ]]; then
      echo "Refusing E2E for branch context ${context}: ${expected_key} is missing from ${ports_env}" >&2
      return 1
    fi
    if [[ "$(normalize_url "$actual")" != "$(normalize_url "$expected")" ]]; then
      echo "Refusing E2E for branch context ${context}: ${name}=${actual} does not match ${expected_key} from ${ports_env}" >&2
      return 1
    fi
  done
}

is_allowed_e2e_context() {
  local context=$1
  case ",${E2E_ALLOWED_CONTEXTS}," in
    *",${context},"*) return 0 ;;
  esac
  is_branch_scoped_e2e_context "$context"
}

current_e2e_context() {
  if [ -n "$E2E_KUBECONTEXT" ]; then
    printf "%s\n" "$E2E_KUBECONTEXT"
  else
    "$KUBECTL_BIN" config current-context 2>/dev/null
  fi
}

kctl() {
  if [ -n "$E2E_KUBECONTEXT" ]; then
    "$KUBECTL_BIN" --context "$E2E_KUBECONTEXT" "$@"
  else
    "$KUBECTL_BIN" "$@"
  fi
}

truncate_rfc1123() {
  printf "%.63s" "$1" | sed -E 's/[^a-z0-9]+$//'
}

require_safe_kube_context() {
  local context
  context="$(current_e2e_context || true)"
  if [ -z "$context" ]; then
    echo "Refusing to run E2E cleanup: Kubernetes context is empty" >&2
    return 1
  fi
  is_allowed_e2e_context "$context" && return 0
  echo "Refusing to run E2E cleanup on context '${context}'." >&2
  echo "Allowed contexts: ${E2E_ALLOWED_CONTEXTS}" >&2
  echo "Generated branch-scoped contexts matching clerum-*-<8hex> are also allowed." >&2
  echo "Set KUBECONTEXT or E2E_ALLOWED_CONTEXTS explicitly if this target is intentional." >&2
  return 1
}

# Detect minikube context (Calico CNI has partial NP enforcement)
_is_minikube() {
  local context
  context="$(current_e2e_context || true)"
  printf "%s\n" "$context" | grep -qi "minikube\|clerum-test" || is_branch_scoped_e2e_context "$context"
}

# MCP Host deployment name: auto-detect (chatllm or mcp-host)
_MCP_HOST_DEPLOY=""
_detect_mcp_host_deploy() {
  if [ -n "$_MCP_HOST_DEPLOY" ]; then echo "$_MCP_HOST_DEPLOY"; return; fi
  for name in mcp-host chatllm; do
    if kctl get deploy "$name" -n "$MCP_HOST_NS" &>/dev/null; then
      _MCP_HOST_DEPLOY="$name"
      echo "$name"
      return
    fi
  done
  echo ""
}

# ─── Logging helpers ─────────────────────────────────────────────────
log()    { echo -e "${CYAN}[E2E]${NC} $*"; }
ok()     { e2e_pass=$((e2e_pass+1)); e2e_total=$((e2e_total+1)); echo -e "${GREEN}  ✅ PASS${NC} — $*"; }
fail()   { e2e_fail=$((e2e_fail+1)); e2e_total=$((e2e_total+1)); echo -e "${RED}  ❌ FAIL${NC} — $*"; }
warn()   { echo -e "${YELLOW}  ⚠️  WARN${NC} — $*"; }
header() { echo -e "\n${BOLD}═══ $* ═══${NC}"; }

# ─── Wait helpers ────────────────────────────────────────────────────

ready_pod_name() {
  local ns=$1 label=$2
  local rows name ready deleting
  rows=$(kctl get pods -n "$ns" -l "$label" --field-selector=status.phase=Running \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
    2>/dev/null || echo "")
  while IFS=$'\t' read -r name ready deleting; do
    if [ -n "$name" ] && [ "$ready" = "True" ] && [ -z "$deleting" ]; then
      printf "%s\n" "$name"
      return 0
    fi
  done <<< "$rows"
  return 1
}

# Count every Running pod, including unready or terminating pods. Suspend
# assertions use this instead of ready_pod_name so an unready pod cannot be
# mistaken for a fully terminated workload.
running_pod_count() {
  local ns=$1 label=$2 rows
  if ! rows="$(kctl get pods -n "$ns" -l "$label" \
    --field-selector=status.phase=Running -o name)"; then
    echo "running_pod_count: failed to list Running pods in ${ns} for selector ${label}" >&2
    return 1
  fi
  printf '%s\n' "$rows" | awk 'NF { count++ } END { print count + 0 }'
}

wait_for_pod() {
  local ns=$1 label=$2 timeout=$3 elapsed=0
  while [ $elapsed -lt "$timeout" ]; do
    ready_pod_name "$ns" "$label" >/dev/null && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

wait_for_statefulset() {
  local ns=$1 name=$2 timeout=$3 elapsed=0
  while [ $elapsed -lt "$timeout" ]; do
    local ready
    ready=$(kctl get statefulset "$name" -n "$ns" \
      -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
    [ "${ready:-0}" -ge 1 ] && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

wait_for_deployment() {
  local ns=$1 name=$2 timeout=$3 elapsed=0
  while [ $elapsed -lt "$timeout" ]; do
    local ready
    ready=$(kctl get deployment "$name" -n "$ns" \
      -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
    [ "${ready:-0}" -ge 1 ] && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

wait_for_cronjob() {
  local ns=$1 name=$2 timeout=$3 elapsed=0
  while [ $elapsed -lt "$timeout" ]; do
    if kctl get cronjob "$name" -n "$ns" &>/dev/null; then
      return 0
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

wait_for_workflowrecipe_deleted() {
  local ns=$1 name=$2 timeout=$3 elapsed=0
  while [ "$elapsed" -lt "$timeout" ]; do
    if ! kctl get workflowrecipe "$name" -n "$ns" &>/dev/null; then
      return 0
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

any_named_resource_exists() {
  local ns=$1 kind=$2
  shift 2
  local name output
  for name in "$@"; do
    if output=$(kctl get "$kind" "$name" -n "$ns" 2>&1 >/dev/null); then
      return 0
    fi
    if printf "%s" "$output" | grep -Eqi "notfound|not found"; then
      continue
    fi
    warn "could not check ${kind}/${name} in ${ns}: ${output}"
    return 2
  done
  return 1
}

any_sandbox_resource_exists() {
  local kind=$1
  shift
  any_named_resource_exists "$SANDBOX_NS" "$kind" "$@"
}

wait_for_named_resources_deleted() {
  local ns=$1 kind=$2 timeout=$3
  shift 3
  local elapsed=0 rc=0
  while [ "$elapsed" -lt "$timeout" ]; do
    if any_named_resource_exists "$ns" "$kind" "$@"; then
      rc=0
    else
      rc=$?
    fi
    if [ "$rc" -eq 1 ]; then
      return 0
    fi
    if [ "$rc" -eq 2 ]; then
      return 1
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

# ─── Phase 0: Prerequisites ─────────────────────────────────────────

check_prerequisites() {
  header "Phase 0 — Prerequisites"

  if kctl cluster-info &>/dev/null; then
    ok "Kubernetes cluster reachable"
  else
    fail "Cannot reach Kubernetes cluster"
    exit 1
  fi

  local seen_namespaces=""
  for ns in "$WORKFLOW_RECIPE_NS" "$RECIPE_NS" "$SANDBOX_NS" "$MCP_HOST_NS" "$CONTROL_NS"; do
    case " ${seen_namespaces} " in
      *" ${ns} "*) continue ;;
    esac
    seen_namespaces="${seen_namespaces} ${ns}"
    if kctl get ns "$ns" &>/dev/null; then ok "Namespace '$ns' exists"
    else fail "Namespace '$ns' not found"; fi
  done

  for crd in workflowrecipes.clerum.io mcpservers.clerum.io contexts.clerum.io hosts.clerum.io; do
    if kctl get crd "$crd" &>/dev/null; then ok "CRD '$crd' installed"
    else fail "CRD '$crd' not found"; fi
  done

  for deploy in host-context-controller host-context-controller-api-gateway workflow-recipes; do
    if kctl get deploy "$deploy" -n "$CONTROL_NS" &>/dev/null; then
      ok "Deployment '$deploy' in control-plane"
    else
      fail "Deployment '$deploy' not found"
    fi
  done

  # Call directly (NOT in subshell) so _MCP_HOST_DEPLOY persists for Phase 7-8
  _detect_mcp_host_deploy
  if [ -n "$_MCP_HOST_DEPLOY" ]; then
    ok "MCP Host deployment '${_MCP_HOST_DEPLOY}' exists"
  else
    fail "MCP Host not found (tried: mcp-host, chatllm)"
  fi

  if kctl get context "$CONTEXT_NAME" -n "$RECIPE_NS" &>/dev/null; then
    ok "Context '${CONTEXT_NAME}' exists"
  else
    fail "Context '${CONTEXT_NAME}' not found"
  fi
}

# ─── Phase 1: Cleanup ───────────────────────────────────────────────

cleanup_recipe() {
  local recipe_name=$1 backend_id=$2 mcp_id=$3
  local backend_type=${4:-statefulset}
  local cleanup_status=0
  local mcp_server_name="${recipe_name}-${mcp_id}"
  local mcp_ingress_np
  mcp_ingress_np="$(truncate_rfc1123 "${recipe_name}-wf-mcp-ingress-${mcp_server_name}")"

  header "Phase 1 — Clean Slate"
  log "Deleting WorkflowRecipe ${recipe_name} from ${WORKFLOW_RECIPE_NS}..."
  kctl delete workflowrecipe "$recipe_name" -n "$WORKFLOW_RECIPE_NS" \
    --ignore-not-found --wait=false 2>/dev/null || cleanup_status=1
  if ! wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$recipe_name" "$TIMEOUT_DELETE"; then
    fail "WorkflowRecipe '${recipe_name}' still exists in ${WORKFLOW_RECIPE_NS} after cleanup timeout"
    kctl get workflowrecipe "$recipe_name" -n "$WORKFLOW_RECIPE_NS" -o yaml 2>/dev/null || true
    cleanup_status=1
  fi
  if [ "$RECIPE_NS" != "$WORKFLOW_RECIPE_NS" ]; then
    kctl delete workflowrecipe "$recipe_name" -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || cleanup_status=1
  fi

  # Backend resources in sandbox
  kctl delete "$backend_type" "$backend_id" -n "$SANDBOX_NS" --ignore-not-found 2>/dev/null || cleanup_status=1
  kctl delete svc "$backend_id" "${backend_id}-headless" -n "$SANDBOX_NS" --ignore-not-found 2>/dev/null || cleanup_status=1

  # MCP resources
  kctl delete deployment "$mcp_id" "$mcp_server_name" -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || cleanup_status=1
  kctl delete svc "$mcp_id" "$mcp_server_name" -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || cleanup_status=1
  kctl delete mcpserver "$mcp_id" "$mcp_server_name" -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || cleanup_status=1

  # NetworkPolicies: keep cleanup scoped to exact policy names generated by WRC.
  kctl delete networkpolicy \
    "${recipe_name}-coord-to-mcp-host" \
    "${recipe_name}-coord-to-mcp-host-ingress" \
    "${recipe_name}-coord-to-wrc" \
    "${recipe_name}-wrc-to-mcp-host" \
    "${recipe_name}-mcp-host-to-servers" \
    "${recipe_name}-mcp-host-to-llm-api" \
    "${recipe_name}-mcp-host-to-approval-gateway" \
    -n "$SANDBOX_NS" --ignore-not-found 2>/dev/null || cleanup_status=1
  kctl delete networkpolicy \
    "$mcp_ingress_np" \
    "${recipe_name}-mcp-servers-egress-internet" \
    -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || cleanup_status=1

  log "Cleanup complete"
  return "$cleanup_status"
}

# ─── Phase 2: Apply ─────────────────────────────────────────────────

apply_recipe() {
  local recipe_file=$1 recipe_name=$2

  header "Phase 2 — Apply Composite WorkflowRecipe"
  log "Applying ${recipe_file}..."
  if kctl apply -f "$recipe_file" 2>&1; then
    ok "WorkflowRecipe '${recipe_name}' applied"
  else
    fail "Failed to apply WorkflowRecipe"
    exit 1
  fi

  if kctl get workflowrecipe "$recipe_name" -n "$WORKFLOW_RECIPE_NS" &>/dev/null; then
    ok "WorkflowRecipe '${recipe_name}' exists in ${WORKFLOW_RECIPE_NS}"
  else
    fail "WorkflowRecipe '${recipe_name}' not found in ${WORKFLOW_RECIPE_NS}"
    exit 1
  fi
}

# ─── Phase 3: Verify backend workload ───────────────────────────────

verify_backend_statefulset() {
  local backend_id=$1 backend_port=$2
  header "Phase 3 — Backend StatefulSet (sandbox-recipes)"

  log "Waiting for ${backend_id} StatefulSet (${TIMEOUT_POD}s)..."
  if wait_for_statefulset "$SANDBOX_NS" "$backend_id" "$TIMEOUT_POD"; then
    ok "${backend_id} StatefulSet ready"
  else
    fail "${backend_id} StatefulSet not ready (timeout)"
    kctl get statefulset -n "$SANDBOX_NS" 2>/dev/null || true
    kctl get pods -n "$SANDBOX_NS" -l "app=${backend_id}" 2>/dev/null || true
    kctl describe pod -n "$SANDBOX_NS" -l "app=${backend_id}" 2>/dev/null | tail -20 || true
  fi

  if kctl get svc "${backend_id}-headless" -n "$SANDBOX_NS" &>/dev/null || \
     kctl get svc "${backend_id}" -n "$SANDBOX_NS" &>/dev/null; then
    ok "${backend_id} Service exists"
  else
    fail "No ${backend_id} Service found"
  fi
}

verify_backend_deployment() {
  local backend_id=$1 backend_port=$2
  header "Phase 3 — Backend Deployment (sandbox-recipes)"

  log "Waiting for ${backend_id} Deployment (${TIMEOUT_POD}s)..."
  if wait_for_deployment "$SANDBOX_NS" "$backend_id" "$TIMEOUT_POD"; then
    ok "${backend_id} Deployment ready"
  else
    fail "${backend_id} Deployment not ready (timeout)"
    kctl get deployment -n "$SANDBOX_NS" 2>/dev/null || true
    kctl get pods -n "$SANDBOX_NS" -l "app=${backend_id}" 2>/dev/null || true
    kctl describe pod -n "$SANDBOX_NS" -l "app=${backend_id}" 2>/dev/null | tail -20 || true
  fi

  if kctl get svc "${backend_id}" -n "$SANDBOX_NS" &>/dev/null; then
    ok "${backend_id} Service exists"
  else
    fail "No ${backend_id} Service found"
  fi
}

# ─── Phase 4: Verify MCP Delegation ─────────────────────────────────

verify_mcp_context_allowlist() {
  local mcp_server_name=$1
  local timeout=${2:-${TIMEOUT_CONTEXT:-180}}
  local context_ref context_servers elapsed=0

  context_ref=$(kctl get mcpserver "$mcp_server_name" -n "$RECIPE_NS" \
    -o jsonpath='{.spec.contextRef}' 2>/dev/null || echo "")
  if [ -z "$context_ref" ]; then
    fail "McpServer '${mcp_server_name}' has no spec.contextRef"
    return
  fi

  while [ "$elapsed" -lt "$timeout" ]; do
    context_servers=$(kctl get context "$context_ref" -n "$RECIPE_NS" \
      -o jsonpath='{.spec.mcpServers}' 2>/dev/null || echo "")
    if printf "%s" "$context_servers" | tr '[],"' '    ' | tr ' ' '\n' | grep -Fxq "$mcp_server_name"; then
      ok "Context '${context_ref}' allowlist includes '${mcp_server_name}'"
      return
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done

  fail "Context '${context_ref}' does not include '${mcp_server_name}' after ${timeout}s"
}

verify_mcp_delegation() {
  local recipe_name=$1 mcp_id=$2 binding_port=${3:-}
  local mcp_server_name="${recipe_name}-${mcp_id}"

  header "Phase 4 — MCP Delegation (McpServer CRD + Service)"

  if kctl get mcpserver "$mcp_server_name" -n "$RECIPE_NS" &>/dev/null; then
    ok "McpServer CRD '${mcp_server_name}' auto-created"
  else
    fail "McpServer CRD '${mcp_server_name}' not found"
  fi

  local managed
  managed=$(kctl get mcpserver "$mcp_server_name" -n "$RECIPE_NS" \
    -o jsonpath='{.spec.managed}' 2>/dev/null || echo "")
  if [ "$managed" = "false" ]; then
    ok "McpServer managed=false (WRC manages deployment)"
  else
    fail "McpServer managed='$managed' (expected: false)"
  fi

  # Check recipe-bindings annotation if port provided
  if [ -n "$binding_port" ]; then
    local bindings_ann
    bindings_ann=$(kctl get mcpserver "$mcp_server_name" -n "$RECIPE_NS" \
      -o jsonpath='{.metadata.annotations.clerum\.io/recipe-bindings}' 2>/dev/null || echo "")
    if echo "$bindings_ann" | grep -q "\"port\":${binding_port}"; then
      ok "McpServer has recipe-bindings (port ${binding_port})"
    else
      fail "McpServer recipe-bindings annotation incomplete"
    fi
  fi

  if kctl get svc "$mcp_server_name" -n "$RECIPE_NS" &>/dev/null; then
    ok "Transport Service '${mcp_server_name}' created"
  else
    fail "Transport Service not found"
  fi

  verify_mcp_context_allowlist "$mcp_server_name"
}

# ─── Phase 5: Verify MCP Server Pod ─────────────────────────────────

verify_mcp_server_pod() {
  local mcp_id=$1
  header "Phase 5 — MCP Server Pod (mcp-server namespace)"

  log "Waiting for MCP Server pod (${TIMEOUT_POD}s)..."
  if wait_for_pod "$RECIPE_NS" "app=${mcp_id}" "$TIMEOUT_POD"; then
    ok "MCP Server pod '${mcp_id}' ready"
  else
    fail "MCP Server pod not ready (timeout)"
    kctl get pods -n "$RECIPE_NS" -l "app=${mcp_id}" 2>/dev/null || true
    kctl describe pod -n "$RECIPE_NS" -l "app=${mcp_id}" 2>/dev/null | tail -20 || true
  fi

  local mcp_logs
  mcp_logs=$(kctl logs -n "$RECIPE_NS" -l "app=${mcp_id}" --tail=30 2>/dev/null || echo "")
  if echo "$mcp_logs" | grep -qi "listening\|started\|transport\|StreamableHTTP"; then
    ok "MCP Server transport started"
  else
    fail "MCP Server transport not detected in logs"
  fi
}

# ─── Phase 5b: Verify stdio MCP Server Pod (sidecar pattern) ─────────

verify_stdio_mcp_server_pod() {
  local mcp_id=$1
  local mcp_server_name=$2
  header "Phase 5 — stdio MCP Server Pod (stdio-bridge sidecar)"

  log "Waiting for stdio MCP Server pod '${mcp_server_name}' (${TIMEOUT_POD}s)..."
  if wait_for_pod "$RECIPE_NS" "app=${mcp_server_name}" "$TIMEOUT_POD"; then
    ok "stdio MCP Server pod '${mcp_server_name}' ready"
  else
    fail "stdio MCP Server pod not ready (timeout)"
    kctl get pods -n "$RECIPE_NS" -l "app=${mcp_server_name}" 2>/dev/null || true
    kctl describe pod -n "$RECIPE_NS" -l "app=${mcp_server_name}" 2>/dev/null | tail -30 || true
    return
  fi

  # Verify pod has stdio-bridge container (injected by HCC)
  local containers
  containers=$(kctl get pods -n "$RECIPE_NS" -l "app=${mcp_server_name}" \
    -o jsonpath='{.items[0].spec.containers[*].name}' 2>/dev/null || echo "")
  if echo "$containers" | grep -q "stdio-bridge"; then
    ok "Pod has 'stdio-bridge' sidecar container"
  else
    fail "Pod missing 'stdio-bridge' container (found: ${containers})"
  fi

  # Verify init container exists (copies binary)
  local init_containers
  init_containers=$(kctl get pods -n "$RECIPE_NS" -l "app=${mcp_server_name}" \
    -o jsonpath='{.items[0].spec.initContainers[*].name}' 2>/dev/null || echo "")
  if [ -n "$init_containers" ]; then
    ok "Pod has init container for binary copy"
  else
    warn "No init container found (may use different binary delivery method)"
  fi

  # Check stdio-bridge logs for startup
  local bridge_logs
  bridge_logs=$(kctl logs -n "$RECIPE_NS" -l "app=${mcp_server_name}" \
    -c stdio-bridge --tail=20 2>/dev/null || echo "")
  if echo "$bridge_logs" | grep -qi "started\|listening\|bridge\|ready\|health"; then
    ok "stdio-bridge sidecar started successfully"
  else
    warn "Could not confirm stdio-bridge startup from logs"
  fi
}

# ─── Phase 6: NetworkPolicy Enforcement ──────────────────────────────

verify_networkpolicy() {
  local mcp_id=$1 backend_id=$2 backend_port=$3
  header "Phase 6 — NetworkPolicy Enforcement"

  # Test deny-all: generic pod → backend
  log "Testing deny-all enforcement..."
  kctl delete pod np-test-deny -n "$RECIPE_NS" --ignore-not-found 2>/dev/null || true
  if kctl run np-test-deny --image=busybox --rm -i --restart=Never -n "$RECIPE_NS" \
     --timeout=10s -- nc -z -w3 "${backend_id}.${SANDBOX_NS}.svc.cluster.local" "$backend_port" &>/dev/null 2>&1; then
    if _is_minikube; then
      warn "Deny-all NOT enforced (expected on minikube/Calico — ${backend_id} reachable)"
    else
      fail "Deny-all NOT enforced: generic pod reached ${backend_id}"
    fi
  else
    ok "Deny-all enforced: generic pod cannot reach ${backend_id}"
  fi

  # Test MCP server → backend connectivity
  log "Testing MCP server → ${backend_id}:${backend_port}..."
  local mcp_pod
  mcp_pod=$(ready_pod_name "$RECIPE_NS" "app=${mcp_id}" || echo "")
  if [ -n "$mcp_pod" ]; then
    local connect_result
    local connected=false
    local attempt
    for attempt in 1 2 3; do
      mcp_pod=$(ready_pod_name "$RECIPE_NS" "app=${mcp_id}" || echo "")
      if [ -z "$mcp_pod" ]; then
        connect_result="no ready pod"
        sleep "$POLL_INTERVAL"
        continue
      fi
      connect_result=$(kctl exec "$mcp_pod" -n "$RECIPE_NS" -- node -e "
        const net = require('net');
        const s = new net.Socket();
        s.setTimeout(5000);
        s.connect(${backend_port}, '${backend_id}.${SANDBOX_NS}.svc.cluster.local', () => {
          console.log('CONNECTED'); s.destroy();
        });
        s.on('error', e => { console.log('ERROR:' + e.message); s.destroy(); });
        s.on('timeout', () => { console.log('TIMEOUT'); s.destroy(); });
      " 2>&1 || echo "ERROR")
      if echo "$connect_result" | grep -q "CONNECTED"; then
        connected=true
        break
      fi
      if echo "$connect_result" | grep -Eqi 'container not found|unable to upgrade connection|not found'; then
        sleep "$POLL_INTERVAL"
        continue
      fi
      break
    done
    if [ "$connected" = "true" ]; then
      ok "MCP server → ${backend_id}:${backend_port} (binding NP works)"
    else
      if [ "${E2E_STRICT_BACKEND_CONNECTIVITY:-false}" = "true" ]; then
        fail "MCP server cannot reach ${backend_id} (${connect_result})"
      else
        warn "MCP server cannot reach ${backend_id} (${connect_result})"
      fi
    fi
  else
    if [ "${E2E_STRICT_BACKEND_CONNECTIVITY:-false}" = "true" ]; then
      fail "Cannot test backend connectivity: MCP server pod not found"
    else
      warn "Cannot test: MCP server pod not found"
    fi
  fi

  # Test internet egress blocked
  log "Testing internet egress blocked..."
  if [ -n "$mcp_pod" ]; then
    local egress_result
    egress_result=$(kctl exec "$mcp_pod" -n "$RECIPE_NS" -- node -e "
      const net = require('net');
      const s = new net.Socket();
      s.setTimeout(3000);
      s.connect(80, '1.1.1.1', () => { console.log('OPEN'); s.destroy(); });
      s.on('error', e => { console.log('BLOCKED'); s.destroy(); });
      s.on('timeout', () => { console.log('BLOCKED'); s.destroy(); });
    " 2>&1 || echo "BLOCKED")
    if echo "$egress_result" | grep -q "BLOCKED"; then
      ok "Internet egress blocked"
    else
      if _is_minikube; then
        warn "Internet egress NOT blocked (expected on minikube/Calico)"
      else
        fail "Internet egress NOT blocked"
      fi
    fi
  fi
}

# ─── Phase 7: MCP Host Discovery ────────────────────────────────────

verify_mcp_host_discovery() {
  local recipe_name=$1 mcp_id=$2
  local mcp_server_name="${recipe_name}-${mcp_id}"
  header "Phase 7 — MCP Host Discovery"

  log "Waiting for MCP Host to discover '${mcp_server_name}' (${TIMEOUT_DISCOVERY}s)..."
  local elapsed=0 discovered=false
  while [ $elapsed -lt "$TIMEOUT_DISCOVERY" ]; do
    local host_logs
    host_logs=$(kctl logs -n "$MCP_HOST_NS" "deploy/${_MCP_HOST_DEPLOY}" --tail=80 2>/dev/null || echo "")
    if echo "$host_logs" | grep -q "MCP server added: ${mcp_server_name}\|Connected successfully.*${mcp_server_name}\|Added server: ${mcp_server_name}"; then
      discovered=true
      break
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done

  if [ "$discovered" = true ]; then
    ok "MCP Host discovered '${mcp_server_name}'"
  else
    fail "MCP Host did not discover '${mcp_server_name}'"
  fi

  local tool_count
  tool_count=$(kctl logs -n "$MCP_HOST_NS" "deploy/${_MCP_HOST_DEPLOY}" --tail=200 2>/dev/null \
    | grep -c "${mcp_server_name}__" 2>/dev/null || true)
  tool_count=${tool_count:-0}
  if [ "$tool_count" -gt 0 ]; then
    ok "MCP Host registered ${tool_count} tools from ${mcp_server_name}"
  else
    fail "MCP Host registered no tools from ${mcp_server_name}"
  fi
}

# ─── Phase 7b: mcp-proxy tools/list + tools/call ───────────────────

verify_mcp_proxy_tool_call() {
  local recipe_name=$1 mcp_id=$2 tool_name=$3 tool_args_json=$4 expected_output=$5
  local mcp_server_name="${recipe_name}-${mcp_id}"
  header "Phase 7 — mcp-proxy Tool Contract"

  if ! kctl get deploy mcp-proxy -n "$RECIPE_NS" &>/dev/null; then
    fail "mcp-proxy deployment not found"
    return
  fi

  log "Verifying ${mcp_server_name}.${tool_name} through mcp-proxy..."
  local result="" ok_value="" tools_value="" output_value="" stage_value="" status_value=""
  local elapsed=0 timeout="${MCP_PROXY_ROUTING_TIMEOUT:-$TIMEOUT_DISCOVERY}"

  while [ "$elapsed" -lt "$timeout" ]; do
    result=$(kctl exec -i -n "$RECIPE_NS" deploy/mcp-proxy \
      -- node - "$mcp_server_name" "$tool_name" "$tool_args_json" "$expected_output" <<'NODE' 2>/dev/null || echo '{"ok":false,"stage":"exec","error":"kctl exec failed"}'
const http = require('http');
const serverName = process.argv[2];
const toolName = process.argv[3];
const argsJson = process.argv[4];
const expectedOutput = process.argv[5];

function parseBody(text) {
  try {
    return JSON.parse(text);
  } catch {
    const lines = text.split('\n');
    for (let i = lines.length - 1; i >= 0; i--) {
      if (lines[i].startsWith('data: ')) {
        return JSON.parse(lines[i].slice(6));
      }
    }
    return { raw: text };
  }
}

function postMcp(id, method, params, sessionId) {
  const body = JSON.stringify({ jsonrpc: '2.0', id, method, params });
  const headers = {
    'Content-Type': 'application/json',
    'Content-Length': Buffer.byteLength(body),
    Accept: 'application/json, text/event-stream',
  };
  if (sessionId) headers['mcp-session-id'] = sessionId;

  return new Promise(resolve => {
    const req = http.request({
      hostname: 'localhost',
      port: 8083,
      path: `/servers/${serverName}/mcp`,
      method: 'POST',
      headers,
    }, res => {
      let text = '';
      res.on('data', chunk => text += chunk);
      res.on('end', () => resolve({
        status: res.statusCode,
        sessionId: res.headers['mcp-session-id'],
        body: text,
        parsed: parseBody(text),
      }));
    });
    req.on('error', error => resolve({ status: 0, error: error.message, body: '' }));
    req.write(body);
    req.end();
  });
}

(async () => {
  let toolArgs;
  try {
    toolArgs = JSON.parse(argsJson);
  } catch {
    console.log(JSON.stringify({ ok: false, stage: 'args', status: 0, body: argsJson }));
    return;
  }

  const init = await postMcp(1, 'initialize', {
    protocolVersion: '2025-03-26',
    capabilities: {},
    clientInfo: { name: 'e2e-workflow-backend-compat', version: '1.0.0' },
  });
  if (init.status !== 200 && init.status !== 201) {
    console.log(JSON.stringify({ ok: false, stage: 'initialize', status: init.status, body: init.body }));
    return;
  }
  const sessionId = init.sessionId || null;

  const listed = await postMcp(2, 'tools/list', {}, sessionId);
  if (listed.status !== 200) {
    console.log(JSON.stringify({ ok: false, stage: 'tools/list', status: listed.status, body: listed.body }));
    return;
  }
  const tools = listed.parsed?.result?.tools || [];
  const names = tools.map(tool => tool.name);
  if (!names.includes(toolName)) {
    console.log(JSON.stringify({ ok: false, stage: 'tools/list', status: listed.status, tools: names }));
    return;
  }

  const called = await postMcp(3, 'tools/call', {
    name: toolName,
    arguments: toolArgs,
  }, sessionId);
  if (called.status !== 200) {
    console.log(JSON.stringify({ ok: false, stage: 'tools/call', status: called.status, body: called.body }));
    return;
  }
  const text = (called.parsed?.result?.content || [])
    .filter(item => item.type === 'text')
    .map(item => item.text)
    .join('\n');
  if (!text.includes(expectedOutput)) {
    console.log(JSON.stringify({ ok: false, stage: 'tools/call', status: called.status, output: text }));
    return;
  }

  console.log(JSON.stringify({ ok: true, tools: names, output: text }));
})();
NODE
)

    ok_value=$(printf "%s" "$result" | python3 -c "import json,sys; print(json.load(sys.stdin).get('ok',''))" 2>/dev/null || echo "")
    tools_value=$(printf "%s" "$result" | python3 -c "import json,sys; print(','.join(json.load(sys.stdin).get('tools',[])))" 2>/dev/null || echo "")
    output_value=$(printf "%s" "$result" | python3 -c "import json,sys; print(json.load(sys.stdin).get('output',''))" 2>/dev/null || echo "")
    stage_value=$(printf "%s" "$result" | python3 -c "import json,sys; print(json.load(sys.stdin).get('stage',''))" 2>/dev/null || echo "")
    status_value=$(printf "%s" "$result" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status',''))" 2>/dev/null || echo "")

    if [ "$ok_value" = "True" ]; then
      ok "mcp-proxy listed ${tool_name} and tools/call returned expected output"
      log "tools: ${tools_value}; output chars=${#output_value}; expected signal matched"
      return
    fi

    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done

  fail "mcp-proxy tool contract failed at ${stage_value:-unknown} HTTP ${status_value:-unknown}: $(printf "%s" "$result" | head -c 260)"
}

# ─── Print Results ───────────────────────────────────────────────────

print_results() {
  header "Results"
  echo ""
  echo -e "${BOLD}Total: ${e2e_total}  |  ${GREEN}Pass: ${e2e_pass}${NC}  |  ${RED}Fail: ${e2e_fail}${NC}"
  echo ""
  # A run that executed ZERO tests is a failure, not a pass. "Nothing failed"
  # and "everything passed" are different claims, and only the second one earns
  # a green gate. Without this guard a fixture that dies before the first
  # assertion — a missing namespace, an image that never pulled, a filter that
  # matched nothing — prints "All tests passed!" and exits 0 over an empty
  # result set. Every e2e gate in this repo shares this function, so the
  # fail-open was underwriting all of them at once.
  if [ "${e2e_total}" -eq 0 ]; then
    echo -e "${RED}${BOLD}FAIL: zero tests executed — the harness never reached a real assertion${NC}" >&2
    return 1
  fi
  if [ "$e2e_fail" -eq 0 ]; then
    echo -e "${GREEN}${BOLD}All tests passed!${NC}"
    return 0
  else
    echo -e "${RED}${BOLD}${e2e_fail} test(s) failed${NC}"
    return 1
  fi
}