Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.6 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# GKE Smoke Test — Lightweight E2E for production clusters
# ═══════════════════════════════════════════════════════════════════════
#
# Validates all Clerum services are operational on GKE without deploying
# heavy workloads (no MongoDB, PostgreSQL, or WorkflowRecipes).
#
# Tests: health endpoints, cross-service connectivity, NetworkPolicy
# enforcement, CRD availability, and auth chain readiness.
#
# Usage:
#   ./scripts/e2e/e2e-gke-smoke.sh
#   ./scripts/e2e/e2e-gke-smoke.sh --verbose
# ═══════════════════════════════════════════════════════════════════════

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

VERBOSE="${1:-}"
PASS=0
FAIL=0
SKIP=0
TOTAL=0

log()   { echo -e "${CYAN}[smoke]${NC} $*"; }
pass()  { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e "  ${GREEN}PASS${NC} $*"; }
fail()  { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e "  ${RED}FAIL${NC} $*"; }
skip()  { SKIP=$((SKIP+1)); TOTAL=$((TOTAL+1)); echo -e "  ${YELLOW}SKIP${NC} $*"; }
detail(){ [[ "$VERBOSE" == "--verbose" ]] && echo -e "       $*"; }

# ── Helper: check health endpoint from within a pod ──────────────────
check_health() {
  local ns="$1" deploy="$2" target_url="$3" label="$4"
  local result
  result=$(kubectl exec -n "$ns" "deployment/$deploy" -- \
    wget -qO- --timeout=5 "$target_url" 2>&1) && {
    pass "$label"
    detail "$result"
    return 0
  } || {
    # Check if it's a connection issue vs HTTP error
    if echo "$result" | grep -qi "timed out\|ETIMEDOUT\|Network is unreachable"; then
      fail "$label (timeout — NetworkPolicy blocking?)"
    elif echo "$result" | grep -qi "Connection refused"; then
      fail "$label (connection refused — service down?)"
    elif echo "$result" | grep -qi "server returned error"; then
      pass "$label (reachable, HTTP error response)"
    else
      fail "$label: $result"
    fi
    return 1
  }
}

# ── Helper: require a successful HTTP response from within a pod ─────
check_http_ok() {
  local ns="$1" deploy="$2" target_url="$3" label="$4"
  local result
  result=$(kubectl exec -n "$ns" "deployment/$deploy" -- \
    wget -qO- --timeout=5 "$target_url" 2>&1) && {
    pass "$label"
    detail "$result"
    return 0
  } || {
    if echo "$result" | grep -qi "timed out\|ETIMEDOUT\|Network is unreachable"; then
      fail "$label (timeout — NetworkPolicy blocking?)"
    elif echo "$result" | grep -qi "Connection refused"; then
      fail "$label (connection refused — service down?)"
    elif echo "$result" | grep -qi "server returned error"; then
      fail "$label (backend reachable but not ready)"
    else
      fail "$label: $result"
    fi
    return 1
  }
}

# ── Helper: check pod readiness ──────────────────────────────────────
check_ready() {
  local ns="$1" deploy="$2"
  local ready
  ready=$(kubectl get deployment "$deploy" -n "$ns" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
  local desired
  desired=$(kubectl get deployment "$deploy" -n "$ns" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
  if [[ "$ready" == "$desired" && "$ready" != "0" ]]; then
    pass "$ns/$deploy: $ready/$desired ready"
    return 0
  else
    fail "$ns/$deploy: $ready/$desired ready"
    return 1
  fi
}

# ── Helper: check blocked connectivity (expect timeout) ──────────────
check_blocked() {
  local ns="$1" deploy="$2" target_url="$3" label="$4"
  kubectl exec -n "$ns" "deployment/$deploy" -- \
    wget -qO- --timeout=3 "$target_url" >/dev/null 2>&1 && {
    fail "$label (REACHABLE — should be blocked!)"
    return 1
  } || {
    pass "$label (blocked as expected)"
    return 0
  }
}

echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD}  Clerum GKE Smoke Test${NC}"
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 1: Pod Readiness
# ═════════════════════════════════════════════════════════════════════
log "Phase 1: Pod Readiness"

check_ready channels clerum-channel-reader
check_ready control-plane control-api
check_ready control-plane control-ui
check_ready control-plane host-context-controller
check_ready control-plane host-context-controller-api-gateway
check_ready control-plane workflow-recipes
check_ready mcp-host chatllm
check_ready mcp-server mcp-proxy
check_ready profiles external-rest-api
check_ready profiles profile-ui
check_ready rpc-proxy rpc-proxy
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 2: Health Endpoints (from within cluster)
# ═════════════════════════════════════════════════════════════════════
log "Phase 2: Health Endpoints (in-cluster)"

check_health control-plane control-api \
  "http://localhost:8090/health" \
  "control-api /health (localhost)"

check_health control-plane host-context-controller \
  "http://localhost:8081/health" \
  "host-context-controller /health (localhost)"

check_health mcp-host chatllm \
  "http://localhost:8080/v1/runtime/health" \
  "mcp-host /v1/runtime/health (localhost)"

check_health mcp-server mcp-proxy \
  "http://localhost:8083/health" \
  "mcp-proxy /health (localhost)"

check_health mcp-server mcp-proxy \
  "http://localhost:8083/ready" \
  "mcp-proxy /ready (localhost)"

check_health rpc-proxy rpc-proxy \
  "http://localhost:8094/health" \
  "rpc-proxy /health (localhost)"

check_health profiles external-rest-api \
  "http://localhost:8091/health" \
  "external-rest-api /health (localhost)"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 3: Cross-Service Connectivity (allowed paths)
# ═════════════════════════════════════════════════════════════════════
log "Phase 3: Cross-Service Connectivity (allowed paths)"

check_health control-plane control-api \
  "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/health" \
  "control-api → rpc-proxy:8094 (SSE path)"

check_health profiles external-rest-api \
  "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/health" \
  "external-rest-api → rpc-proxy:8094 (SSE path)"

check_health rpc-proxy rpc-proxy \
  "http://chatllm.mcp-host.svc.cluster.local:8080/v1/runtime/health" \
  "rpc-proxy → mcp-host:8080 (message relay)"

check_http_ok mcp-host chatllm \
  "http://host-context-controller-api-gateway.control-plane.svc.cluster.local:8081/ready" \
  "mcp-host → HCC API Gateway:8081 (context discovery ready)"

check_http_ok mcp-server mcp-proxy \
  "http://host-context-controller-api-gateway.control-plane.svc.cluster.local:8081/ready" \
  "mcp-proxy → HCC API Gateway:8081 (server polling ready)"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 4: NetworkPolicy Enforcement (blocked paths)
# ═════════════════════════════════════════════════════════════════════
log "Phase 4: NetworkPolicy Enforcement (blocked paths)"

# deny-all-mcp-host was added in v0.9.6 security hardening.
# chatllm should NOT be able to reach rpc-proxy or control-api directly.
check_blocked mcp-host chatllm \
  "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/health" \
  "mcp-host → rpc-proxy (reverse blocked by deny-all)"

check_blocked mcp-host chatllm \
  "http://control-api.control-plane.svc.cluster.local:8090/health" \
  "mcp-host → control-api (no direct access)"

check_blocked control-plane control-api \
  "http://chatllm.mcp-host.svc.cluster.local:8080/health" \
  "control-api → mcp-host (must go through rpc-proxy)"

check_blocked profiles external-rest-api \
  "http://chatllm.mcp-host.svc.cluster.local:8080/health" \
  "external-rest-api → mcp-host (must go through rpc-proxy)"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 5: CRD Availability
# ═════════════════════════════════════════════════════════════════════
log "Phase 5: CRD Availability"

for crd in communicationchannels.clerum.io contexts.clerum.io hosts.clerum.io mcpservers.clerum.io workflowrecipes.clerum.io workflowrecipepolicies.clerum.io; do
  if kubectl get crd "$crd" >/dev/null 2>&1; then
    pass "CRD $crd exists"
  else
    fail "CRD $crd missing"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 6: Image Version Verification
# ═════════════════════════════════════════════════════════════════════
log "Phase 6: Image Version Verification"

EXPECTED_TAG="${EXPECTED_TAG:-0.9.5}"
for pair in "channels/clerum-channel-reader" "control-plane/control-api" "control-plane/control-ui" "control-plane/host-context-controller" "control-plane/workflow-recipes" "mcp-host/chatllm" "mcp-server/mcp-proxy" "profiles/external-rest-api" "profiles/profile-ui" "rpc-proxy/rpc-proxy"; do
  ns="${pair%%/*}"
  dep="${pair##*/}"
  image=$(kubectl get deployment "$dep" -n "$ns" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)
  if echo "$image" | grep -q ":${EXPECTED_TAG}$"; then
    pass "$ns/$dep: $EXPECTED_TAG"
  else
    fail "$ns/$dep: expected $EXPECTED_TAG, got $(echo "$image" | grep -o ':[^:]*$')"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 7: RBAC Validation
# ═════════════════════════════════════════════════════════════════════
log "Phase 7: RBAC Validation"

for cr in control-api control-api-pod-reader clerum-channel-reader workflow-recipes-cluster-watch; do
  if kubectl get clusterrole "$cr" >/dev/null 2>&1; then
    pass "ClusterRole $cr exists"
  else
    fail "ClusterRole $cr missing"
  fi
done

for crb in control-api control-api-pod-reader clerum-channel-reader workflow-recipes-cluster-watch; do
  if kubectl get clusterrolebinding "$crb" >/dev/null 2>&1; then
    pass "ClusterRoleBinding $crb exists"
  else
    fail "ClusterRoleBinding $crb missing"
  fi
done

for pair in "control-plane/control-api" "mcp-host/mcp-host" "channels/clerum-channel-reader" "control-plane/workflow-recipes"; do
  ns="${pair%%/*}"
  sa="${pair##*/}"
  if kubectl get serviceaccount "$sa" -n "$ns" >/dev/null 2>&1; then
    pass "ServiceAccount $ns/$sa exists"
  else
    fail "ServiceAccount $ns/$sa missing"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 8: Secrets & ConfigMaps Presence
# ═════════════════════════════════════════════════════════════════════
log "Phase 8: Secrets & ConfigMaps Presence"

for pair in "control-plane/control-api-secrets" "rpc-proxy/rpc-proxy-secrets" "profiles/external-rest-api-secrets" "control-plane/clerum-wrc-signing-key"; do
  ns="${pair%%/*}"
  name="${pair##*/}"
  if kubectl get secret "$name" -n "$ns" >/dev/null 2>&1; then
    pass "Secret $ns/$name exists"
  else
    fail "Secret $ns/$name missing"
  fi
done

for pair in "mcp-host/mcp-host-config" "control-plane/control-api-config" "mcp-host/clerum-model-secret-mapping" "sandbox-recipes/clerum-wrc-public-key"; do
  ns="${pair%%/*}"
  name="${pair##*/}"
  if kubectl get configmap "$name" -n "$ns" >/dev/null 2>&1; then
    pass "ConfigMap $ns/$name exists"
  else
    fail "ConfigMap $ns/$name missing"
  fi
done

# Auth config validation
auth_enabled=$(kubectl get configmap mcp-host-config -n mcp-host -o jsonpath='{.data.CLERUM_ENABLE_AUTH}' 2>/dev/null)
if [[ "$auth_enabled" == "true" ]]; then
  pass "CLERUM_ENABLE_AUTH=true"
else
  fail "CLERUM_ENABLE_AUTH=${auth_enabled:-missing} (expected true)"
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 9: Pod Security Compliance
# ═════════════════════════════════════════════════════════════════════
log "Phase 9: Pod Security Compliance (spot checks)"

for pair in "channels/clerum-channel-reader" "control-plane/control-api" "control-plane/workflow-recipes" "mcp-server/mcp-proxy" "rpc-proxy/rpc-proxy" "profiles/external-rest-api"; do
  ns="${pair%%/*}"
  dep="${pair##*/}"
  non_root=$(kubectl get deployment "$dep" -n "$ns" -o jsonpath='{.spec.template.spec.containers[0].securityContext.runAsNonRoot}' 2>/dev/null)
  caps=$(kubectl get deployment "$dep" -n "$ns" -o jsonpath='{.spec.template.spec.containers[0].securityContext.capabilities.drop[0]}' 2>/dev/null)
  if [[ "$non_root" == "true" ]]; then
    pass "$ns/$dep: runAsNonRoot=true"
  else
    fail "$ns/$dep: runAsNonRoot=${non_root:-missing}"
  fi
  if [[ "$caps" == "ALL" ]]; then
    pass "$ns/$dep: capabilities.drop=[ALL]"
  else
    fail "$ns/$dep: capabilities.drop=${caps:-missing}"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 10: deny-all NetworkPolicies (all namespaces)
# ═════════════════════════════════════════════════════════════════════
log "Phase 10: deny-all NetworkPolicies"

for ns in channels profiles control-plane mcp-host mcp-server rpc-proxy sandbox-recipes; do
  nps=$(kubectl get networkpolicy -n "$ns" -o name 2>/dev/null || echo "")
  if echo "$nps" | grep -qi "deny-all"; then
    pass "$ns: deny-all NetworkPolicy present"
  else
    fail "$ns: deny-all NetworkPolicy MISSING"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# SUMMARY
# ═════════════════════════════════════════════════════════════════════
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
if [[ $FAIL -eq 0 ]]; then
  echo -e "${GREEN}${BOLD}  ALL PASSED: $PASS/$TOTAL tests passed ($SKIP skipped)${NC}"
else
  echo -e "${RED}${BOLD}  FAILURES: $FAIL/$TOTAL tests failed ($PASS passed, $SKIP skipped)${NC}"
fi
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"

exit $FAIL