Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.8 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# GKE Auth Chain E2E — JWT auth chain validation for production clusters
# ═══════════════════════════════════════════════════════════════════════
#
# Validates the full JWT authentication chain:
#   control-api (signs JWT) → Desktop App → rpc-proxy (validates) → mcp-host (validates)
#
# Token invariants:
#   iss = control-api
#   aud = rpc-proxy
#   TTL = 300s
#   Algorithm = RSA (public key in PEM format)
#
# Tests: ConfigMap/Secret auth fields, key sync between services,
# unauthenticated rejection, invalid token rejection, service reachability.
#
# Usage:
#   ./scripts/e2e/e2e-gke-auth-chain.sh
#   ./scripts/e2e/e2e-gke-auth-chain.sh --verbose
# ═══════════════════════════════════════════════════════════════════════

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

VERBOSE="${1:-}"
PASS=0
FAIL=0
SKIP=0
TOTAL=0

log()   { echo -e "${CYAN}[auth-chain]${NC} $*"; }
pass()  { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e "  ${GREEN}PASS${NC} $*"; }
fail()  { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e "  ${RED}FAIL${NC} $*"; }
skip()  { SKIP=$((SKIP+1)); TOTAL=$((TOTAL+1)); echo -e "  ${YELLOW}SKIP${NC} $*"; }
detail(){ [[ "$VERBOSE" == "--verbose" ]] && echo -e "       $*" || true; }

echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD}  Clerum GKE Auth Chain E2E${NC}"
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo ""

# ── Prerequisite: cluster connectivity ─────────────────────────────────
log "Prerequisite: Cluster connectivity"

if kubectl cluster-info >/dev/null 2>&1; then
  pass "Cluster connected"
else
  fail "Cluster not connected"
  exit 1
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 1: Auth Config Presence (mcp-host-config ConfigMap)
# ═════════════════════════════════════════════════════════════════════
log "Phase 1: Auth Config Presence (mcp-host-config)"

# 1a. CLERUM_ENABLE_AUTH is declared (accepts "true" or "false" — dev/prod
#     overlays intentionally set this to "false"; only minikube enables auth).
#     The auth chain must still be wired (issuer/audience/public key) so the
#     feature can be flipped on without additional config work.
enable_auth=$(kubectl get configmap mcp-host-config -n mcp-host \
  -o jsonpath='{.data.CLERUM_ENABLE_AUTH}' 2>/dev/null || echo "")
if [ "$enable_auth" = "true" ] || [ "$enable_auth" = "false" ]; then
  pass "CLERUM_ENABLE_AUTH declared: '$enable_auth'"
else
  fail "CLERUM_ENABLE_AUTH unset or invalid (got '$enable_auth', expected 'true' or 'false')"
fi

# 1b. CLERUM_AUTH_JWT_ISSUER=control-api
jwt_issuer=$(kubectl get configmap mcp-host-config -n mcp-host \
  -o jsonpath='{.data.CLERUM_AUTH_JWT_ISSUER}' 2>/dev/null || echo "")
if [ "$jwt_issuer" = "control-api" ]; then
  pass "CLERUM_AUTH_JWT_ISSUER=control-api"
else
  fail "CLERUM_AUTH_JWT_ISSUER expected 'control-api', got '$jwt_issuer'"
fi

# 1c. CLERUM_AUTH_JWT_AUDIENCE=rpc-proxy
jwt_audience=$(kubectl get configmap mcp-host-config -n mcp-host \
  -o jsonpath='{.data.CLERUM_AUTH_JWT_AUDIENCE}' 2>/dev/null || echo "")
if [ "$jwt_audience" = "rpc-proxy" ]; then
  pass "CLERUM_AUTH_JWT_AUDIENCE=rpc-proxy"
else
  fail "CLERUM_AUTH_JWT_AUDIENCE expected 'rpc-proxy', got '$jwt_audience'"
fi

# 1d. CLERUM_AUTH_JWT_PUBLIC_KEY is non-empty and starts with PEM header
jwt_public_key=$(kubectl get configmap mcp-host-config -n mcp-host \
  -o jsonpath='{.data.CLERUM_AUTH_JWT_PUBLIC_KEY}' 2>/dev/null || echo "")
if [ -z "$jwt_public_key" ]; then
  fail "CLERUM_AUTH_JWT_PUBLIC_KEY is empty"
elif echo "$jwt_public_key" | head -1 | grep -q "^-----BEGIN PUBLIC KEY-----"; then
  pass "CLERUM_AUTH_JWT_PUBLIC_KEY is non-empty and starts with -----BEGIN PUBLIC KEY-----"
  detail "Key length: ${#jwt_public_key} chars"
else
  fail "CLERUM_AUTH_JWT_PUBLIC_KEY does not start with '-----BEGIN PUBLIC KEY-----'"
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 2: rpc-proxy Secrets (RPC_PROXY_JWT_PUBLIC_KEY)
# ═════════════════════════════════════════════════════════════════════
# Canonical Secret key is RPC_PROXY_JWT_PUBLIC_KEY (see rpc-proxy/src/config.ts).
# The legacy name JWT_PUBLIC_KEY still exists in some minikube fixtures; we
# prefer the canonical name but fall back for backwards compatibility.
log "Phase 2: rpc-proxy Secrets"

# 2a. rpc-proxy-secrets exists and has the JWT public key
rpc_proxy_key_b64=$(kubectl get secret rpc-proxy-secrets -n rpc-proxy \
  -o jsonpath='{.data.RPC_PROXY_JWT_PUBLIC_KEY}' 2>/dev/null || echo "")
rpc_key_field="RPC_PROXY_JWT_PUBLIC_KEY"
if [ -z "$rpc_proxy_key_b64" ]; then
  rpc_proxy_key_b64=$(kubectl get secret rpc-proxy-secrets -n rpc-proxy \
    -o jsonpath='{.data.JWT_PUBLIC_KEY}' 2>/dev/null || echo "")
  rpc_key_field="JWT_PUBLIC_KEY (legacy)"
fi
if [ -z "$rpc_proxy_key_b64" ]; then
  fail "rpc-proxy-secrets JWT public key is missing (checked RPC_PROXY_JWT_PUBLIC_KEY and JWT_PUBLIC_KEY)"
  rpc_proxy_key=""
else
  rpc_proxy_key=$(echo "$rpc_proxy_key_b64" | base64 -d 2>/dev/null || echo "")
  if echo "$rpc_proxy_key" | head -1 | grep -q "^-----BEGIN PUBLIC KEY-----"; then
    pass "rpc-proxy-secrets.$rpc_key_field is a valid PEM public key"
    detail "Key length: ${#rpc_proxy_key} chars"
  else
    fail "rpc-proxy-secrets.$rpc_key_field is not a valid PEM key"
  fi
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 3: Key Sync (mcp-host-config == rpc-proxy-secrets)
# ═════════════════════════════════════════════════════════════════════
log "Phase 3: Key Sync Validation"

if [ -z "$jwt_public_key" ] || [ -z "$rpc_proxy_key" ]; then
  skip "Key sync check (one or both keys are missing)"
else
  # Normalize whitespace for comparison (trim trailing newlines/spaces)
  normalized_cm_key=$(echo "$jwt_public_key" | sed 's/[[:space:]]*$//' | sed '/^$/d')
  normalized_rpc_key=$(echo "$rpc_proxy_key" | sed 's/[[:space:]]*$//' | sed '/^$/d')

  if [ "$normalized_cm_key" = "$normalized_rpc_key" ]; then
    pass "Public key in mcp-host-config matches rpc-proxy-secrets"
  else
    fail "Public key MISMATCH between mcp-host-config and rpc-proxy-secrets"
    detail "This causes 401 errors. Run: make gcp-sync-auth-key"
  fi
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 4: rpc-proxy Rejects Unauthenticated Requests
# ═════════════════════════════════════════════════════════════════════
log "Phase 4: Unauthenticated Request Rejection"

# Find a pod to exec into for in-cluster curl. Use control-api since it
# has wget/curl and is always present.
EXEC_POD=""
EXEC_NS=""
for pair in "control-plane/control-api" "rpc-proxy/rpc-proxy" "mcp-host/chatllm"; do
  ns="${pair%%/*}"
  dep="${pair##*/}"
  if kubectl get deployment "$dep" -n "$ns" >/dev/null 2>&1; then
    EXEC_POD="deployment/$dep"
    EXEC_NS="$ns"
    break
  fi
done

if [ -z "$EXEC_POD" ]; then
  skip "No exec-capable pod found for in-cluster HTTP tests"
  skip "rpc-proxy unauthenticated rejection (no exec pod)"
  skip "rpc-proxy invalid token rejection (no exec pod)"
else
  detail "Using $EXEC_NS/$EXEC_POD for in-cluster HTTP tests"

  # 4a. No token — expect 401
  no_auth_code=$(kubectl exec -n "$EXEC_NS" "$EXEC_POD" -- \
    wget -qO- --timeout=5 -S \
    "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/api/v1/rpc/hosts/chatllm/activity/stream" \
    2>&1 | grep -o "HTTP/[0-9.]* [0-9]*" | tail -1 | grep -o "[0-9]*$" || echo "")

  # wget returns non-zero for 4xx, so also check stderr for status code
  if [ -z "$no_auth_code" ]; then
    no_auth_result=$(kubectl exec -n "$EXEC_NS" "$EXEC_POD" -- \
      wget -qO- --timeout=5 -S \
      "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/api/v1/rpc/hosts/chatllm/activity/stream" \
      2>&1 || true)
    if echo "$no_auth_result" | grep -q "401\|Unauthorized"; then
      pass "rpc-proxy rejects unauthenticated requests (401)"
    elif echo "$no_auth_result" | grep -qi "timed out\|ETIMEDOUT"; then
      fail "rpc-proxy unreachable (timeout — NetworkPolicy blocking?)"
    else
      fail "rpc-proxy did not return 401 for unauthenticated request"
      detail "Response: $(echo "$no_auth_result" | head -3)"
    fi
  elif [ "$no_auth_code" = "401" ]; then
    pass "rpc-proxy rejects unauthenticated requests (401)"
  else
    fail "rpc-proxy returned HTTP $no_auth_code instead of 401 for unauthenticated request"
  fi
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 5: rpc-proxy Rejects Invalid Token
# ═════════════════════════════════════════════════════════════════════
log "Phase 5: Invalid Token Rejection"

if [ -z "$EXEC_POD" ]; then
  skip "rpc-proxy invalid token rejection (no exec pod)"
else
  # 5a. Bogus token — expect 401
  invalid_result=$(kubectl exec -n "$EXEC_NS" "$EXEC_POD" -- \
    wget -qO- --timeout=5 -S \
    --header="Authorization: Bearer invalid-token-e2e-test" \
    "http://rpc-proxy.rpc-proxy.svc.cluster.local:8094/api/v1/rpc/hosts/chatllm/activity/stream" \
    2>&1 || true)

  if echo "$invalid_result" | grep -q "401\|Unauthorized"; then
    pass "rpc-proxy rejects invalid token (401)"
  elif echo "$invalid_result" | grep -q "403\|Forbidden"; then
    pass "rpc-proxy rejects invalid token (403)"
  elif echo "$invalid_result" | grep -qi "timed out\|ETIMEDOUT"; then
    fail "rpc-proxy unreachable (timeout)"
  else
    fail "rpc-proxy did not reject invalid token"
    detail "Response: $(echo "$invalid_result" | head -3)"
  fi
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 6: control-api Health Reachable (in-cluster)
# ═════════════════════════════════════════════════════════════════════
log "Phase 6: control-api Health Reachable"

if [ -z "$EXEC_POD" ]; then
  skip "control-api health check (no exec pod)"
else
  health_result=$(kubectl exec -n "$EXEC_NS" "$EXEC_POD" -- \
    wget -qO- --timeout=5 \
    "http://control-api.control-plane.svc.cluster.local:8090/health" \
    2>&1 || echo "UNREACHABLE")

  if [ "$health_result" = "UNREACHABLE" ]; then
    fail "control-api /health unreachable from $EXEC_NS/$EXEC_POD"
  elif echo "$health_result" | grep -qi "ok\|healthy\|{"; then
    pass "control-api /health reachable from cluster"
    detail "Response: $(echo "$health_result" | head -1)"
  else
    # If we got any response, the service is reachable
    pass "control-api /health reachable (response received)"
    detail "Response: $(echo "$health_result" | head -1)"
  fi
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 7: Token Issuer/Audience Match (cross-check)
# ═════════════════════════════════════════════════════════════════════
log "Phase 7: Token Issuer/Audience Consistency"

# The control-api signs with iss=control-api, aud=rpc-proxy.
# mcp-host-config must match these values exactly.

# 7a. Issuer must be 'control-api' (what control-api signs as)
if [ "$jwt_issuer" = "control-api" ]; then
  pass "Issuer matches control-api signing identity (iss=control-api)"
else
  fail "Issuer mismatch: mcp-host expects '$jwt_issuer' but control-api signs 'control-api'"
fi

# 7b. Audience must be 'rpc-proxy' (NOT 'mcp-host')
if [ "$jwt_audience" = "rpc-proxy" ]; then
  pass "Audience matches control-api signing claim (aud=rpc-proxy)"
else
  fail "Audience mismatch: mcp-host expects '$jwt_audience' but tokens carry aud=rpc-proxy"
  detail "CRITICAL: If aud=mcp-host, all tokens will be rejected by mcp-host"
fi

# 7c. Verify the auth chain is internally consistent. CLERUM_ENABLE_AUTH is a
#     deliberate design toggle (dev/prod overlays ship it "false"); what must
#     hold in every environment is that issuer and audience are wired
#     correctly so the feature can be flipped without redeploy of ConfigMap.
if [ "$jwt_issuer" = "control-api" ] && [ "$jwt_audience" = "rpc-proxy" ]; then
  pass "Auth chain wiring consistent: iss=control-api, aud=rpc-proxy (enable_auth=$enable_auth)"
else
  fail "Auth chain wiring inconsistent"
  detail "enable_auth=$enable_auth, issuer=$jwt_issuer, audience=$jwt_audience"
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# SUMMARY
# ═════════════════════════════════════════════════════════════════════
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
if [ $FAIL -eq 0 ]; then
  echo -e "${GREEN}${BOLD}  ALL PASSED: $PASS/$TOTAL tests passed ($SKIP skipped)${NC}"
else
  echo -e "${RED}${BOLD}  FAILURES: $FAIL/$TOTAL tests failed ($PASS passed, $SKIP skipped)${NC}"
fi
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"

exit $FAIL