Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.5 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# GKE Secrets & ConfigMaps Validation — E2E for production clusters
# ═══════════════════════════════════════════════════════════════════════
#
# Validates that all required Secrets and ConfigMaps exist, contain the
# expected keys, have no placeholder values, and that JWT keys are in
# sync across services. Also checks pod logs for leaked API keys.
#
# Tests:
#   Phase 1 — Required Secrets exist with expected keys
#   Phase 2 — Required ConfigMaps exist with expected keys
#   Phase 3 — JWT key sync (mcp-host-config ↔ rpc-proxy-secrets)
#   Phase 4 — No placeholder values in Secrets
#   Phase 5 — API key leak check in pod logs
#
# Usage:
#   ./scripts/e2e/e2e-gke-secrets.sh
#   ./scripts/e2e/e2e-gke-secrets.sh --verbose
# ═══════════════════════════════════════════════════════════════════════

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

VERBOSE="${1:-}"
PASS=0
FAIL=0
SKIP=0
TOTAL=0

log()   { echo -e "${CYAN}[secrets]${NC} $*"; }
pass()  { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e "  ${GREEN}PASS${NC} $*"; }
fail()  { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e "  ${RED}FAIL${NC} $*"; }
skip()  { SKIP=$((SKIP+1)); TOTAL=$((TOTAL+1)); echo -e "  ${YELLOW}SKIP${NC} $*"; }
detail(){ [[ "$VERBOSE" == "--verbose" ]] && echo -e "       $*"; }

# ── Helper: check a Secret exists in a namespace ──────────────────────
check_secret_exists() {
  local ns="$1" name="$2"
  if kubectl get secret "$name" -n "$ns" >/dev/null 2>&1; then
    pass "Secret $ns/$name exists"
    return 0
  else
    fail "Secret $ns/$name NOT FOUND"
    return 1
  fi
}

# ── Helper: check a key exists within a Secret ────────────────────────
check_secret_key() {
  local ns="$1" name="$2" key="$3" optional="${4:-}"
  if kubectl get secret "$name" -n "$ns" -o jsonpath='{.data}' 2>/dev/null | grep -q "$key"; then
    pass "Secret $ns/$name has key '$key'"
    return 0
  else
    if [[ "$optional" == "optional" ]]; then
      skip "Secret $ns/$name key '$key' (optional, not present)"
    else
      fail "Secret $ns/$name MISSING key '$key'"
    fi
    return 1
  fi
}

# ── Helper: check a ConfigMap exists in a namespace ───────────────────
check_configmap_exists() {
  local ns="$1" name="$2"
  if kubectl get configmap "$name" -n "$ns" >/dev/null 2>&1; then
    pass "ConfigMap $ns/$name exists"
    return 0
  else
    fail "ConfigMap $ns/$name NOT FOUND"
    return 1
  fi
}

# ── Helper: check a key exists within a ConfigMap ─────────────────────
check_configmap_key() {
  local ns="$1" name="$2" key="$3"
  if kubectl get configmap "$name" -n "$ns" -o jsonpath="{.data.$key}" 2>/dev/null | grep -q .; then
    pass "ConfigMap $ns/$name has key '$key'"
    return 0
  else
    fail "ConfigMap $ns/$name MISSING or empty key '$key'"
    return 1
  fi
}

echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD}  Clerum GKE Secrets & ConfigMaps Validation${NC}"
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 1: Required Secrets Exist with Expected Keys
# ═════════════════════════════════════════════════════════════════════
log "Phase 1: Required Secrets"

# ── control-api-secrets (control-plane) ───────────────────────────────
log "  control-api-secrets (control-plane)"
if check_secret_exists control-plane control-api-secrets; then
  check_secret_key control-plane control-api-secrets CONTROL_API_SESSION_JWT_PRIVATE_KEY
  check_secret_key control-plane control-api-secrets CONTROL_API_RPC_JWT_PRIVATE_KEY
  check_secret_key control-plane control-api-secrets CONTROL_API_ADMIN_JWT_PRIVATE_KEY
  check_secret_key control-plane control-api-secrets CONTROL_API_ADMIN_BOOTSTRAP_PASSWORD_HASH
fi

# ── rpc-proxy-secrets (rpc-proxy) ─────────────────────────────────────
log "  rpc-proxy-secrets (rpc-proxy)"
if check_secret_exists rpc-proxy rpc-proxy-secrets; then
  check_secret_key rpc-proxy rpc-proxy-secrets JWT_PUBLIC_KEY
fi

# ── external-rest-api-secrets (profiles) ──────────────────────────────
log "  external-rest-api-secrets (profiles)"
if check_secret_exists profiles external-rest-api-secrets; then
  check_secret_key profiles external-rest-api-secrets EXTERNAL_REST_API_JWT_PUBLIC_KEY
fi

# ── per-Host channel-reader credentials (channels ns) ─────────────────
# (#273) The legacy `clerum-channel-reader-credentials` Secret has been
# retired. Per-Host credentials are `channel-reader-<host>-credentials`,
# written via control-api's `/admin/channel-secrets` endpoint (Control UI),
# not at bootstrap. They're optional — a Host with no Telegram/Slack/email
# wired has no Secret. We log how many exist; absence is not a failure.
channel_secret_count=$(kubectl get secret -n channels \
  -l clerum.io/component=channel-reader \
  -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | grep -c . || true)
log "  per-Host channel-reader credentials: ${channel_secret_count} Secret(s)"

# ── clerum-wrc-signing-key (control-plane) ────────────────────────────
log "  clerum-wrc-signing-key (control-plane)"
if check_secret_exists control-plane clerum-wrc-signing-key; then
  check_secret_key control-plane clerum-wrc-signing-key private.pem
  check_secret_key control-plane clerum-wrc-signing-key public.pem
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 2: Required ConfigMaps Exist with Expected Keys
# ═════════════════════════════════════════════════════════════════════
log "Phase 2: Required ConfigMaps"

# ── mcp-host-config (mcp-host) ────────────────────────────────────────
log "  mcp-host-config (mcp-host)"
if check_configmap_exists mcp-host mcp-host-config; then
  check_configmap_key mcp-host mcp-host-config CLERUM_AUTH_JWT_ISSUER
  check_configmap_key mcp-host mcp-host-config CLERUM_AUTH_JWT_AUDIENCE
  check_configmap_key mcp-host mcp-host-config CLERUM_AUTH_JWT_PUBLIC_KEY
  check_configmap_key mcp-host mcp-host-config CLERUM_ENABLE_AUTH
fi

# ── control-api-config (control-plane) ────────────────────────────────
log "  control-api-config (control-plane)"
check_configmap_exists control-plane control-api-config

# ── clerum-model-secret-mapping (mcp-host) ────────────────────────────
# Post-refactor (WRC Secret Broker): ConfigMap co-located with the Secret
# it points to. Lives in mcp-host namespace alongside chatllm-api-keys.
log "  clerum-model-secret-mapping (mcp-host)"
check_configmap_exists mcp-host clerum-model-secret-mapping

# ── clerum-wrc-public-key (sandbox-recipes) ───────────────────────────
log "  clerum-wrc-public-key (sandbox-recipes)"
check_configmap_exists sandbox-recipes clerum-wrc-public-key
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 3: JWT Key Sync Validation
# ═════════════════════════════════════════════════════════════════════
log "Phase 3: JWT Key Sync (mcp-host-config vs rpc-proxy-secrets)"

# Extract the JWT public key from mcp-host-config ConfigMap
MCP_HOST_KEY=$(kubectl get configmap mcp-host-config -n mcp-host \
  -o jsonpath='{.data.CLERUM_AUTH_JWT_PUBLIC_KEY}' 2>/dev/null || echo "")

# Extract the JWT public key from rpc-proxy-secrets Secret (base64-decoded)
RPC_PROXY_KEY=$(kubectl get secret rpc-proxy-secrets -n rpc-proxy \
  -o jsonpath='{.data.JWT_PUBLIC_KEY}' 2>/dev/null | base64 -d 2>/dev/null || echo "")

if [[ -z "$MCP_HOST_KEY" ]]; then
  fail "JWT sync: mcp-host-config CLERUM_AUTH_JWT_PUBLIC_KEY is empty"
elif [[ -z "$RPC_PROXY_KEY" ]]; then
  fail "JWT sync: rpc-proxy-secrets JWT_PUBLIC_KEY is empty"
elif [[ "$MCP_HOST_KEY" == "$RPC_PROXY_KEY" ]]; then
  pass "JWT sync: mcp-host-config public key matches rpc-proxy-secrets"
  detail "Key fingerprint: $(echo "$MCP_HOST_KEY" | head -2 | tail -1 | cut -c1-40)..."
else
  fail "JWT sync: PUBLIC KEY MISMATCH between mcp-host-config and rpc-proxy-secrets"
  detail "Run 'make gcp-sync-auth-key' to fix"
fi
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 4: No Placeholder Values in Secrets
# ═════════════════════════════════════════════════════════════════════
log "Phase 4: No Placeholder Values in Secrets"

PLACEHOLDER_PATTERNS="replace-me|replace-with|REPLACE|changeme|TODO"
SECRETS_TO_CHECK=(
  "control-plane/control-api-secrets"
  "rpc-proxy/rpc-proxy-secrets"
  "profiles/external-rest-api-secrets"
  "control-plane/clerum-wrc-signing-key"
  # (#273) clerum-channel-reader-credentials retired — per-Host Secrets
  # are written via control-api's /admin/channel-secrets and never contain
  # placeholder values (operators paste real bot tokens via Control UI).
)

for entry in "${SECRETS_TO_CHECK[@]}"; do
  ns="${entry%%/*}"
  name="${entry##*/}"

  # Get all base64-encoded values, decode them, and check for placeholders
  raw_data=$(kubectl get secret "$name" -n "$ns" -o jsonpath='{.data}' 2>/dev/null || echo "")
  if [[ -z "$raw_data" || "$raw_data" == "{}" ]]; then
    skip "Secret $ns/$name: no data keys to check"
    continue
  fi

  # Extract each base64 value, decode, and grep for placeholder patterns
  found_placeholder=false
  # Get individual keys and their values
  keys=$(kubectl get secret "$name" -n "$ns" -o json 2>/dev/null \
    | python3 -c "import sys,json; d=json.load(sys.stdin).get('data',{}); [print(k) for k in d]" 2>/dev/null || echo "")

  for key in $keys; do
    decoded=$(kubectl get secret "$name" -n "$ns" -o jsonpath="{.data.$key}" 2>/dev/null \
      | base64 -d 2>/dev/null || echo "")
    if echo "$decoded" | grep -qiE "$PLACEHOLDER_PATTERNS"; then
      fail "Secret $ns/$name key '$key' contains placeholder value"
      found_placeholder=true
    fi
  done

  if [[ "$found_placeholder" == "false" ]]; then
    pass "Secret $ns/$name: no placeholder values"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# PHASE 5: API Key Leak Check in Pod Logs
# ═════════════════════════════════════════════════════════════════════
log "Phase 5: API Key Leak Check (pod logs)"

API_KEY_PATTERNS='sk-[A-Za-z0-9]{20,}|sk-ant-[A-Za-z0-9]{20,}|zai-[A-Za-z0-9]{20,}'
DEPLOYMENTS_TO_CHECK=(
  "control-plane/control-api"
  "mcp-host/chatllm"
  "rpc-proxy/rpc-proxy"
  "profiles/external-rest-api"
  "control-plane/host-context-controller"
  "mcp-server/mcp-proxy"
)

for entry in "${DEPLOYMENTS_TO_CHECK[@]}"; do
  ns="${entry%%/*}"
  deploy="${entry##*/}"

  # Get pod name for this deployment (first pod)
  pod=$(kubectl get pods -n "$ns" -l "app=$deploy" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
  if [[ -z "$pod" ]]; then
    # Try without label selector — use deployment name match
    pod=$(kubectl get pods -n "$ns" --field-selector=status.phase=Running \
      -o jsonpath="{.items[?(@.metadata.labels.app=='$deploy')].metadata.name}" 2>/dev/null | awk '{print $1}' || echo "")
  fi

  if [[ -z "$pod" ]]; then
    skip "$ns/$deploy: no running pod found for log check"
    continue
  fi

  # Fetch last 500 lines of logs and scan for API key patterns
  logs=$(kubectl logs "$pod" -n "$ns" --tail=500 2>/dev/null || echo "")
  if [[ -z "$logs" ]]; then
    skip "$ns/$deploy: no logs available"
    continue
  fi

  if echo "$logs" | grep -oE "$API_KEY_PATTERNS" >/dev/null 2>&1; then
    leak_count=$(echo "$logs" | grep -oE "$API_KEY_PATTERNS" | wc -l | tr -d ' ')
    fail "$ns/$deploy: $leak_count API key pattern(s) found in logs!"
    detail "Patterns matched: $(echo "$logs" | grep -oE "$API_KEY_PATTERNS" | head -3 | sed 's/.\{8\}$/********/')"
  else
    pass "$ns/$deploy: no API keys leaked in logs"
  fi
done
echo ""

# ═════════════════════════════════════════════════════════════════════
# SUMMARY
# ═════════════════════════════════════════════════════════════════════
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
if [[ $FAIL -eq 0 ]]; then
  echo -e "${GREEN}${BOLD}  ALL PASSED: $PASS/$TOTAL tests passed ($SKIP skipped)${NC}"
else
  echo -e "${RED}${BOLD}  FAILURES: $FAIL/$TOTAL tests failed ($PASS passed, $SKIP skipped)${NC}"
fi
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"

exit $FAIL