Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
5.1 KB
Raw Blame
#!/usr/bin/env bash
# ======================================================================
# Verify Secret Keys — pre-rollout sanity check
# ======================================================================
#
# After `kubectl apply`, Deployments reference Secrets via secretKeyRef
# and envFrom.secretRef. If the referenced keys don't exist, pods enter
# CreateContainerConfigError or CrashLoopBackOff — and the only signal
# is a 120s rollout timeout.
#
# This script inspects every Deployment across the target namespaces,
# extracts Secret references, and verifies each key exists on the
# cluster. Runs in ~5s and gives an actionable error message.
#
# Usage:
#   ./deploy/scripts/verify-secrets.sh [NAMESPACE...]
#   CONTEXT=gke_... ./deploy/scripts/verify-secrets.sh
#
# If no namespaces are given, defaults to the standard Clerum set.
# ======================================================================

set -euo pipefail

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

kctl() {
  if [ -n "${CONTEXT:-}" ]; then
    kubectl --context "$CONTEXT" "$@"
  else
    kubectl "$@"
  fi
}

DEFAULT_NAMESPACES="channels control-plane mcp-host mcp-server profiles registry rpc-proxy sandbox-recipes"
NAMESPACES="${*:-$DEFAULT_NAMESPACES}"

missing=0
checked=0

echo -e "${CYAN}[verify-secrets]${NC} Checking Secret references in Deployments..."
echo ""

for ns in $NAMESPACES; do
  # Get all Deployments in this namespace as JSON
  deploys_json=$(kctl get deploy -n "$ns" -o json 2>/dev/null || echo '{"items":[]}')
  deploy_count=$(echo "$deploys_json" | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null || echo "0")

  if [[ "$deploy_count" == "0" ]]; then
    continue
  fi

  # Extract all secretKeyRef entries: (deployName, secretName, key)
  # and all envFrom.secretRef entries: (deployName, secretName)
  refs=$(echo "$deploys_json" | python3 -c "
import json, sys

data = json.load(sys.stdin)
refs = []

for deploy in data.get('items', []):
    dname = deploy['metadata']['name']
    for container in deploy['spec']['template']['spec'].get('containers', []):
        # envFrom → secretRef (all keys in the secret are loaded)
        for ef in container.get('envFrom', []):
            sr = ef.get('secretRef')
            if sr:
                optional = ef.get('optional', sr.get('optional', False))
                refs.append(('secretRef', dname, sr['name'], '*', str(optional)))

        # env → valueFrom → secretKeyRef (specific key)
        for env in container.get('env', []):
            vf = env.get('valueFrom', {})
            skr = vf.get('secretKeyRef')
            if skr:
                optional = skr.get('optional', False)
                refs.append(('secretKeyRef', dname, skr['name'], skr['key'], str(optional)))

for r in refs:
    print('|'.join(r))
" 2>/dev/null || true)

  if [[ -z "$refs" ]]; then
    continue
  fi

  ns_header_shown=false

  while IFS='|' read -r ref_type deploy_name secret_name key_name optional; do
    [[ -z "$ref_type" ]] && continue

    # Skip optional refs — K8s won't block the pod if they're missing
    if [[ "$optional" == "True" ]]; then
      continue
    fi

    checked=$((checked + 1))

    if [[ "$ref_type" == "secretRef" ]]; then
      # Check the Secret itself exists
      if ! kctl get secret "$secret_name" -n "$ns" >/dev/null 2>&1; then
        if [[ "$ns_header_shown" == false ]]; then
          echo -e "${BOLD}── $ns ──${NC}"
          ns_header_shown=true
        fi
        echo -e "  ${RED}MISSING${NC}  Secret ${BOLD}$secret_name${NC} (used by deploy/$deploy_name via envFrom.secretRef)"
        missing=$((missing + 1))
      fi
    else
      # secretKeyRef — check specific key
      val=$(kctl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${key_name}}" 2>/dev/null || echo "")
      if [[ -z "$val" ]]; then
        if [[ "$ns_header_shown" == false ]]; then
          echo -e "${BOLD}── $ns ──${NC}"
          ns_header_shown=true
        fi
        # Distinguish "secret missing" from "key missing"
        if ! kctl get secret "$secret_name" -n "$ns" >/dev/null 2>&1; then
          echo -e "  ${RED}MISSING${NC}  Secret ${BOLD}$secret_name${NC} (used by deploy/$deploy_name)"
        else
          echo -e "  ${RED}MISSING${NC}  Key ${BOLD}$key_name${NC} in Secret ${BOLD}$secret_name${NC} (used by deploy/$deploy_name)"
        fi
        missing=$((missing + 1))
      fi
    fi
  done <<< "$refs"
done

echo ""
if [[ $missing -gt 0 ]]; then
  echo -e "${RED}${BOLD}FAILED${NC}: $missing Secret reference(s) missing out of $checked checked."
  echo ""
  echo -e "${YELLOW}To fix:${NC}"
  echo "  • For dev cluster:  make gcp-dev-gen-keys   (regenerates all secrets including desktop tokens)"
  echo "  • For prod cluster: CONFIRM=yes make gcp-prod-gen-keys"
  echo "  • For minikube:     make minikube-gen-keys"
  echo ""
  echo "Then re-run the deploy. Pods referencing missing Secrets will enter"
  echo "CreateContainerConfigError or CrashLoopBackOff until the keys are seeded."
  exit 1
else
  echo -e "${GREEN}${BOLD}OK${NC}: All $checked Secret reference(s) verified."
fi