Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.4 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail

# Apply inter-service tokens as native K8s Secrets using `kubectl patch`.
#
# These secrets authenticate service-to-service calls:
#   - external-rest-api → control-api   (token X)
#   - rpc-proxy         → control-api   (token Y)
#   - workflow-approval-request-reader → control-api   (Figure D consulta token)
#   - workflow-approval-request-reader → channel-reader (Slack webhook handoff)
#   - workflow-recipes / host-context-controller → control-api
#                                      (shared InternalControl JWT HMAC)
#
# Why a dedicated script (not plain `gen-jwt-keys.sh`):
#   `gen-jwt-keys.sh` uses `kubectl create ... --dry-run=client | kubectl apply`,
#   which records the Secret in `last-applied-configuration`. The next CI
#   `kubectl apply -k` would then three-way-merge against base's
#   `stringData: {}` canary and wipe the token back to empty.
#
#   This script uses `kubectl patch --type=merge`, which writes keys OUTSIDE
#   the apply envelope. `kubectl apply -k` sees `stringData: {}` in base
#   AND the matching last-applied annotation — no diff, no wipe. Real tokens
#   survive every CI redeploy. Same pattern proven by apply-registry-secrets.sh.
#
# Idempotency: most tokens are resolved in this order — env var → existing
# Secret value → fresh `openssl rand -hex 32`. Re-running the script never
# rotates tokens silently. Member-registration HMAC is stricter: env var →
# existing Secret value → minikube local dev value, and outside minikube it
# fails closed unless the env var or existing Secret value is present.
#
# Env vars consumed:
#   CONTEXT                              kubectl context (unset = current)
#   CONTROL_API_INTERNAL_TOKEN_EXT_REST  override token X (default: preserve-or-generate)
#   CONTROL_API_INTERNAL_TOKEN_RPC       override token Y (default: preserve-or-generate)
#   CONTROL_API_INTERNAL_TOKEN_WA_READER override workflow-approval-request-reader → control-api
#                                       consulta token (Figure D cross-bot fix PR1;
#                                       default: preserve-or-generate)
#   CHANNEL_READER_HANDOFF_TOKEN        override workflow-approval-request-reader → channel-reader
#                                       handoff token (default: preserve-or-generate)
#   CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET
#                                        override member-registration HMAC.
#                                        Required outside minikube unless the
#                                        control-api Secret already has a value.
#   INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET override WRC HMAC (default: preserve-or-generate)
#   INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET override HCC HMAC (default: preserve-or-generate)
#   FORCE_CONSUMER_RESTART               if true, restart every consumer including
#                                        HCC even when the HCC HMAC is unchanged
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=lib/clerum-minikube-context.sh
source "${SCRIPT_DIR}/lib/clerum-minikube-context.sh"
MEMBER_REGISTRATION_HMAC_KEY="CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET"

kctl() {
  if [ -n "${CONTEXT:-}" ]; then
    kubectl --context "$CONTEXT" "$@"
  else
    kubectl "$@"
  fi
}

log() { printf '[inter-service-tokens] %s\n' "$*" >&2; }
die() { printf '[inter-service-tokens] ERROR: %s\n' "$*" >&2; exit 1; }

# Read a plain string key from a Secret. Returns empty if Secret/key missing
# or if the current value looks like a legacy `replace-with-*` placeholder
# (so we treat those as "no real value present" and regenerate).
read_secret_key() {
  local ns="$1" secret="$2" key="$3"
  local encoded decoded
  encoded="$(kctl -n "$ns" get secret "$secret" -o jsonpath="{.data.$key}" 2>/dev/null || true)"
  [ -z "$encoded" ] && { echo ""; return; }
  decoded="$(printf '%s' "$encoded" | base64 --decode 2>/dev/null || echo "")"
  case "$decoded" in
    replace-with-*|"") echo ""; return ;;
  esac
  printf '%s' "$decoded"
}

resolve_token() {
  local env_var="$1" ns="$2" secret="$3" key="$4"
  local env_val="${!env_var:-}"
  if [ -n "$env_val" ]; then
    printf '%s' "$env_val"; return
  fi
  local existing
  existing="$(read_secret_key "$ns" "$secret" "$key")"
  if [ -n "$existing" ]; then
    printf '%s' "$existing"; return
  fi
  openssl rand -hex 32
}

is_minikube_context() {
  is_clerum_minikube_context
}

read_minikube_member_registration_hmac() {
  local candidate script
  for candidate in \
    "${CLERUM_PROJECT_DIR:-}/scripts/minikube/deploy-evenfire-member-registration.sh" \
    "${REPO_ROOT}/scripts/minikube/deploy-evenfire-member-registration.sh"; do
    [ -n "$candidate" ] || continue
    if [ -f "$candidate" ]; then
      script="$candidate"
      break
    fi
  done
  [ -n "${script:-}" ] || return 1
  awk -F= '$1 == "DEV_HMAC_SECRET" {
    gsub(/^"/, "", $2)
    gsub(/"$/, "", $2)
    print $2
    exit
  }' "$script"
}

resolve_member_registration_hmac() {
  local env_val="${!MEMBER_REGISTRATION_HMAC_KEY:-}"
  if [ -n "$env_val" ]; then
    printf '%s' "$env_val"; return
  fi

  local existing
  existing="$(read_secret_key control-plane control-api-internal-tokens "$MEMBER_REGISTRATION_HMAC_KEY")"
  if [ -n "$existing" ]; then
    printf '%s' "$existing"; return
  fi

  if is_minikube_context; then
    local minikube_hmac
    minikube_hmac="$(read_minikube_member_registration_hmac || true)"
    if [ -n "$minikube_hmac" ]; then
      printf '%s' "$minikube_hmac"; return
    fi
    # The evenfire-member-registration sibling is optional in the OSS single-tenant
    # tree (it lives in a separate repo and is excluded from the public snapshot).
    # When it is absent, generate a local dev HMAC so `make minikube-setup` continues
    # instead of aborting — member invitation / registration flows stay non-functional
    # until the sibling is deployed, per docs/deploy/minikube.md ("setup simply prints a
    # warning and continues"). The value is preserved across re-runs by the existing-Secret
    # read above. In the monorepo the sibling script is present, so this branch is inert.
    printf '[inter-service-tokens] WARN: evenfire-member-registration sibling not found; generating a local dev member-registration HMAC (invitation/registration flows will fail until it is deployed)\n' >&2
    openssl rand -hex 32; return
  fi

  die "CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET is required when no existing control-api Secret value is present"
}

ensure_secret() {
  local ns="$1" name="$2"
  if ! kctl -n "$ns" get secret "$name" >/dev/null 2>&1; then
    kctl -n "$ns" create secret generic "$name"
  fi
}

ensure_namespace() {
  local ns="$1"
  if ! kctl get ns "$ns" >/dev/null 2>&1; then
    log "Namespace '$ns' missing — creating it"
    kctl create ns "$ns"
  fi
}

log "Resolving inter-service tokens (context: ${CONTEXT:-<current>})"

# jq is expected on GitHub runners and operator workstations. JSON patches
# avoid YAML quoting surprises with commas/equals in the token strings.
command -v jq >/dev/null 2>&1 || die "jq is required (apt-get install jq)"

for ns in control-plane profiles rpc-proxy channels webhook-ingress; do
  ensure_namespace "$ns"
done
# Resolve each token from the side that already holds it (if any). We read
# from the *consumer* Secret first because the consumer is the only authority
# on its own token — if someone rotated it out-of-band, we want to keep the
# rotated value in sync on the control-api side too.
TOKEN_EXT_REST="$(resolve_token CONTROL_API_INTERNAL_TOKEN_EXT_REST \
  profiles external-rest-api-secrets EXTERNAL_REST_API_CONTROL_API_SERVICE_TOKEN)"
TOKEN_RPC="$(resolve_token CONTROL_API_INTERNAL_TOKEN_RPC \
  rpc-proxy rpc-proxy-secrets RPC_PROXY_CONTROL_API_SERVICE_TOKEN)"
TOKEN_WEBHOOK_PROXY="$(resolve_token CONTROL_API_INTERNAL_TOKEN_WEBHOOK_PROXY \
  webhook-ingress webhook-proxy-secrets WEBHOOK_PROXY_CONTROL_API_SERVICE_TOKEN)"
TOKEN_WA_READER="$(resolve_token CONTROL_API_INTERNAL_TOKEN_WA_READER \
  channels workflow-approval-request-reader-credentials control-api-token)"
TOKEN_CODEX_LLM_PROXY="$(resolve_token CONTROL_API_INTERNAL_TOKEN_CODEX_LLM_PROXY \
  control-plane codex-llm-proxy-secrets CODEX_LLM_PROXY_CONTROL_API_TOKEN)"
TOKEN_GROK_LLM_PROXY="$(resolve_token CONTROL_API_INTERNAL_TOKEN_GROK_LLM_PROXY \
  control-plane grok-llm-proxy-secrets GROK_LLM_PROXY_CONTROL_API_TOKEN)"
READER_HANDOFF_TOKEN_VALUE="$(resolve_token CHANNEL_READER_HANDOFF_TOKEN \
  channels workflow-approval-request-reader-credentials channel-reader-handoff-token)"
INTERNAL_CONTROL_WRC_HMAC="$(resolve_token INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET \
  control-plane internal-control-jwt-secrets INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET)"
INTERNAL_CONTROL_HCC_HMAC="$(resolve_token INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET \
  control-plane internal-control-jwt-secrets INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET)"
MEMBER_REGISTRATION_HMAC="$(resolve_member_registration_hmac)"

for pair in \
  "EXT_REST:$TOKEN_EXT_REST" \
  "RPC:$TOKEN_RPC" \
  "WEBHOOK_PROXY:$TOKEN_WEBHOOK_PROXY" \
  "WA_READER:$TOKEN_WA_READER" \
  "CODEX_LLM_PROXY:$TOKEN_CODEX_LLM_PROXY" \
  "GROK_LLM_PROXY:$TOKEN_GROK_LLM_PROXY" \
  "CHANNEL_READER_HANDOFF:$READER_HANDOFF_TOKEN_VALUE" \
  "INTERNAL_CONTROL_WRC_HMAC:$INTERNAL_CONTROL_WRC_HMAC" \
  "INTERNAL_CONTROL_HCC_HMAC:$INTERNAL_CONTROL_HCC_HMAC" \
  "MEMBER_REGISTRATION_HMAC:$MEMBER_REGISTRATION_HMAC"; do
  name="${pair%%:*}"
  val="${pair#*:}"
  [ -n "$val" ] || die "token $name resolved to empty — refusing to patch"
done

if [ "$TOKEN_CODEX_LLM_PROXY" = "$TOKEN_GROK_LLM_PROXY" ]; then
  die "codex-llm-proxy and grok-llm-proxy tokens must be distinct"
fi
SERVICE_TOKENS_MAP="external-rest-api=${TOKEN_EXT_REST},rpc-proxy=${TOKEN_RPC},webhook-proxy=${TOKEN_WEBHOOK_PROXY},workflow-approval-reader=${TOKEN_WA_READER},codex-llm-proxy=${TOKEN_CODEX_LLM_PROXY},grok-llm-proxy=${TOKEN_GROK_LLM_PROXY}"
INTERNAL_TOKENS_LIST="${TOKEN_EXT_REST},${TOKEN_RPC},${TOKEN_WEBHOOK_PROXY},${TOKEN_WA_READER},${TOKEN_CODEX_LLM_PROXY},${TOKEN_GROK_LLM_PROXY}"

# --- 1. control-api-internal-tokens (control-plane) ---
log "Patching Secret control-api-internal-tokens (control-plane)"
ensure_secret control-plane control-api-internal-tokens
CA_PATCH="$(jq -cn \
  --arg map  "$SERVICE_TOKENS_MAP" \
  --arg list "$INTERNAL_TOKENS_LIST" \
  --arg memberRegistrationHmacKey "$MEMBER_REGISTRATION_HMAC_KEY" \
  --arg memberRegistrationHmac "$MEMBER_REGISTRATION_HMAC" \
  '{stringData: {
    CONTROL_API_INTERNAL_SERVICE_TOKENS: $map,
    CONTROL_API_INTERNAL_TOKENS: $list,
    ($memberRegistrationHmacKey): $memberRegistrationHmac
  }}')"
kctl -n control-plane patch secret control-api-internal-tokens --type=merge -p "$CA_PATCH"

# --- 1b. retired reader -> control-api service token cleanup ---
log "Removing legacy reader internal control-api tokens if present (channels)"
kctl -n channels delete secret channel-reader-internal-tokens --ignore-not-found=true >/dev/null
kctl -n channels patch secret workflow-approval-request-reader-credentials \
  --type=json \
  -p='[{"op":"remove","path":"/data/service-token"}]' >/dev/null 2>&1 || true

# --- 2. internal-control-jwt-secrets (control-plane) ---
# Compare the HCC-consumed key only. The merge patch writes WRC + HCC together,
# so a whole-Secret resourceVersion (or parsing `patched (no change)`) would
# Recreate HCC on a WRC-only rotation. Values stay in shell locals — never logged.
HCC_HMAC_BEFORE="$(read_secret_key control-plane internal-control-jwt-secrets INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET)"
if [ "${FORCE_CONSUMER_RESTART:-}" = "true" ] || \
   [ -z "$HCC_HMAC_BEFORE" ] || \
   [ "$HCC_HMAC_BEFORE" != "$INTERNAL_CONTROL_HCC_HMAC" ]; then
  HCC_SECRET_CHANGED=true
else
  HCC_SECRET_CHANGED=false
fi
log "Patching Secret internal-control-jwt-secrets (control-plane)"
ensure_secret control-plane internal-control-jwt-secrets
INTERNAL_CONTROL_PATCH="$(jq -cn \
  --arg wrc "$INTERNAL_CONTROL_WRC_HMAC" \
  --arg hcc "$INTERNAL_CONTROL_HCC_HMAC" \
  '{stringData: {
    INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET: $wrc,
    INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET: $hcc
  }}')"
kctl -n control-plane patch secret internal-control-jwt-secrets --type=merge -p "$INTERNAL_CONTROL_PATCH"
LEGACY_INTERNAL_CONTROL_KEY="$(printf 'INTERNAL_CONTROL_JWT_%s_%s' HMAC SECRET)"
if kctl -n control-plane get secret internal-control-jwt-secrets -o json \
  | jq -e --arg key "$LEGACY_INTERNAL_CONTROL_KEY" '.data[$key]' >/dev/null; then
  LEGACY_INTERNAL_CONTROL_REMOVE_PATCH="$(jq -cn \
    --arg path "/data/${LEGACY_INTERNAL_CONTROL_KEY}" \
    '[{op: "remove", path: $path}]')"
  kctl -n control-plane patch secret internal-control-jwt-secrets --type=json \
    -p="$LEGACY_INTERNAL_CONTROL_REMOVE_PATCH"
fi

# --- 3. external-rest-api-secrets (profiles) ---
# Adds the SERVICE_TOKEN key alongside the JWT public key that gen-jwt-keys.sh
# already writes. Patch only adds keys — does not touch JWT_PUBLIC_KEY.
log "Patching Secret external-rest-api-secrets (profiles)"
ensure_secret profiles external-rest-api-secrets
EXT_PATCH="$(jq -cn --arg t "$TOKEN_EXT_REST" \
  '{stringData: {
    EXTERNAL_REST_API_CONTROL_API_SERVICE_TOKEN: $t
  }}')"
kctl -n profiles patch secret external-rest-api-secrets --type=merge -p "$EXT_PATCH"

# --- 4. rpc-proxy-secrets (rpc-proxy) ---
# Adds/updates RPC_PROXY_CONTROL_API_SERVICE_TOKEN alongside JWT public key
# and desktop tokens written by gen-jwt-keys.sh.
log "Patching Secret rpc-proxy-secrets (rpc-proxy)"
ensure_secret rpc-proxy rpc-proxy-secrets
RPC_PATCH="$(jq -cn --arg t "$TOKEN_RPC" \
  '{stringData: {RPC_PROXY_CONTROL_API_SERVICE_TOKEN: $t}}')"
kctl -n rpc-proxy patch secret rpc-proxy-secrets --type=merge -p "$RPC_PATCH"

# --- 4b. webhook-proxy-secrets (webhook-ingress) ---
# Adds/updates WEBHOOK_PROXY_CONTROL_API_SERVICE_TOKEN. webhook-proxy uses
# this token to authenticate against control-api's registry endpoint
# (/api/v1/internal/webhook/registry/...).
log "Patching Secret webhook-proxy-secrets (webhook-ingress)"
ensure_secret webhook-ingress webhook-proxy-secrets
WEBHOOK_PROXY_PATCH="$(jq -cn --arg t "$TOKEN_WEBHOOK_PROXY" \
  '{stringData: {WEBHOOK_PROXY_CONTROL_API_SERVICE_TOKEN: $t}}')"
kctl -n webhook-ingress patch secret webhook-proxy-secrets --type=merge -p "$WEBHOOK_PROXY_PATCH"

# --- 4d. codex-llm-proxy-secrets (control-plane) ---
# Dedicated service token only. The proxy never receives the full
# CONTROL_API_INTERNAL_SERVICE_TOKENS map.
log "Patching Secret codex-llm-proxy-secrets (control-plane)"
ensure_secret control-plane codex-llm-proxy-secrets
CODEX_LLM_PROXY_PATCH="$(jq -cn --arg t "$TOKEN_CODEX_LLM_PROXY" \
  '{stringData: {CODEX_LLM_PROXY_CONTROL_API_TOKEN: $t}}')"
kctl -n control-plane patch secret codex-llm-proxy-secrets --type=merge -p "$CODEX_LLM_PROXY_PATCH"

# --- 4e. grok-llm-proxy-secrets (control-plane) ---
log "Patching Secret grok-llm-proxy-secrets (control-plane)"
ensure_secret control-plane grok-llm-proxy-secrets
GROK_LLM_PROXY_PATCH="$(jq -cn --arg t "$TOKEN_GROK_LLM_PROXY" \
  '{stringData: {GROK_LLM_PROXY_CONTROL_API_TOKEN: $t}}')"
kctl -n control-plane patch secret grok-llm-proxy-secrets --type=merge -p "$GROK_LLM_PROXY_PATCH"

# --- 4c. workflow-approval-request-reader-credentials (channels) ---
# Figure D cross-bot fix (PR1): the reader uses this token to authenticate its
# consulta call to control-api
# (GET /api/v1/internal/workflow-approval-reader/approvals/:id/can-approve).
# Same internalServiceAuth pattern as rpc-proxy / webhook-proxy.
log "Patching Secret workflow-approval-request-reader-credentials (channels) — control-api-token"
ensure_secret channels workflow-approval-request-reader-credentials
WA_READER_PATCH="$(jq -cn \
  --arg controlApiToken "$TOKEN_WA_READER" \
  --arg channelReaderHandoffToken "$READER_HANDOFF_TOKEN_VALUE" \
  '{stringData: {
    "control-api-token": $controlApiToken,
    "channel-reader-handoff-token": $channelReaderHandoffToken
  }}')"
kctl -n channels patch secret workflow-approval-request-reader-credentials --type=merge -p "$WA_READER_PATCH"

# Roll the consumers so they pick up the fresh tokens on first deploy. Safe
# no-ops if the Deployments do not yet exist. HCC is Recreate + replicas:1
# and reloads HMAC only at process start, so skip that extra rollout when the
# HCC key this run writes equals the key already in the Secret (image/spec
# applies still roll HCC themselves). This is not "the running pod env matches
# the Secret": an out-of-band kubectl patch of the HCC key, then a re-run that
# preserve-or-generates from the Secret, will skip. Heal that with
# FORCE_CONSUMER_RESTART=true.
for pair in "control-plane:control-api" \
            "control-plane:codex-llm-proxy" \
            "control-plane:grok-llm-proxy" \
            "control-plane:workflow-recipes" \
            "control-plane:host-context-controller" \
            "profiles:external-rest-api" \
            "rpc-proxy:rpc-proxy" \
            "webhook-ingress:webhook-proxy" \
            "channels:clerum-workflow-approval-request-reader"; do
  ns="${pair%%:*}"
  dep="${pair#*:}"
  if [ "$ns" = "control-plane" ] && [ "$dep" = "host-context-controller" ] && \
     [ "$HCC_SECRET_CHANGED" != "true" ]; then
    log "Skipping rollout of $ns/$dep: hcc-hmac unchanged"
    continue
  fi
  if kctl -n "$ns" get deploy "$dep" >/dev/null 2>&1; then
    log "Rolling deployment $ns/$dep to pick up fresh Secret values"
    kctl -n "$ns" rollout restart deploy "$dep" >/dev/null
  fi
done

if kctl -n channels get deploy -l app=channel-reader >/dev/null 2>&1; then
  log "Rolling channel-reader deployments to pick up fresh Secret values"
  kctl -n channels rollout restart deploy -l app=channel-reader >/dev/null || true
fi

log "Done."