Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# ======================================================================
# Clerum Minikube Full Setup — Idempotent Orchestrator
# ======================================================================
#
# Single script that sets up the entire Clerum minikube cluster from
# scratch. Safe to run multiple times (idempotent).
#
# Usage:
# ./scripts/minikube/full-setup.sh # Full setup
# ./scripts/minikube/full-setup.sh --skip-build # Skip image build
# ./scripts/minikube/full-setup.sh --skip-uis # Skip UIs and Desktop App
# ./scripts/minikube/full-setup.sh --defer-bootstrap-seed
# # Leave first-run setup for browser E2E
# ./scripts/minikube/full-setup.sh --reset-db # Reset postgres DB
# ./scripts/minikube/full-setup.sh --force-keys # Regenerate JWT keys
# ./scripts/minikube/full-setup.sh -h | --help
#
# Environment:
# MINIKUBE_PROFILE=clerum-test Target minikube profile.
# MINIKUBE_MULTI_NODE=true Opt into a two-node local cluster.
# MINIKUBE_NODES=2 Explicit node count for multi-node gates.
# MINIKUBE_RECREATE_PROFILE=true Allow destructive profile recreation only
# CONFIRM_PROFILE=<profile> when it matches the exact target profile.
# BRANCH_PROFILE_DEPLOY_DIR=<dir> Optional branch-profile deploy cache root.
# IMAGE_SOURCE=ghcr|local ghcr (default) pulls published images;
# local builds all of them from source.
# MINIKUBE_IMAGE_TAG=<tag> Render-time override of the committed
# ghcr pin. Never committed. Use `latest`
# before a release tag exists.
#
# Loads .env from project root if present (API keys, credentials).
# ======================================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
# shellcheck source=scripts/e2e/load-dotenv.sh
source "${PROJECT_DIR}/scripts/e2e/load-dotenv.sh"
# shellcheck source=scripts/e2e/admin-credentials.sh
source "${PROJECT_DIR}/scripts/e2e/admin-credentials.sh"
# shellcheck source=scripts/e2e/minimal-bootstrap-contract.sh
source "${PROJECT_DIR}/scripts/e2e/minimal-bootstrap-contract.sh"
# ── Color helpers ──────────────────────────────────────────────────────
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
log() { echo -e "${CYAN}[SETUP]${NC} $*"; }
ok() { echo -e "${GREEN} OK${NC} -- $*"; }
warn() { echo -e "${YELLOW} WARN${NC} -- $*"; }
err() { echo -e "${RED} ERROR${NC} -- $*"; }
step_header() {
local num=$1 total=$2 title=$3
echo ""
echo -e "${BOLD}================================================================${NC}"
echo -e "${BOLD} Step ${num}/${total}: ${title}${NC}"
echo -e "${BOLD}================================================================${NC}"
}
# ── McpServer envSecret validation ─────────────────────────────────────
# Fail-fast guard: when an McpServer CRD declares `spec.envSecret.name`,
# the referenced Secret MUST exist in the same namespace or HCC silently
# hangs without reconciling the Deployment. This function enumerates all
# McpServers, checks each envSecret reference, and aborts the setup with
# a clear error message if any are missing.
#
# Respects MINIKUBE_SKIP_SECRET_VERIFY=true to bypass validation (rollback
# escape hatch per plan §6 rollback matrix).
#
# Returns: 0 on success, 1 if any envSecret reference is unresolved.
validate_mcpserver_secrets() {
if [ "${MINIKUBE_SKIP_SECRET_VERIFY:-false}" = "true" ]; then
warn "MINIKUBE_SKIP_SECRET_VERIFY=true — skipping McpServer envSecret validation"
return 0
fi
log "Validating McpServer envSecret references..."
local rows
rows=$($KC get mcpservers -A \
-o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.spec.envSecret.name}{"\n"}{end}' \
2>/dev/null || true)
if [ -z "$rows" ]; then
ok "No McpServer CRDs found — nothing to validate"
return 0
fi
local failed=0
local missing_lines=""
while IFS=$'\t' read -r ns name secret_name; do
[ -z "${ns:-}" ] && continue
[ -z "${secret_name:-}" ] && continue
if ! $KC get secret "$secret_name" -n "$ns" >/dev/null 2>&1; then
failed=$((failed + 1))
missing_lines+=" ${RED}✗${NC} McpServer ${ns}/${name} references missing Secret ${ns}/${secret_name}"$'\n'
fi
done <<< "$rows"
if [ "$failed" -gt 0 ]; then
err "McpServer envSecret validation failed (${failed} missing binding(s)):"
# shellcheck disable=SC2059
printf "${missing_lines}"
echo ""
echo -e " ${BOLD}Hint:${NC} Create the missing Secrets before re-running setup. Example:"
echo -e " ${CYAN}kubectl --context=${PROFILE} create secret generic <name> \\${NC}"
echo -e " ${CYAN} -n <namespace> --from-literal=<key>=<value>${NC}"
echo -e " Or drop a gitignored ${CYAN}mcp-servers/<svc>/secret.yaml${NC} and re-run."
echo -e " To bypass this check (not recommended): ${CYAN}MINIKUBE_SKIP_SECRET_VERIFY=true${NC}"
return 1
fi
ok "All McpServer envSecret references resolved"
return 0
}
# ── Parse flags ────────────────────────────────────────────────────────
SKIP_BUILD=false
SKIP_UIS="${MINIKUBE_SKIP_UIS:-false}"
RESET_DB=true
FORCE_KEYS=false
DEFER_BOOTSTRAP_SEED="${MINIKUBE_DEFER_BOOTSTRAP_SEED:-false}"
CONTROL_DB_RESET_PVC_UID="${CONTROL_DB_RESET_PVC_UID:-}"
CONTROL_DB_RESET_RESUME="${CONTROL_DB_RESET_RESUME:-false}"
case "${SKIP_UIS}" in
true|1|yes) SKIP_UIS=true ;;
*) SKIP_UIS=false ;;
esac
# ghcr (default): pull published ghcr.io/evenfire-ai images and render the
# -ghcr overlays. local: build every image from source, today's behaviour.
#
# This is what an ACQUIRING run would do. A --skip-build run acquires nothing,
# so it re-resolves the mode from what the cluster actually holds -- see the
# image_mode_source block after the flag loop.
IMAGE_SOURCE="${IMAGE_SOURCE:-ghcr}"
case "$IMAGE_SOURCE" in
ghcr|local) ;;
*) err "Unknown IMAGE_SOURCE: '${IMAGE_SOURCE}' (expected: ghcr | local)"; exit 1 ;;
esac
export IMAGE_SOURCE
# A render-time override of the committed pin, NEVER committed. It exists for
# two real needs the pin alone cannot serve: exercising the pull path before a
# release tag exists (MINIKUBE_IMAGE_TAG=latest) and recovering from a bad
# release forward (MINIKUBE_IMAGE_TAG=v0.6.1). See apply_image_tag_override.
export MINIKUBE_IMAGE_TAG="${MINIKUBE_IMAGE_TAG:-}"
# Rebuild the DB from scratch by default (RESET_DB=true above) so a stale
# control-postgres volume from an older build cannot drift from the current
# schema/grant contract: the migration gate enforces an exact runtime-access
# contract, and already-recorded migrations never re-run to reconcile it. Opt
# out with REUSE_DB=true (or --keep-db) to preserve an existing volume.
case "${REUSE_DB:-false}" in
true|1|yes) RESET_DB=false ;;
*) : ;;
esac
SEED_PROFILE="${MINIKUBE_SEED_PROFILE:-minimal}"
# Unlike the SKIP_UIS boolean normalizer, an unrecognized profile is a hard
# error — silently falling back to `minimal` would skip the E2E fixtures a
# caller explicitly asked for, and the failure would surface much later as a
# confusing test failure.
case "${SEED_PROFILE}" in
minimal|e2e) ;;
*) err "Unknown SEED_PROFILE: '${SEED_PROFILE}' (expected: minimal | e2e)"; exit 1 ;;
esac
usage() {
cat <<EOF
Usage: $(basename "$0") [OPTIONS]
Idempotent orchestrator for Clerum minikube full-stack setup.
Options:
--skip-build Skip Docker image builds (use pre-loaded images)
--skip-uis Skip Control UI, Profile UI, and Desktop App image builds.
Control/Profile UI cluster deployments are also skipped.
--seed-profile=<minimal|e2e>
minimal (default): a clean install — one agent, no
test fixtures. e2e: adds the E2E fixtures (test user,
e2e-* recipes, demo MCP servers). Requires ADMIN_PASSWORD
unless --seed-profile=e2e.
--defer-bootstrap-seed
local branch-E2E mode: skip Step 10's admin/user
seed so the browser owns the one-shot
/api/v1/admin/auth/setup bootstrap. Requires
IMAGE_SOURCE=local and --seed-profile=minimal.
--reset-db Force a postgres DB rebuild (already the default; deletes PVC, re-deploys)
--keep-db Preserve the existing postgres DB volume (skip the default rebuild)
--force-keys Force JWT key regeneration (invalidates existing tokens)
-h, --help Show this help message
Environment:
MINIKUBE_PROFILE Target profile (default: clerum-test)
MINIKUBE_MULTI_NODE Set true to opt into a two-node local cluster
MINIKUBE_NODES Explicit node count; values >1 imply multi-node
MINIKUBE_SKIP_UIS Set true to skip Control UI, Profile UI, and Desktop App
REUSE_DB Set true to preserve the existing postgres DB volume;
default rebuilds control-postgres from scratch every run
MINIKUBE_SEED_PROFILE, ADMIN_PASSWORD
See --seed-profile above
MINIKUBE_DEFER_BOOTSTRAP_SEED
Set true to select the same browser-bootstrap mode
as --defer-bootstrap-seed
MINIKUBE_RECREATE_PROFILE
Set true to allow destructive broken-profile recreation
CONFIRM_PROFILE Must match MINIKUBE_PROFILE before any profile deletion
CONTROL_DB_RESET_PVC_UID
Required with --reset-db: exact approved PVC UID, or
"none" when no control-postgres-data PVC exists
CONTROL_DB_RESET_RESUME
Set true only to resume the same interrupted reset
BRANCH_PROFILE_DEPLOY_DIR
Optional deploy cache root; when set, kustomize uses
<dir>/overlays/minikube instead of PROJECT_DIR/deploy
IMAGE_SOURCE ghcr (default) pulls published ghcr.io/evenfire-ai
images; local builds every image from source. Ignored
with --skip-build, which acquires nothing and so
follows what the cluster already holds
MINIKUBE_IMAGE_TAG Render-time override applied to a temp copy of deploy/
and never committed. Without it the tag is whatever the
last acquisition recorded, else the committed pin. Use
'latest' before a release tag exists
Loads .env from project root if present. Supported variables:
OPENAI_API_KEY, CLAUDE_API_KEY, ZAI_API_KEY, BAILIAN_API_KEY
CLERUM_TELEGRAM_BOT_TOKEN, CLERUM_SLACK_BOT_TOKEN
CLERUM_EMAIL_USERNAME, CLERUM_EMAIL_PASSWORD
CLERUM_MODEL_PROVIDER, CLERUM_MODEL_NAME
Host model (scripts/minikube/host-model.sh): CLERUM_MODEL_PROVIDER wins, with
CLERUM_MODEL_NAME or that provider's default. CLERUM_MODEL_NAME without
CLERUM_MODEL_PROVIDER is refused. Otherwise the first key present, in the
order above, picks the provider; with no key the Host gets
openai/gpt-5.4-mini and the agent does not reply. The pair must be an
enabled llm_allowed_models row, or setup exits 1 before applying the Host.
Examples:
$(basename "$0") # Full setup from scratch
$(basename "$0") --skip-build # Re-deploy without rebuilding images
$(basename "$0") --skip-uis # Deploy backend services without UIs
$(basename "$0") --seed-profile=e2e # Full E2E fixture set
$(basename "$0") --defer-bootstrap-seed # Leave first-run setup to browser E2E
$(basename "$0") --keep-db # Preserve the existing DB (skip the default rebuild)
$(basename "$0") --force-keys # Regenerate all JWT keys
EOF
exit 0
}
for arg in "$@"; do
case "$arg" in
--skip-build) SKIP_BUILD=true ;;
--skip-uis) SKIP_UIS=true ;;
--seed-profile=*) SEED_PROFILE="${arg#*=}" ;;
--defer-bootstrap-seed) DEFER_BOOTSTRAP_SEED=true ;;
--reset-db) RESET_DB=true ;;
--keep-db) RESET_DB=false ;;
--force-keys) FORCE_KEYS=true ;;
-h|--help) usage ;;
*) err "Unknown flag: $arg"; usage ;;
esac
done
# Re-validate: --seed-profile= above can override the env-derived value, so
# the same check that guards MINIKUBE_SEED_PROFILE must run again here or
# `--seed-profile=bogus` slips through unvalidated.
case "${SEED_PROFILE}" in
minimal|e2e) ;;
*) err "Unknown --seed-profile: '${SEED_PROFILE}' (expected: minimal | e2e)"; exit 1 ;;
esac
case "${DEFER_BOOTSTRAP_SEED}" in
true|1|yes) DEFER_BOOTSTRAP_SEED=true ;;
false|0|no|"") DEFER_BOOTSTRAP_SEED=false ;;
*) err "Unknown MINIKUBE_DEFER_BOOTSTRAP_SEED: '${DEFER_BOOTSTRAP_SEED}' (expected: true | false)"; exit 1 ;;
esac
if [ "$DEFER_BOOTSTRAP_SEED" = true ]; then
if [ "$SEED_PROFILE" != "minimal" ]; then
err "--defer-bootstrap-seed requires --seed-profile=minimal; the e2e fixture profile owns its seed flow."
exit 1
fi
if [ "$IMAGE_SOURCE" != "local" ]; then
err "--defer-bootstrap-seed requires IMAGE_SOURCE=local so browser evidence can correspond to this worktree HEAD."
exit 1
fi
fi
# ── The mode follows the cluster on a run that acquires nothing ────────
#
# THE OVERLAY MUST FOLLOW THE IMAGES ON THE NODE, NOT THIS SHELL. --skip-build
# re-deploys WITHOUT pulling or building anything, so the only truthful source
# for "which images does this cluster run" is what the last acquisition
# recorded. Taking IMAGE_SOURCE instead rendered the -ghcr overlays over a
# locally built cluster -- every Deployment then names a ghcr ref nothing ever
# pulled, i.e. cluster-wide ImagePullBackOff -- and the mirror image (clerum/*
# refs over a pulled cluster) is the same bug with the operands swapped. It is
# not a rare path: `make minikube-setup` ALWAYS passes IMAGE_SOURCE (the
# Makefile defaults it to ghcr), so the environment can never be read as "the
# operator asked for this".
#
# On an ACQUIRING run the environment still decides, because that run is about
# to rewrite the record -- otherwise `make minikube-setup-local` over a pulled
# cluster could never switch a cluster back to local builds.
#
# image_mode_source falls back to the environment when nothing is recorded, so
# the FIRST run on a fresh cluster is unchanged; this is the same resolver
# `make minikube-deploy-all` and `build-images.sh --verify-only` already use.
# shellcheck source=scripts/minikube/image-mode.sh
source "${SCRIPT_DIR}/image-mode.sh"
# Host model resolution and its llm_allowed_models check (step 6f).
# shellcheck source=scripts/minikube/host-model.sh
source "${SCRIPT_DIR}/host-model.sh"
if [ "$SKIP_BUILD" = true ]; then
RECORDED_IMAGE_SOURCE="$(image_mode_source "$PROJECT_DIR")" || exit 1
if [ "$RECORDED_IMAGE_SOURCE" != "$IMAGE_SOURCE" ]; then
warn "This cluster's images were acquired as IMAGE_SOURCE=${RECORDED_IMAGE_SOURCE} (deploy/minikube/.image-manifest.json)."
warn "--skip-build acquires nothing, so the overlay follows the cluster, not IMAGE_SOURCE=${IMAGE_SOURCE}."
warn "To change the mode, re-run WITHOUT --skip-build so the images are actually re-acquired."
IMAGE_SOURCE="$RECORDED_IMAGE_SOURCE"
export IMAGE_SOURCE
fi
fi
# ── Load .env ──────────────────────────────────────────────────────────
# Worktrees resolve through git-common-dir to the primary checkout. Dotenv is
# parsed as data (never sourced as shell code), and admin aliases use the
# shared layer-first contract: canonical .env, then process, then local fallback.
ENV_FILE="$(dotenv_canonical_root "${PROJECT_DIR}")"
if [ -n "$ENV_FILE" ]; then
log "Loading .env from ${ENV_FILE}..."
dotenv_load_file "$ENV_FILE"
ok ".env loaded"
else
warn "No .env found in ${PROJECT_DIR} or main repo — channel-reader + LLM secrets will use placeholders"
fi
ADMIN_PASSWORD="$(e2e_resolve_admin_password "${PROJECT_DIR}" || true)"
export ADMIN_PASSWORD
PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
KC="kubectl --context=${PROFILE}"
# Full setup is a mutating profile transition as well. It acquires the same
# branch-owned lease used by T2 when run standalone, or validates the inherited
# lease when called by the T2 orchestrator. This is initialized before the
# config-only seam but acquired only after the profile is started and reachable.
T2_PROJECT_DIR="${T2_PROJECT_DIR:-${PROJECT_DIR}}"
T2_PROFILE="${T2_PROFILE:-${PROFILE}}"
T2_CONTEXT="${T2_CONTEXT:-${PROFILE}}"
T2_GATE_ID="${T2_GATE_ID:-minikube-setup}"
T2_SKIP_LOCK="${T2_SKIP_LOCK:-false}"
T2_SETUP_HANDOFF_REQUIRED="${T2_SETUP_HANDOFF_REQUIRED:-false}"
T2_SETUP_HANDOFF_TRANSITION="${T2_SETUP_HANDOFF_TRANSITION:-}"
T2_SETUP_HANDOFF_ROOT="${T2_SETUP_HANDOFF_ROOT:-${PROJECT_DIR}/.local-notes/infra/t2-setup-handoffs}"
T2_SETUP_HANDOFF_TTL_SECONDS="${T2_SETUP_HANDOFF_TTL_SECONDS:-300}"
# shellcheck source=scripts/minikube/t2-common.sh
source "${SCRIPT_DIR}/t2-common.sh"
case "${T2_SETUP_HANDOFF_REQUIRED}" in
true|false) ;;
*) err "T2_SETUP_HANDOFF_REQUIRED must be true or false"; exit 1 ;;
esac
if [ "${T2_SETUP_HANDOFF_REQUIRED}" = true ]; then
if [ "${T2_SKIP_LOCK}" != true ] || [ -z "${T2_RUN_ID}" ]; then
err "T2 setup handoff requires the live inherited T2 lease and run identity"
exit 1
fi
case "${T2_SETUP_HANDOFF_TRANSITION}" in
full-bootstrap|full-reconcile) ;;
*) err "T2 setup handoff requires a full-bootstrap or full-reconcile transition"; exit 1 ;;
esac
if [ "${IMAGE_SOURCE}" != local ]; then
err "T2 setup handoff requires IMAGE_SOURCE=local"
exit 1
fi
fi
TOTAL_STEPS=12
# MINIKUBE_IMAGE_TAG overrides the committed pin AT RENDER TIME ONLY.
#
# A stdout filter cannot do this: the overlay is rendered from a DIRECTORY at
# four independent sites (`kubectl apply -k` at Step 6b, two
# `kubectl kustomize | apply -f -` calls, and run-control-api-db-migration.sh,
# which takes --overlay and extracts the control-api image from the render).
# `kustomize edit set image` is not available either -- everything here uses
# `kubectl kustomize`, and kubectl has no `edit` subcommand, so it would mean a
# new prerequisite. Mutating the committed component in place would make this
# operator lever a SECOND WRITER of the release coordinate, which is exactly
# what the one-writer rule forbids.
#
# So: copy deploy/ to a temp dir, rewrite the component's newTag THERE, and
# point the existing BRANCH_PROFILE_DEPLOY_DIR hook at the copy. The whole tree
# must move because `resources: ../minikube` and
# deploy/scripts/minikube-detect-k8s-api-ip.sh both resolve relative to that
# root.
apply_image_tag_override() {
[ "$IMAGE_SOURCE" = ghcr ] || return 0
[ -n "$MINIKUBE_IMAGE_TAG" ] || return 0
local src override_root
src="${BRANCH_PROFILE_DEPLOY_DIR:-${PROJECT_DIR}/deploy}"
override_root="$(mktemp -d)"
cp -R "$src" "${override_root}/deploy"
local component="${override_root}/deploy/components/ghcr-images/kustomization.yaml"
[ -f "$component" ] || { err "ghcr component not found in the deploy copy at ${component}"; exit 1; }
# In-place on the COPY only. -i.bak plus rm keeps this portable across BSD
# and GNU sed; `sed -i ''` is a BSD-only spelling that GNU sed rejects.
sed -i.bak "s|^\([[:space:]]*newTag:[[:space:]]*\).*$|\1${MINIKUBE_IMAGE_TAG}|" "$component"
rm -f "${component}.bak"
local rewritten
rewritten="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\)[[:space:]]*$/\1/p' "$component" | sort -u)"
[ "$rewritten" = "$MINIKUBE_IMAGE_TAG" ] \
|| { err "tag override did not apply cleanly: copy now carries '${rewritten}', expected '${MINIKUBE_IMAGE_TAG}'"; exit 1; }
export BRANCH_PROFILE_DEPLOY_DIR="${override_root}/deploy"
}
apply_image_tag_override
ACTIVE_MINIKUBE_DEPLOY_DIR="${BRANCH_PROFILE_DEPLOY_DIR:-${PROJECT_DIR}/deploy}"
# The kustomize dir is where the generated k8s-api-ip.yaml lands and where
# patches/k8s-api-ip.yaml.template lives. It NEVER moves; only the render dir
# switches between the plain and the -ghcr overlay.
ACTIVE_MINIKUBE_KUSTOMIZE_DIR="${ACTIVE_MINIKUBE_DEPLOY_DIR}/overlays/minikube"
if [ "$IMAGE_SOURCE" = ghcr ]; then
if [ "$SKIP_UIS" = true ]; then
ACTIVE_MINIKUBE_RENDER_DIR="${ACTIVE_MINIKUBE_DEPLOY_DIR}/overlays/minikube-no-uis-ghcr"
else
ACTIVE_MINIKUBE_RENDER_DIR="${ACTIVE_MINIKUBE_DEPLOY_DIR}/overlays/minikube-ghcr"
fi
else
if [ "$SKIP_UIS" = true ]; then
ACTIVE_MINIKUBE_RENDER_DIR="${ACTIVE_MINIKUBE_DEPLOY_DIR}/overlays/minikube-no-uis"
else
ACTIVE_MINIKUBE_RENDER_DIR="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}"
fi
fi
# The effective tag and where it came from, for the banner and for every error
# message that has to tell an operator what to change.
#
# Reading the committed pin here ignored the tag the cluster actually holds: on
# the documented `MINIKUBE_IMAGE_TAG=latest make minikube-setup` bootstrap, any
# LATER run (which has no such variable) reported and acted on v0.6.0 against a
# cluster running :latest. image_mode_ghcr_tag applies override -> recorded ->
# pin, the same precedence pull-images.sh and build-images.sh --verify-only
# use; the `= ghcr` branch above is this file's mode gate, which is why the
# ungated ghcr resolver is the right one to call.
#
# The pin is read from PROJECT_DIR, not ACTIVE_MINIKUBE_DEPLOY_DIR: the two
# differ only when MINIKUBE_IMAGE_TAG rewrote a copy, and that override already
# wins ahead of any pin.
EFFECTIVE_IMAGE_TAG=""
TAG_ORIGIN=""
if [ "$IMAGE_SOURCE" = ghcr ]; then
EFFECTIVE_IMAGE_TAG="$(image_mode_ghcr_tag "$PROJECT_DIR")" || exit 1
TAG_ORIGIN="$(image_mode_tag_origin "$PROJECT_DIR")" || exit 1
# Never continue on an empty tag: every ref built from it would be tagless,
# and the banner would confidently print nothing at all.
if [ -z "$EFFECTIVE_IMAGE_TAG" ] || [ -z "$TAG_ORIGIN" ]; then
err "could not resolve the ghcr image tag (tag='${EFFECTIVE_IMAGE_TAG}' origin='${TAG_ORIGIN}')"
err "Set MINIKUBE_IMAGE_TAG=<tag>, or fix deploy/components/ghcr-images/kustomization.yaml."
exit 1
fi
fi
print_image_source_banner() {
echo ""
echo -e "${BOLD}================================================================${NC}"
if [ "$IMAGE_SOURCE" = ghcr ]; then
echo -e "${BOLD} Images: PULLED from ghcr.io/evenfire-ai${NC}"
echo -e " tag: ${EFFECTIVE_IMAGE_TAG}"
echo -e " origin: ${TAG_ORIGIN}"
echo -e " build locally instead with: ${BOLD}make minikube-setup-local${NC}"
else
echo -e "${BOLD} Images: BUILT LOCALLY from source (IMAGE_SOURCE=local)${NC}"
echo -e " This is the slow path. The default pulls published images."
fi
echo -e " render: ${ACTIVE_MINIKUBE_RENDER_DIR#${ACTIVE_MINIKUBE_DEPLOY_DIR}/}"
echo -e "${BOLD}================================================================${NC}"
}
print_image_source_banner
# Test seam: everything above resolves configuration only, with no cluster or
# network call. `MINIKUBE_FULL_SETUP_CONFIG_ONLY=true` stops here so
# scripts/tests/test-minikube-full-setup.sh can read the resolved values
# without a cluster. It must stay immediately after the resolution block and
# before the first minikube call.
if [ "${MINIKUBE_FULL_SETUP_CONFIG_ONLY:-false}" = "true" ]; then
return 0 2>/dev/null || exit 0
fi
MINIKUBE_START_SCRIPT="${MINIKUBE_START_SCRIPT:-${SCRIPT_DIR}/start.sh}"
MINIKUBE_SETUP_DEADLINE_RUNNER="${MINIKUBE_SETUP_DEADLINE_RUNNER:-${SCRIPT_DIR}/run-with-deadline.mjs}"
MINIKUBE_SETUP_STATUS_TIMEOUT_SECONDS="${MINIKUBE_SETUP_STATUS_TIMEOUT_SECONDS:-60}"
MINIKUBE_SETUP_START_TIMEOUT_SECONDS="${MINIKUBE_SETUP_START_TIMEOUT_SECONDS:-900}"
MINIKUBE_SETUP_VALIDATE_TIMEOUT_SECONDS="${MINIKUBE_SETUP_VALIDATE_TIMEOUT_SECONDS:-300}"
MINIKUBE_SETUP_DELETE_TIMEOUT_SECONDS="${MINIKUBE_SETUP_DELETE_TIMEOUT_SECONDS:-300}"
MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS="${MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS:-30}"
validate_setup_deadline() {
local name="$1" value="$2" maximum="$3"
if ! [[ "${value}" =~ ^[1-9][0-9]*$ ]] || (( 10#${value} > maximum )); then
err "${name} must be an integer from 1 to ${maximum}"
return 1
fi
}
validate_setup_deadline MINIKUBE_SETUP_STATUS_TIMEOUT_SECONDS "${MINIKUBE_SETUP_STATUS_TIMEOUT_SECONDS}" 300
validate_setup_deadline MINIKUBE_SETUP_START_TIMEOUT_SECONDS "${MINIKUBE_SETUP_START_TIMEOUT_SECONDS}" 1800
validate_setup_deadline MINIKUBE_SETUP_VALIDATE_TIMEOUT_SECONDS "${MINIKUBE_SETUP_VALIDATE_TIMEOUT_SECONDS}" 900
validate_setup_deadline MINIKUBE_SETUP_DELETE_TIMEOUT_SECONDS "${MINIKUBE_SETUP_DELETE_TIMEOUT_SECONDS}" 900
validate_setup_deadline MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS "${MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS}" 300
[[ -f "${MINIKUBE_SETUP_DEADLINE_RUNNER}" ]] || {
err "bounded runtime helper is missing: ${MINIKUBE_SETUP_DEADLINE_RUNNER}"
exit 1
}
# Acquire the branch-owned mutation lease before the first profile status probe.
# A standalone full setup may classify a partial profile as destructive and
# delete it, so ownership and serialization must be established before any
# status/start/delete operation can touch Minikube.
SETUP_LOCK_CLEANUP_DONE=false
K8S_API_POLICY_TMP_DIR=""
cleanup_setup_lock() {
local status="${1:-$?}" cleanup_status=0
if [ "$SETUP_LOCK_CLEANUP_DONE" = true ]; then
return "$status"
fi
SETUP_LOCK_CLEANUP_DONE=true
trap - EXIT
trap '' INT TERM
if [ -n "${K8S_API_POLICY_TMP_DIR:-}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ]; then
rm -rf -- "${K8S_API_POLICY_TMP_DIR}" || cleanup_status=1
fi
t2_lock_release "$status" || cleanup_status=$?
if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
status="$cleanup_status"
fi
return "$status"
}
handle_setup_signal() {
local signal="$1" status
case "$signal" in
INT) status=130 ;;
TERM) status=143 ;;
*) status=1 ;;
esac
cleanup_setup_lock "$status" || status=$?
exit "$status"
}
handle_setup_exit() {
local status=$?
cleanup_setup_lock "$status" || status=$?
exit "$status"
}
trap handle_setup_exit EXIT
trap 'handle_setup_signal INT' INT
trap 'handle_setup_signal TERM' TERM
t2_require_commands
t2_repo_metadata
t2_profile_scope
t2_mutation_lock
export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_SKIP_LOCK T2_LOCK_TOKEN
run_setup_with_deadline() {
local label="$1" timeout_seconds="$2"
shift 2
node "${MINIKUBE_SETUP_DEADLINE_RUNNER}" \
--timeout-seconds "${timeout_seconds}" \
--heartbeat-seconds "${MINIKUBE_DOCKER_HEARTBEAT_SECONDS:-20}" \
--kill-grace-seconds "${MINIKUBE_DOCKER_KILL_GRACE_SECONDS:-5}" \
--label "${label}" -- "$@"
}
minikube_status_snapshot() {
local output status=0
output="$(run_setup_with_deadline minikube-setup-status \
"${MINIKUBE_SETUP_STATUS_TIMEOUT_SECONDS}" \
minikube -p "$PROFILE" status)" || status=$?
# `minikube status` legitimately returns a small non-zero code for an absent
# or partially started profile while still printing the state we classify
# below. Runner timeout/signal/spawn failures are operational failures and
# must not be reinterpreted as permission to start or recreate a profile.
if (( status >= 124 )); then
err "Minikube status probe failed before profile state could be classified (status ${status})"
return "$status"
fi
printf '%s' "$output"
}
minikube_status_is_healthy() {
local status="${1:-}"
[[ "$status" == *"host: Running"* ]] &&
[[ "$status" == *"kubelet: Running"* ]] &&
[[ "$status" == *"apiserver: Running"* ]] &&
[[ "$status" != *"Stopped"* ]]
}
minikube_status_is_broken_profile() {
local status="${1:-}"
[[ "$status" == *"host: Running"* ]] && {
[[ "$status" == *"kubelet: Stopped"* ]] ||
[[ "$status" == *"apiserver: Stopped"* ]]
}
}
recreate_broken_minikube_profile() {
if [[ "${MINIKUBE_RECREATE_PROFILE:-false}" != "true" || "${CONFIRM_PROFILE:-}" != "${PROFILE}" ]]; then
err "Refusing to delete minikube profile '${PROFILE}' without explicit confirmation."
echo " Re-run only if intended with: MINIKUBE_RECREATE_PROFILE=true CONFIRM_PROFILE=${PROFILE}" >&2
exit 1
fi
warn "Minikube profile '${PROFILE}' is partially started (host up, control plane down). Recreating it from scratch..."
if ! run_setup_with_deadline minikube-setup-delete \
"${MINIKUBE_SETUP_DELETE_TIMEOUT_SECONDS}" \
minikube delete -p "$PROFILE" >/dev/null; then
err "Failed to delete explicitly confirmed broken profile '${PROFILE}' within the deadline"
exit 1
fi
ok "Removed broken '${PROFILE}' profile"
}
start_minikube_cluster() {
log "Starting minikube cluster '${PROFILE}'..."
# Delegates to start.sh which honors MINIKUBE_MEMORY / MINIKUBE_CPUS /
# MINIKUBE_MULTI_NODE / MINIKUBE_NODES. See scripts/minikube/start.sh.
MINIKUBE_PROFILE="${PROFILE}" \
MINIKUBE_MULTI_NODE="${MINIKUBE_MULTI_NODE:-false}" \
MINIKUBE_NODES="${MINIKUBE_NODES:-}" \
run_setup_with_deadline minikube-setup-start \
"${MINIKUBE_SETUP_START_TIMEOUT_SECONDS}" \
"${MINIKUBE_START_SCRIPT}"
ok "Minikube cluster '${PROFILE}' started"
}
validate_minikube_cluster() {
log "Validating minikube cluster '${PROFILE}'..."
MINIKUBE_PROFILE="${PROFILE}" \
MINIKUBE_MULTI_NODE="${MINIKUBE_MULTI_NODE:-false}" \
MINIKUBE_NODES="${MINIKUBE_NODES:-}" \
run_setup_with_deadline minikube-setup-validate \
"${MINIKUBE_SETUP_VALIDATE_TIMEOUT_SECONDS}" \
"${MINIKUBE_START_SCRIPT}" --validate-only
ok "Minikube cluster '${PROFILE}' validated"
}
maybe_exit_after_cluster_step() {
if [ "${MINIKUBE_SETUP_EXIT_AFTER_CLUSTER:-false}" = "true" ]; then
if [ "${T2_SETUP_HANDOFF_REQUIRED}" = true ]; then
err "T2 strict setup cannot stop after cluster verification"
return 1
fi
log "MINIKUBE_SETUP_EXIT_AFTER_CLUSTER=true — stopping after cluster verification"
exit 0
fi
}
postgres_has_invalid_checkpoint() {
local pods pod logs
pods="$($KC get pods -n control-plane -l app=control-postgres -o name)" || return 1
pod="${pods%%$'\n'*}"
[ -n "$pod" ] || return 1
logs="$($KC logs "$pod" -n control-plane --tail=50)" || return 1
grep -q "invalid checkpoint record" <<<"$logs"
}
ensure_control_postgres_ready() {
if $KC rollout status deployment/control-postgres -n control-plane --timeout=180s >/dev/null 2>&1; then
return 0
fi
if ! postgres_has_invalid_checkpoint; then
err "control-plane/control-postgres did not become Ready and no supported WAL recovery signature was found"
return 1
fi
err "Postgres reports an invalid checkpoint; automatic destructive recovery is disabled"
err "Re-run with --reset-db and CONTROL_DB_RESET_PVC_UID=<exact approved UID>"
return 1
}
# A previous full-setup may have been interrupted after it fenced one or more
# database writers. Read and validate that durable journal before the first
# namespace/CRD/Secret mutation. If the journal is in an active recovery phase,
# close all four writers again before the later recovery code resumes. The
# journal is deliberately not advanced here: if this guard is interrupted, the
# old phase remains conservative and the next run repeats the fence.
guard_interrupted_writer_deployment() {
local deployment="$1" selector="$2" desired pods
$KC -n control-plane scale "deployment/${deployment}" --replicas=0 >/dev/null || return 1
desired="$($KC -n control-plane get "deployment/${deployment}" \
-o 'jsonpath={.spec.replicas}')" || return 1
[ "$desired" = 0 ] || { err "${deployment} did not converge to zero desired replicas before setup mutation"; return 1; }
pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1
[ -z "$pods" ] || $KC -n control-plane wait --for=delete pod \
-l "$selector" --timeout=180s >/dev/null || return 1
pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1
[ -z "$pods" ] || { err "${deployment} pods remain after the pre-mutation fence"; return 1; }
}
guard_interrupted_writer_recovery() {
local state_helper="${SCRIPT_DIR}/writer-recovery-state.py"
local state_file="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery/${PROFILE}.json"
local state_output state_status=0 phase hcc_replicas workflow_replicas trace_replicas control_api_replicas state_head
[[ -f "${state_file}" ]] || return 0
[[ -f "${state_helper}" ]] || {
err "Durable writer-recovery state exists but its validator is missing; refusing profile mutation"
return 1
}
state_output="$(python3 "${state_helper}" read \
--path "${state_file}" \
--profile "${PROFILE}" \
--context "${T2_CONTEXT}" \
--worktree "${PROJECT_DIR}" \
--branch "${T2_BRANCH}" \
--head "${T2_HEAD}" \
--include-head)" || state_status=$?
if [ "${state_status}" -ne 0 ]; then
err "Unable to validate durable writer-recovery state before cluster mutation (status ${state_status})"
return 1
fi
[[ "${state_output}" == NONE ]] && return 0
IFS='|' read -r phase hcc_replicas workflow_replicas trace_replicas \
control_api_replicas state_head <<<"${state_output}"
[[ "${phase}" == planned ]] && return 0
[[ "${phase}" =~ ^(hcc-fencing|hcc-fenced|workflow-fencing|workflow-fenced|trace-fencing|trace-fenced|api-fencing|api-fenced|policy-ready|roles-ready|api-restoring|api-restored|overlay-applying|overlay-applied)$ ]] || {
err "Durable writer-recovery state has an unsupported phase before cluster mutation: ${phase}"
return 1
}
[[ "${hcc_replicas}" =~ ^[0-9]+$ ]] &&
[[ "${workflow_replicas}" =~ ^[0-9]+$ ]] &&
[[ "${trace_replicas}" =~ ^[0-9]+$ ]] &&
[[ "${control_api_replicas}" =~ ^[1-9][0-9]*$ ]] || {
err "Durable writer-recovery state has invalid replica counts; refusing profile mutation"
return 1
}
log "Resuming interrupted writer recovery from phase ${phase} (recorded HEAD ${state_head}) — fencing all writers before setup mutations"
guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller' || return 1
guard_interrupted_writer_deployment workflow-recipes 'app=workflow-recipes' || return 1
guard_interrupted_writer_deployment trace-maintenance-worker 'app=trace-maintenance-worker' || return 1
guard_interrupted_writer_deployment control-api 'app=control-api,!clerum.io/component' || return 1
}
# ======================================================================
# Step 1: Preconditions
# ======================================================================
step_header 1 $TOTAL_STEPS "Preconditions"
# Check Docker through the same isolated, bounded local-endpoint contract used
# by T1 and image builds. This avoids paying setup cost after an ambient Docker
# context or credential helper has stalled.
if ! MINIKUBE_DOCKER_INFO_TIMEOUT_SECONDS="${MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS}" \
MINIKUBE_DOCKER_START_PROBE_TIMEOUT_SECONDS="${MINIKUBE_SETUP_DOCKER_TIMEOUT_SECONDS}" \
bash "${SCRIPT_DIR}/docker-cli-env.sh" --check-info >/dev/null; then
err "Docker is not running. Please start Docker Desktop first."
exit 1
fi
ok "Docker is running"
# Check minikube is installed
if ! command -v minikube &>/dev/null; then
err "minikube is not installed. Install with: brew install minikube"
exit 1
fi
ok "minikube is installed"
# Check kubectl is installed
if ! command -v kubectl &>/dev/null; then
err "kubectl is not installed. Install with: brew install kubectl"
exit 1
fi
ok "kubectl is installed"
# Check python3 is available (needed for JWT sync)
if ! command -v python3 &>/dev/null; then
err "python3 is not installed (required for JWT key sync)."
exit 1
fi
ok "python3 is available"
# Check ruby is available (run-control-api-db-migration.sh renders the kustomize
# overlay with `ruby -ryaml -rjson`; ruby ships with macOS but not every Linux).
if ! command -v ruby &>/dev/null; then
err "ruby is not installed (required to render the control-api DB migration overlay). Install with: brew install ruby (macOS) / apt-get install ruby (Debian/Ubuntu)."
exit 1
fi
ok "ruby is available"
# Fail here, not at Step 10. Image builds dominate a 5-10 min first run
# (README.md:178); aborting after them for a missing .env line is hostile.
if [ "$SEED_PROFILE" = "minimal" ] && [ "$DEFER_BOOTSTRAP_SEED" != true ] && [ -z "${ADMIN_PASSWORD:-}" ]; then
err "ADMIN_PASSWORD is not set."
err ""
err " This is your Control UI admin password AND your Desktop App login."
err " Set it in .env at the project root:"
err ""
err " ADMIN_PASSWORD=<choose-a-password>"
err ""
err " No default password ships with Evenfire. For the E2E fixture profile"
err " (which pins a known test password), run: make minikube-setup-e2e"
exit 1
fi
# Reject a too-short/too-long password here rather than at seed time (Step 10).
# control-api enforces 8-256 chars (control-api/src/routes/admin/auth.ts); if we
# defer, the bootstrap POST fails with "password must be between 8 and 256
# characters" only AFTER image builds + deploy, and the admin account may still
# hold the publicly-known default. Keep these bounds in sync with auth.ts. Scope
# to the minimal profile: the E2E profile may use its local-only fallback when
# no canonical or explicit admin credential is configured.
if [ "$SEED_PROFILE" = "minimal" ] && [ "$DEFER_BOOTSTRAP_SEED" != true ] && [ -n "${ADMIN_PASSWORD:-}" ]; then
pw_len=${#ADMIN_PASSWORD}
if [ "$pw_len" -lt 8 ] || [ "$pw_len" -gt 256 ]; then
err "ADMIN_PASSWORD must be between 8 and 256 characters (got ${pw_len})."
err ""
err " Set a longer value in .env at the project root:"
err ""
err " ADMIN_PASSWORD=<choose-a-password-8-256-chars>"
exit 1
fi
fi
if [ "$DEFER_BOOTSTRAP_SEED" = true ]; then
ok "Bootstrap admin/user seed deferred; the first-run browser setup owns /api/v1/admin/auth/setup"
elif [ -n "${ADMIN_PASSWORD:-}" ]; then
ok "ADMIN_PASSWORD resolved from the canonical environment contract"
else
ok "No admin password configured; the E2E-only local fallback will be used at seed time"
fi
# ======================================================================
# Step 2: Cluster
# ======================================================================
step_header 2 $TOTAL_STEPS "Cluster"
CLUSTER_STATUS="$(minikube_status_snapshot)"
if minikube_status_is_healthy "$CLUSTER_STATUS"; then
ok "Minikube cluster '${PROFILE}' is already running"
validate_minikube_cluster
else
if minikube_status_is_broken_profile "$CLUSTER_STATUS"; then
recreate_broken_minikube_profile
fi
start_minikube_cluster
fi
# Verify cluster is reachable
if ! $KC cluster-info &>/dev/null; then
CLUSTER_STATUS="$(minikube_status_snapshot)"
if minikube_status_is_broken_profile "$CLUSTER_STATUS"; then
warn "Cluster '${PROFILE}' is still unhealthy after start. Recreating the profile and retrying once..."
recreate_broken_minikube_profile
start_minikube_cluster
fi
if ! $KC cluster-info &>/dev/null; then
err "Cluster '${PROFILE}' not reachable after start."
exit 1
fi
fi
ok "Cluster '${PROFILE}' is reachable"
maybe_exit_after_cluster_step
# Validate the explicit context and exact Minikube profile identity before an
# interrupted-recovery journal is allowed to scale any writer. Reachability
# alone is not authority: a kubeconfig can name another local cluster.
t2_profile_status
t2_context_check
t2_profile_context_identity_check
# This is intentionally after the profile becomes reachable but before the
# first setup apply. A stopped profile may be started safely; an active journal
# must be fenced before any workload/configuration mutation.
guard_interrupted_writer_recovery || exit 1
# ======================================================================
# Step 3: Namespaces + CRDs
# ======================================================================
step_header 3 $TOTAL_STEPS "Namespaces + CRDs"
log "Creating namespaces..."
$KC apply -f "${PROJECT_DIR}/deploy/base/namespaces.yaml"
ns_count=$(grep -c '^kind: Namespace' "${PROJECT_DIR}/deploy/base/namespaces.yaml" || true)
ok "${ns_count} namespaces created/verified"
log "Installing CRDs..."
$KC apply -f "${PROJECT_DIR}/charts/clerum-crds/crds/"
ok "All CRDs installed"
# ======================================================================
# Step 4: Secrets
# ======================================================================
step_header 4 $TOTAL_STEPS "Secrets"
# 4a. JWT signing keys
log "Generating JWT signing keys..."
if [ "$FORCE_KEYS" = true ]; then
FORCE_REGEN=true bash "${SCRIPT_DIR}/generate-keys.sh" --apply
ok "JWT signing keys regenerated (--force-keys)"
else
bash "${SCRIPT_DIR}/generate-keys.sh" --apply
ok "JWT signing keys (existing or newly generated)"
fi
# 4b. Inter-service tokens
# The `inter-service-tokens.yaml` overlay only seeds one Secret in control-plane
# with DEV placeholder values. The real service-to-service auth (external-rest-api,
# rpc-proxy, workflow-recipes → control-api) lives in 4 *different* Secrets that
# `apply-inter-service-tokens.sh` patches via `kubectl patch --type=merge`:
# - control-plane/control-api-internal-tokens
# - control-plane/workflow-recipes-secrets
# - profiles/external-rest-api-secrets
# - rpc-proxy/rpc-proxy-secrets
# Without this call, external-rest-api crashes with:
# "Missing required environment variable: EXTERNAL_REST_API_CONTROL_API_SERVICE_TOKEN"
# The script is idempotent: existing tokens are preserved, missing ones are
# generated via `openssl rand -hex 32`.
log "Applying inter-service tokens (overlay + real Secret patches)..."
$KC apply -f "${PROJECT_DIR}/deploy/overlays/minikube/secrets/inter-service-tokens.yaml"
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-inter-service-tokens.sh"
ok "Inter-service tokens applied (4 Secrets patched)"
# 4c. Channel credentials — retired in #273.
# The static `clerum-channel-reader-credentials` Secret used to be applied here
# (from deploy/overlays/minikube/secrets/channel-credentials.yaml) and then
# patched with .env overrides. With the static `clerum-channel-reader`
# Deployment retired, channel credentials are per-Host and written via
# control-api's `/admin/channel-secrets` endpoint from the Control UI's
# Channel credentials panel. No bootstrap-time Secret needed.
#
# For minikube developers who want to skip the UI step: after `make minikube-setup`
# finishes, send a PUT to /api/v1/admin/channel-secrets with the bot token JSON
# (one POST per Host). See docs/deploy/minikube.md for the curl-based recipe.
# 4d. LLM API keys (override from .env if present)
log "Applying LLM API keys..."
$KC apply -f "${PROJECT_DIR}/deploy/overlays/minikube/secrets/llm-api-keys.yaml"
# If .env has real keys, patch the secret with actual values
if [ -n "${OPENAI_API_KEY:-}" ] || [ -n "${CLAUDE_API_KEY:-}" ] || [ -n "${ZAI_API_KEY:-}" ] || [ -n "${BAILIAN_API_KEY:-}" ]; then
log "Patching LLM API keys with .env overrides..."
$KC create secret generic chatllm-api-keys \
--namespace=mcp-host \
--from-literal=openai-api-key="${OPENAI_API_KEY:-sk-test-placeholder-openai-key-00000000000000000000}" \
--from-literal=claude-api-key="${CLAUDE_API_KEY:-sk-ant-api03-test-placeholder-claude-key-000000000000000000000000000000000000000000000000000000}" \
--from-literal=zai-api-key="${ZAI_API_KEY:-zai-test-placeholder-zai-key-00000000000000000000}" \
--from-literal=bailian-api-key="${BAILIAN_API_KEY:-sk-test-placeholder-bailian-key-00000000000000000000}" \
--dry-run=client -o yaml | $KC apply -f -
ok "LLM API keys patched with .env values"
# Post-refactor (WRC Secret Broker): the 4 per-provider Secrets in control-plane
# (clerum-{openai,claude,zai,bailian}-api-key) have been removed. WRC now reads
# directly from chatllm-api-keys in mcp-host (the block above) — single source
# of truth. See .ralph/plans/rosy-tinkering-llama.md for full rationale.
fi
ok "LLM API keys applied"
# 4e. Search API keys (if file exists)
if [ -f "${PROJECT_DIR}/deploy/overlays/minikube/secrets/search-api-keys.yaml" ]; then
$KC apply -f "${PROJECT_DIR}/deploy/overlays/minikube/secrets/search-api-keys.yaml"
ok "Search API keys applied"
fi
# 4f. MCP server secrets (MongoDB, Airtable)
# Preferred source: gitignored ${svc}/secret.yaml with real credentials.
# Fallback: create dev-grade placeholders so HCC can reconcile the McpServer
# CRDs (envSecret references must resolve or the Deployment is never created).
# Real credentials can be injected later via AIRTABLE_API_KEY / MONGODB_CONNECTION_STRING
# in .env and re-running setup, or via scripts/create-k8s-secrets.sh.
#
# Gated with SEED_PROFILE=e2e together with the McpServer instances themselves
# (deploy/overlays/minikube/instances-e2e/, Step 6e below). They MUST move as
# a pair: validate_mcpserver_secrets (below) aborts the whole setup if any
# McpServer.envSecret is unresolvable, so a server applied without its secret
# is a hard failure — a secret applied without its server is merely unused.
log "Applying MCP server secrets..."
$KC create namespace mcp-server --dry-run=client -o yaml | $KC apply -f - >/dev/null
if [ "$SEED_PROFILE" = "e2e" ]; then
if [ -f "${PROJECT_DIR}/mcp-servers/airtable/secret.yaml" ]; then
$KC apply -f "${PROJECT_DIR}/mcp-servers/airtable/secret.yaml"
ok "mcp-airtable-credentials from mcp-servers/airtable/secret.yaml"
else
$KC create secret generic mcp-airtable-credentials \
--namespace=mcp-server \
--from-literal=api-key="${AIRTABLE_API_KEY:-placeholder-airtable-api-key-for-e2e}" \
--dry-run=client -o yaml | $KC apply -f - >/dev/null
ok "mcp-airtable-credentials (dev placeholder; override via AIRTABLE_API_KEY or secret.yaml)"
fi
if [ -f "${PROJECT_DIR}/mcp-servers/mongodb/secret.yaml" ]; then
$KC apply -f "${PROJECT_DIR}/mcp-servers/mongodb/secret.yaml"
ok "mcp-mongodb-credentials from mcp-servers/mongodb/secret.yaml"
else
$KC create secret generic mcp-mongodb-credentials \
--namespace=mcp-server \
--from-literal=connection-string="${MONGODB_CONNECTION_STRING:-mongodb://placeholder:placeholder@localhost:27017/placeholder}" \
--dry-run=client -o yaml | $KC apply -f - >/dev/null
ok "mcp-mongodb-credentials (dev placeholder; override via MONGODB_CONNECTION_STRING or secret.yaml)"
fi
else
log "Skipping MCP server secrets (SEED_PROFILE=minimal) — no demo McpServers to back."
fi
# 4g. JWT sync — copy RPC public key to mcp-host-config
# On fresh clusters, mcp-host-config doesn't exist yet (created in Step 6).
# Only sync if BOTH the secret AND the configmap exist; otherwise defer to post-deploy sync.
if $KC get secret rpc-proxy-secrets -n rpc-proxy &>/dev/null && \
$KC get configmap mcp-host-config -n mcp-host &>/dev/null; then
log "Syncing JWT public key to mcp-host-config..."
$KC get secret rpc-proxy-secrets -n rpc-proxy \
-o jsonpath='{.data.RPC_PROXY_JWT_PUBLIC_KEY}' | base64 -d \
| python3 -c "import sys,subprocess,json; \
key=sys.stdin.read(); \
patch=json.dumps([{'op':'replace','path':'/data/CLERUM_AUTH_JWT_PUBLIC_KEY','value':key}]); \
subprocess.run(['kubectl','--context=${PROFILE}','patch','configmap','mcp-host-config','-n','mcp-host','--type=json','-p',patch],check=True)"
ok "JWT public key synced to mcp-host-config"
else
log "Deferring JWT sync to post-deploy (mcp-host-config not yet created)"
fi
# ======================================================================
# Step 5: Build images
# ======================================================================
step_header 5 $TOTAL_STEPS "Build Images"
if [ "$SKIP_BUILD" = true ]; then
# Even with --skip-build, warn if local code is newer than the images the
# cluster will run. The ADVICE has to branch on the mode: "rebuild" is wrong
# in ghcr mode, where nothing was built and rebuilding is precisely what the
# default path exists to avoid.
LAST_BUILD_MARKER="deploy/minikube/.image-manifest.json"
if [ -f "$LAST_BUILD_MARKER" ]; then
NEWEST_SRC=$(find \
mcp-host/src \
workflow-recipes/src \
packages/workflow-sdk \
tests/e2e/fixtures/custom-workflow-coordinator \
control-api/src \
control-ui/components \
control-ui/lib \
-type f -newer "$LAST_BUILD_MARKER" 2>/dev/null | head -1 || true)
if [ -n "$NEWEST_SRC" ]; then
if [ "$IMAGE_SOURCE" = ghcr ]; then
warn "Local source is newer than the pulled images (e.g., $NEWEST_SRC)."
warn "This cluster runs RELEASE images (${EFFECTIVE_IMAGE_TAG}); your local edits are NOT in them."
warn "To test local edits: 'make minikube-pre-gate-sync' shadow-builds only the changed"
warn "services over the release tag, or 'make minikube-setup-local' builds everything."
else
warn "Source files changed since last image build (e.g., $NEWEST_SRC)."
warn "Images may be STALE. Run without --skip-build to rebuild."
fi
else
log "Skipping image acquisition (--skip-build). No source changes detected."
fi
else
if [ "$IMAGE_SOURCE" = ghcr ]; then
warn "No image manifest found — images may never have been pulled."
warn "Run without --skip-build, or 'make minikube-pull-images', if pods show ImagePullBackOff."
else
warn "No image manifest found — images may never have been built."
warn "Run without --skip-build if pods show ImagePullBackOff or model-config-failed."
fi
fi
elif [ "$IMAGE_SOURCE" = ghcr ]; then
# ORDER MATTERS. build-images.sh regenerates
# deploy/minikube/.image-manifest.json on EVERY invocation, including
# --only= runs (its header comment claims otherwise; the code at the bottom
# of the script is unconditional). So every build-images.sh call has to come
# BEFORE the pull, or it clobbers the puller's manifest -- the one file the
# --skip-build staleness check reads -- with a mostly-NOT_BUILT map. (The
# `imageSource` key survives that clobber either way: an --only= run carries
# the recorded value forward rather than claiming the cluster is local.)
#
# No clerum image build runs on this path, so nothing else pulls the public
# third-party images (postgres, redis, nginx, ...) into the daemon.
log "Loading public base images (postgres, redis, nginx, ...) into minikube..."
bash "${SCRIPT_DIR}/build-images.sh" --public-only
ok "Public base images loaded"
# No local build runs on this path any more. The published:false images
# (workflow-custom-sdk-e2e, workflow-plugin-sdk-e2e) have no ghcr
# counterpart, and both are now built only by the path that actually needs
# them: `make minikube-setup-e2e`.
log "Pulling published images (${EFFECTIVE_IMAGE_TAG}) into minikube..."
# --skip-uis is forwarded for the same reason the local build path passes it:
# the -no-uis-ghcr overlay DELETES the control-ui and profile-ui Deployments,
# so pulling them spends ~470 MiB of transfer and disk on images no pod will
# ever reference.
PULL_IMAGE_ARGS=()
if [ "$SKIP_UIS" = true ]; then
PULL_IMAGE_ARGS+=(--skip-uis)
fi
# `${PULL_IMAGE_ARGS[@]+"${PULL_IMAGE_ARGS[@]}"}`: expanding an EMPTY array as
# `"${arr[@]}"` is an unbound-variable abort under `set -u` in bash 3.2, which
# is what /bin/bash on macOS still is.
#
# `${MINIKUBE_MULTI_NODE:-false}`, not `$MINIKUBE_MULTI_NODE`: this script
# never assigns that variable (every other use site defaults it inline), so
# the bare form is an unbound-variable abort under `set -u` on the default
# single-node path.
MINIKUBE_PROFILE="$PROFILE" MINIKUBE_MULTI_NODE="${MINIKUBE_MULTI_NODE:-false}" \
bash "${SCRIPT_DIR}/pull-images.sh" ${PULL_IMAGE_ARGS[@]+"${PULL_IMAGE_ARGS[@]}"}
ok "All published images pulled"
else
log "Building and loading images into minikube..."
BUILD_IMAGE_ARGS=(--skip-public)
if [ "$SKIP_UIS" = true ]; then
BUILD_IMAGE_ARGS+=(--skip-uis)
fi
bash "${SCRIPT_DIR}/build-images.sh" "${BUILD_IMAGE_ARGS[@]}"
ok "All images built and loaded"
fi
# ======================================================================
# Step 6: Deploy
# ======================================================================
step_header 6 $TOTAL_STEPS "Deploy"
# 6a. Optional DB reset
if [ "$RESET_DB" = true ] && [ -z "$CONTROL_DB_RESET_PVC_UID" ]; then
# Never-bootstrapped auto-detect: with no control-postgres PVC AND no reset
# recovery state there is nothing a storage reset could destroy or resume,
# and the reset preflight (ready GFS secrets, running gfsc deployments)
# cannot succeed before the first deploy. Continue on the same path a
# REUSE_DB=true bootstrap takes. Any other combination keeps the hard-fail.
FRESH_PVC_UID="$($KC -n control-plane get pvc control-postgres-data \
-o 'jsonpath={.metadata.uid}' --ignore-not-found)"
FRESH_RESET_STATE="$($KC -n control-plane get configmap control-db-reset-state \
-o name --ignore-not-found)"
if [ -z "$FRESH_PVC_UID" ] && [ -z "$FRESH_RESET_STATE" ]; then
log "Fresh cluster: control DB was never bootstrapped; nothing to reset — continuing without a storage reset"
RESET_DB=false
fi
fi
if [ "$RESET_DB" = true ]; then
[ -n "$CONTROL_DB_RESET_PVC_UID" ] \
|| { err "CONTROL_DB_RESET_PVC_UID is required with --reset-db (pass the current control-postgres PVC UID, 'none' for a cluster without that PVC, or skip the reset with REUSE_DB=true/--keep-db). A cluster with no control-postgres PVC and no reset state is detected automatically and skips the reset."; exit 1; }
if [ "$CONTROL_DB_RESET_PVC_UID" = none ]; then
RESET_STORAGE_ARGS=(--expect-no-pvc)
else
RESET_STORAGE_ARGS=(--expected-pvc-uid "$CONTROL_DB_RESET_PVC_UID")
fi
case "$CONTROL_DB_RESET_RESUME" in
true|1|yes) RESET_STORAGE_ARGS+=(--resume) ;;
esac
log "Resetting postgres database (--reset-db)..."
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/reset-control-db-storage.sh" "${RESET_STORAGE_ARGS[@]}"
RESET_REPLICA_STATE="$($KC -n control-plane get configmap control-db-reset-state -o \
'jsonpath={.data.hccReplicas}{"|"}{.data.workflowReplicas}{"|"}{.data.traceReplicas}{"|"}{.data.writerReplicas}{"|"}{.data.readerReplicas}{"|"}{.data.controlApiReplicas}')"
IFS='|' read -r RESET_HCC_REPLICAS RESET_WORKFLOW_REPLICAS RESET_TRACE_REPLICAS \
RESET_WRITER_REPLICAS RESET_READER_REPLICAS RESET_CONTROL_API_REPLICAS <<<"$RESET_REPLICA_STATE"
[[ "$RESET_HCC_REPLICAS" =~ ^[0-9]+$ ]] \
&& [[ "$RESET_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] \
&& [[ "$RESET_TRACE_REPLICAS" =~ ^[0-9]+$ ]] \
&& [[ "$RESET_WRITER_REPLICAS" =~ ^[1-9][0-9]*$ ]] \
&& [[ "$RESET_READER_REPLICAS" =~ ^[1-9][0-9]*$ ]] \
&& [[ "$RESET_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] \
|| { err "Reset replica recovery state is invalid"; exit 1; }
ok "Postgres PVCs deleted"
fi
# 6b. Deploy via kustomize
log "Refreshing minikube K8s API endpoint CIDRs..."
CONTEXT="${PROFILE}" OVERLAY_DIR="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}" "${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"
# patches/k8s-api-ip.yaml is GENERATED and gitignored -- overlays/minikube
# commits only the .template -- and overlays/minikube-ghcr renders ../minikube,
# which patches with it.
#
# With MINIKUBE_IMAGE_TAG set, ACTIVE_MINIKUBE_KUSTOMIZE_DIR is the mktemp COPY
# (apply_image_tag_override), so the line above writes the patch THERE and the
# working tree never gets one. Every later consumer renders from the working
# tree, or from image-mode.sh's own fresh copy of it: the next
# `make minikube-pre-gate-sync` renders the control-api migration overlay
# BEFORE `make minikube-deploy-all` regenerates the patch, and kustomize dies
# with an evalsymlink error naming a file nothing ever wrote. Generate it in the
# working tree too whenever the active dir is not the working tree.
PROJECT_MINIKUBE_KUSTOMIZE_DIR="${PROJECT_DIR}/deploy/overlays/minikube"
if [ "$ACTIVE_MINIKUBE_KUSTOMIZE_DIR" != "$PROJECT_MINIKUBE_KUSTOMIZE_DIR" ]; then
CONTEXT="${PROFILE}" OVERLAY_DIR="${PROJECT_MINIKUBE_KUSTOMIZE_DIR}" "${PROJECT_DIR}/deploy/scripts/minikube-detect-k8s-api-ip.sh"
fi
ok "Minikube K8s API CIDRs refreshed"
# Upgrade path: stage the additive reader credential before the full overlay
# changes HCC. Fresh bootstrap has no ready control-api yet and remains
# fail-closed until the post-migration branch immediately below.
#
# A partially bootstrapped REUSE_DB profile is a distinct state from both a
# fresh bootstrap and a healthy upgrade: an existing GFS writer proves that the
# profile already owns a serving plane, while an unready control-api can mean
# that the runtime-role Secret is still waiting for the migration below. Do not
# cut HCC over in that state, but do not deadlock recovery on the same readiness
# condition either. Fence the control-plane writers before migrations/roles,
# then complete the full overlay and GFS credential reconciliation only after
# runtime-role readiness has been proved.
WRITER_RECOVERY=false
PARTIAL_BOOTSTRAP_RECOVERY=false
PARTIAL_CONTROL_API_REPLICAS=""
PARTIAL_WORKFLOW_REPLICAS=""
PARTIAL_HCC_REPLICAS=""
PARTIAL_TRACE_REPLICAS=""
PARTIAL_CONTROL_API_FENCED=false
PARTIAL_WORKFLOW_FENCED=false
PARTIAL_HCC_FENCED=false
PARTIAL_TRACE_FENCED=false
WRITER_RECOVERY_STATE_HELPER="${SCRIPT_DIR}/writer-recovery-state.py"
WRITER_RECOVERY_STATE_ROOT="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery"
WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"
WRITER_RECOVERY_STATE_LOADED=false
WRITER_RECOVERY_STATE_PHASE=""
WRITER_RECOVERY_STATE_HEAD=""
WRITER_RECOVERY_FENCE_PENDING=false
WRITER_RECOVERY_MIGRATIONS_COMPLETE=false
K8S_API_EGRESS_POLICY_FILE=""
K8S_API_EGRESS_POLICY_DRIFT=false
RECOVERY_OVERLAY_FILE=""
writer_recovery_state_cli() {
local command="$1"
shift
python3 "$WRITER_RECOVERY_STATE_HELPER" "$command" \
--path "$WRITER_RECOVERY_STATE_FILE" \
--profile "$PROFILE" \
--context "$T2_CONTEXT" \
--worktree "$PROJECT_DIR" \
--branch "$T2_BRANCH" \
--head "$T2_HEAD" "$@"
}
writer_recovery_state_write() {
local phase="$1"
writer_recovery_state_cli write \
--phase "$phase" \
--hcc "$PARTIAL_HCC_REPLICAS" \
--workflow "$PARTIAL_WORKFLOW_REPLICAS" \
--trace "$PARTIAL_TRACE_REPLICAS" \
--control-api "$PARTIAL_CONTROL_API_REPLICAS"
}
writer_recovery_state_phase() {
writer_recovery_state_write "$1"
WRITER_RECOVERY_STATE_LOADED=true
WRITER_RECOVERY_STATE_PHASE="$1"
}
writer_recovery_state_clear() {
writer_recovery_state_cli clear
WRITER_RECOVERY_STATE_LOADED=false
WRITER_RECOVERY_STATE_PHASE=""
WRITER_RECOVERY_FENCE_PENDING=false
}
writer_recovery_state_load() {
local state_output state_status=0
state_output="$(writer_recovery_state_cli read --include-head)" || state_status=$?
if [ "$state_status" -ne 0 ]; then
err "Unable to validate the durable writer-recovery state; refusing profile mutation"
return 1
fi
if [ "$state_output" = NONE ]; then
return 0
fi
IFS='|' read -r WRITER_RECOVERY_STATE_PHASE PARTIAL_HCC_REPLICAS \
PARTIAL_WORKFLOW_REPLICAS PARTIAL_TRACE_REPLICAS \
PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD <<<"$state_output"
[ -n "$WRITER_RECOVERY_STATE_PHASE" ] &&
[[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] &&
[[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] &&
[[ "$PARTIAL_TRACE_REPLICAS" =~ ^[0-9]+$ ]] &&
[[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || {
err "Durable writer-recovery state is incomplete; refusing profile mutation"
return 1
}
WRITER_RECOVERY_STATE_LOADED=true
case "$WRITER_RECOVERY_STATE_PHASE" in
planned)
# No scale has been authorized yet. Re-run classification with the
# recorded original replica counts and either clear or resume it.
;;
hcc-fencing|hcc-fenced)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
workflow-fencing|workflow-fenced)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
trace-fencing|trace-fenced)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
api-fencing)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
api-fenced)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
policy-ready)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
# A persisted policy phase is historical evidence only. Re-fence and
# restage the current rendered policy before any writer is restored so a
# Minikube API endpoint change cannot resurrect the original egress drift.
WRITER_RECOVERY_FENCE_PENDING=true
;;
roles-ready|api-restoring)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then
WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
else
log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"
fi
# The phase is written before the control-api restore. An interruption
# after scale but before the next phase must close every writer again.
WRITER_RECOVERY_FENCE_PENDING=true
;;
api-restored|overlay-applying|overlay-applied)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then
WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
else
log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"
fi
# The overlay is rendered with all four database-writer Deployments at zero.
# Re-fence on every post-migration resume because an interrupted apply
# may have recreated a writer before its durable phase was updated.
WRITER_RECOVERY_FENCE_PENDING=true
;;
*)
err "Durable writer-recovery state has an unsupported phase: $WRITER_RECOVERY_STATE_PHASE"
return 1
;;
esac
log "Resuming durable writer recovery at phase ${WRITER_RECOVERY_STATE_PHASE}"
}
writer_recovery_state_prepare() {
local state
if [ "$WRITER_RECOVERY_STATE_LOADED" = true ]; then
return 0
fi
state="$($KC -n control-plane get deployment/host-context-controller \
-o 'jsonpath={.spec.replicas}')" || return 1
PARTIAL_HCC_REPLICAS="$state"
state="$($KC -n control-plane get deployment/workflow-recipes \
-o 'jsonpath={.spec.replicas}')" || return 1
PARTIAL_WORKFLOW_REPLICAS="$state"
state="$($KC -n control-plane get deployment/trace-maintenance-worker \
-o 'jsonpath={.spec.replicas}')" || return 1
PARTIAL_TRACE_REPLICAS="$state"
state="$($KC -n control-plane get deployment/control-api \
-o 'jsonpath={.spec.replicas}')" || return 1
PARTIAL_CONTROL_API_REPLICAS="$state"
[[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] || return 1
[[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] || return 1
[[ "$PARTIAL_TRACE_REPLICAS" =~ ^[0-9]+$ ]] || return 1
[[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || return 1
writer_recovery_state_phase planned
}
writer_recovery_policy_ready() {
case "$WRITER_RECOVERY_STATE_PHASE" in
policy-ready|roles-ready|api-restored|overlay-applied) return 0 ;;
*) return 1 ;;
esac
}
control_api_is_ready() {
local state replicas ready
if ! state="$($KC -n control-plane get deployment/control-api \
-o 'jsonpath={.spec.replicas}{"|"}{.status.readyReplicas}' 2>/dev/null)"; then
return 2
fi
IFS='|' read -r replicas ready <<<"$state"
ready="${ready:-0}"
if [[ ! "$replicas" =~ ^[1-9][0-9]*$ || ! "$ready" =~ ^[0-9]+$ ]]; then
return 2
fi
[[ "$ready" == "$replicas" ]]
}
runtime_role_secret_is_missing() {
local dsn
if ! dsn="$($KC -n control-plane get secret control-api-postgres-runtime \
-o 'jsonpath={.data.connection-string}' --ignore-not-found 2>/dev/null)"; then
return 2
fi
[ -z "$dsn" ]
}
fence_partial_control_api() {
local pods
if [[ ! "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then
err "control-api replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
log "Fencing control-api writers at ${PARTIAL_CONTROL_API_REPLICAS} replica(s)"
writer_recovery_state_phase api-fencing
$KC -n control-plane scale deployment/control-api --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l 'app=control-api,!clerum.io/component' -o name)" \
|| return 1
if [ -n "$pods" ]; then
$KC -n control-plane wait --for=delete pod \
-l 'app=control-api,!clerum.io/component' --timeout=180s >/dev/null
fi
PARTIAL_CONTROL_API_FENCED=true
writer_recovery_state_phase api-fenced
}
fence_partial_workflow_reconciler() {
local pods
if [[ ! "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]]; then
err "workflow-recipes replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
log "Fencing workflow-recipes at ${PARTIAL_WORKFLOW_REPLICAS} replica(s)"
writer_recovery_state_phase workflow-fencing
$KC -n control-plane scale deployment/workflow-recipes --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l app=workflow-recipes -o name)" \
|| return 1
if [ -n "$pods" ]; then
$KC -n control-plane wait --for=delete pod \
-l app=workflow-recipes --timeout=180s >/dev/null
fi
PARTIAL_WORKFLOW_FENCED=true
writer_recovery_state_phase workflow-fenced
}
fence_partial_trace_worker() {
local pods desired
if [[ ! "${PARTIAL_TRACE_REPLICAS}" =~ ^[0-9]+$ ]]; then
err "trace-maintenance-worker replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
log "Fencing trace-maintenance-worker at zero replicas (restore ${PARTIAL_TRACE_REPLICAS} after recovery)"
writer_recovery_state_phase trace-fencing
$KC -n control-plane scale deployment/trace-maintenance-worker --replicas=0 >/dev/null
desired="$($KC -n control-plane get deployment/trace-maintenance-worker \
-o 'jsonpath={.spec.replicas}')" || return 1
if [[ "${desired}" != 0 ]]; then
err "trace-maintenance-worker did not converge to zero desired replicas; refusing recovery"
return 1
fi
pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" \
|| return 1
if [ -n "${pods}" ]; then
$KC -n control-plane wait --for=delete pod \
-l app=trace-maintenance-worker --timeout=180s >/dev/null
fi
pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" \
|| return 1
if [ -n "${pods}" ]; then
err "trace-maintenance-worker pods remain after the zero-replica fence; refusing recovery"
return 1
fi
PARTIAL_TRACE_FENCED=true
writer_recovery_state_phase trace-fenced
}
fence_partial_hcc() {
local pods
if [[ ! "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]]; then
err "host-context-controller replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
log "Fencing host-context-controller at ${PARTIAL_HCC_REPLICAS} replica(s)"
writer_recovery_state_phase hcc-fencing
$KC -n control-plane scale deployment/host-context-controller --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l app=host-context-controller -o name)" \
|| return 1
if [ -n "$pods" ]; then
$KC -n control-plane wait --for=delete pod \
-l app=host-context-controller --timeout=180s >/dev/null
fi
PARTIAL_HCC_FENCED=true
writer_recovery_state_phase hcc-fenced
}
fence_partial_bootstrap_writers() {
# HCC is fenced first so it cannot reconcile the existing GlobalFileSystem
# while the control-plane writer window is being closed. The control-api
# selector excludes migration Jobs, which intentionally share app=control-api.
writer_recovery_state_prepare
fence_partial_hcc
fence_partial_workflow_reconciler
fence_partial_trace_worker
fence_partial_control_api
}
apply_refreshed_k8s_api_network_policies() {
if [ ! -s "$K8S_API_EGRESS_POLICY_FILE" ]; then
err "Validated Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"
return 1
fi
if writer_recovery_policy_ready && [ "$K8S_API_EGRESS_POLICY_DRIFT" != true ]; then
log "Validated Kubernetes API egress policy already staged; resuming writer recovery"
return 0
fi
if [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; then
# The generated k8s-api-ip.yaml is a strategic-merge patch and omits the
# inherited podSelector. Apply only the complete object extracted from the
# active kustomize render, while every control-plane writer is fenced.
log "Applying the complete refreshed Kubernetes API egress policy while writers are fenced"
$KC apply -f "$K8S_API_EGRESS_POLICY_FILE" >/dev/null
else
log "Current Kubernetes API egress policy already matches the rendered endpoint"
fi
writer_recovery_state_phase policy-ready
}
prepare_refreshed_k8s_api_network_policy() {
local endpoint_ip rendered_file live_file live_status=0 check_status=0
local octet
endpoint_ip="$($KC get endpoints kubernetes \
-o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null)" || {
err "Unable to read the Kubernetes API endpoint; refusing writer recovery"
return 1
}
if [[ ! "$endpoint_ip" =~ ^[0-9]+(\.[0-9]+){3}$ ]]; then
err "Kubernetes API endpoint is not exactly one IPv4 address; refusing writer recovery"
return 1
fi
IFS=. read -r endpoint_octet_1 endpoint_octet_2 endpoint_octet_3 endpoint_octet_4 <<<"$endpoint_ip"
for octet in "$endpoint_octet_1" "$endpoint_octet_2" "$endpoint_octet_3" "$endpoint_octet_4"; do
if ! [[ "$octet" =~ ^[0-9]+$ ]] || (( 10#$octet > 255 )); then
err "Kubernetes API endpoint contains an invalid IPv4 octet; refusing writer recovery"
return 1
fi
done
[ -f "$WRITER_RECOVERY_STATE_HELPER" ] || {
err "Writer recovery state helper is missing"
return 1
}
[ -f "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" ] || {
err "Kubernetes API policy validator is missing; refusing writer recovery"
return 1
}
K8S_API_POLICY_TMP_DIR="$(mktemp -d "${T2_TMP_ROOT:-${TMPDIR:-/tmp}}/evenfire-k8s-api-policy.XXXXXX")"
rendered_file="${K8S_API_POLICY_TMP_DIR}/rendered.yaml"
K8S_API_EGRESS_POLICY_FILE="${K8S_API_POLICY_TMP_DIR}/control-plane-policy.yaml"
live_file="${K8S_API_POLICY_TMP_DIR}/live.json"
# Render the active overlay first so the object applied during recovery
# includes every inherited field from the base NetworkPolicy.
if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" >"$rendered_file"; then
err "Unable to render the active overlay; refusing writer recovery"
return 1
fi
if ! RUBYOPT=--disable=gems ruby \
"${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --extract "${endpoint_ip}/32" \
<"$rendered_file" >"$K8S_API_EGRESS_POLICY_FILE"; then
err "Rendered Kubernetes API NetworkPolicy is incomplete or unsafe; refusing writer recovery"
return 1
fi
if $KC -n control-plane get networkpolicy/allow-k8s-api-egress-control-plane \
-o json --ignore-not-found >"$live_file" 2>/dev/null; then
:
else
live_status=$?
err "Unable to inspect the live Kubernetes API NetworkPolicy (status ${live_status}); refusing writer recovery"
return 1
fi
if [ ! -s "$live_file" ] || [ "$(tr -d '[:space:]' <"$live_file")" = "{}" ]; then
K8S_API_EGRESS_POLICY_DRIFT=true
log "Kubernetes API egress NetworkPolicy is absent; a complete rendered policy will be staged after fencing"
return 0
fi
if RUBYOPT=--disable=gems ruby \
"${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --check-live "${endpoint_ip}/32" \
<"$live_file" >/dev/null; then
K8S_API_EGRESS_POLICY_DRIFT=false
log "Kubernetes API egress NetworkPolicy matches the current endpoint"
return 0
else
check_status=$?
fi
if [ "$check_status" -eq 1 ]; then
K8S_API_EGRESS_POLICY_DRIFT=true
log "Kubernetes API egress NetworkPolicy has a recognized endpoint/port drift; a complete policy will be staged after fencing"
return 0
fi
err "Live Kubernetes API NetworkPolicy has an unexpected shape; refusing writer recovery"
return 1
}
restore_partial_control_api() {
if [ "$PARTIAL_CONTROL_API_FENCED" != true ]; then
if [[ "$WRITER_RECOVERY_STATE_PHASE" == api-restored ||
"$WRITER_RECOVERY_STATE_PHASE" == overlay-applied ]]; then
if control_api_is_ready; then
return 0
fi
return 1
fi
return 1
fi
log "Restoring control-api to ${PARTIAL_CONTROL_API_REPLICAS} replica(s) after runtime-role provisioning"
writer_recovery_state_phase api-restoring || return 1
$KC -n control-plane scale deployment/control-api \
--replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null || return 1
$KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null || return 1
control_api_is_ready || return 1
PARTIAL_CONTROL_API_FENCED=false
writer_recovery_state_phase api-restored || return 1
}
restore_partial_non_api_writers() {
if [ "$PARTIAL_WORKFLOW_FENCED" = true ]; then
$KC -n control-plane scale deployment/workflow-recipes \
--replicas="$PARTIAL_WORKFLOW_REPLICAS" >/dev/null || return 1
PARTIAL_WORKFLOW_FENCED=false
fi
if [ "$PARTIAL_TRACE_FENCED" = true ]; then
$KC -n control-plane scale deployment/trace-maintenance-worker \
--replicas="$PARTIAL_TRACE_REPLICAS" >/dev/null || return 1
PARTIAL_TRACE_FENCED=false
fi
if [ "$PARTIAL_HCC_FENCED" = true ]; then
$KC -n control-plane scale deployment/host-context-controller \
--replicas="$PARTIAL_HCC_REPLICAS" >/dev/null || return 1
PARTIAL_HCC_FENCED=false
fi
}
reconcile_existing_gfs_credentials() {
# A staged sync may have deferred active-pod proof until the reconciler
# created the reader DSN. Complete that proof before the overlay changes HCC.
GFS_AUTH_SYNC_ALLOW_STAGED=true \
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
# If gfsc-reader is already Ready, settle the leftover claim first so
# reconcile does not rollout restart and race HCC's gfsReconciler. The
# gfs-rollout-shim makes the wait judge readiness rather than a leftover
# template generation.
GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \
T2_SKIP_LOCK=true \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
T2_SKIP_LOCK=true \
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
# The staged sync may have deferred active-pod proof until this reconcile
# produced the reader DSN; the strict pass is the durable consumer attestation.
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
}
render_fenced_recovery_overlay() {
local rendered_file="${K8S_API_POLICY_TMP_DIR}/recovery-rendered.yaml"
[ -n "${K8S_API_POLICY_TMP_DIR}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ] || {
err "Recovery overlay workspace is unavailable; refusing writer recovery"
return 1
}
log "Rendering the recovery overlay with HCC, workflow-recipes, trace-maintenance-worker, and control-api fenced"
if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | \
RUBYOPT=--disable=gems ruby "${SCRIPT_DIR}/render-fenced-writer-deployments.rb" \
--target control-plane/host-context-controller \
--target control-plane/workflow-recipes \
--target control-plane/trace-maintenance-worker \
--target control-plane/control-api >"$rendered_file"; then
err "Unable to render a complete fenced recovery overlay; refusing writer recovery"
return 1
fi
[ -s "$rendered_file" ] || {
err "Fenced recovery overlay is empty; refusing writer recovery"
return 1
}
RECOVERY_OVERLAY_FILE="$rendered_file"
}
apply_fenced_recovery_overlay() {
render_fenced_recovery_overlay || return 1
writer_recovery_state_phase overlay-applying || return 1
log "Applying the complete recovery overlay while all control-plane writers remain fenced"
$KC apply -f "$RECOVERY_OVERLAY_FILE" >/dev/null || return 1
writer_recovery_state_phase overlay-applied || return 1
}
writer_recovery_state_load || exit 1
prepare_refreshed_k8s_api_network_policy || exit 1
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
if [ "$RESET_DB" = true ]; then
log "Database reset path — HCC cutover deferred until post-convergence verification"
else
writer_dsn="$($KC -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"
if [ -n "${writer_dsn}" ]; then
if [ "$WRITER_RECOVERY" = true ]; then
log "Existing GFS writer recovery is already in progress; preserving the durable fence state"
elif control_api_is_ready; then
log "Existing GFS writer detected — reconciling credentials before overlay upgrade"
reconcile_existing_gfs_credentials
else
control_api_ready_status=$?
if [ "$control_api_ready_status" -ne 1 ]; then
err "Unable to determine control-api readiness; refusing to classify the GFS upgrade path"
exit 1
fi
if runtime_role_secret_is_missing; then
runtime_secret_status=0
else
runtime_secret_status=$?
fi
if [ "$runtime_secret_status" -eq 0 ]; then
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
log "Existing GFS writer and empty control-api runtime Secret detected — fencing writers until migrations and runtime roles converge"
elif [ "$runtime_secret_status" -eq 1 ]; then
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
if [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; then
log "Existing GFS writer detected with control-api not Ready — repairing the stale API-egress policy before runtime-role convergence"
else
log "Existing GFS writer detected with control-api not Ready — fencing all database writers before runtime-role convergence"
fi
else
err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"
exit 1
fi
fence_partial_bootstrap_writers
apply_refreshed_k8s_api_network_policies
fi
else
log "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"
fi
fi
if [ "$WRITER_RECOVERY" = true ] && [ "$WRITER_RECOVERY_FENCE_PENDING" = true ]; then
fence_partial_bootstrap_writers
apply_refreshed_k8s_api_network_policies
WRITER_RECOVERY_FENCE_PENDING=false
fi
if [ "$WRITER_RECOVERY_STATE_LOADED" = true ] &&
[ "$WRITER_RECOVERY_STATE_PHASE" = planned ] &&
[ "$WRITER_RECOVERY" = false ]; then
writer_recovery_state_clear
fi
log "Applying kustomize overlay (${ACTIVE_MINIKUBE_RENDER_DIR})..."
if [ "$RESET_DB" = true ]; then
# Keep control-api scaled to zero until migrations and role restoration are
# complete; applying the full overlay here would race it against a fresh DB.
$KC apply -k "$ACTIVE_MINIKUBE_RENDER_DIR" -l app=control-postgres
elif [ "$WRITER_RECOVERY" = true ]; then
log "Deferring the full kustomize overlay until control-api/runtime roles converge"
else
$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
fi
ok "Kustomize overlay applied or safely deferred"
if [ "$RESET_DB" = true ]; then
log "Rebuilding database contracts and restoring GFS roles after reset..."
$KC rollout status deployment/control-postgres -n control-plane --timeout=180s >/dev/null
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/converge-control-db-after-reset.sh" \
--overlay "$ACTIVE_MINIKUBE_RENDER_DIR" \
--job-name control-api-db-migrate-reset
$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
# The overlay declares ordinary one-replica defaults. Reassert the exact
# pre-reset operating counts after applying it; credentials are verified and
# HCC is restored by convergence before this safe rollout boundary.
$KC -n gfs scale deployment/gfsc-writer --replicas="$RESET_WRITER_REPLICAS" >/dev/null
$KC -n gfs scale deployment/gfsc-reader --replicas="$RESET_READER_REPLICAS" >/dev/null
$KC -n control-plane scale deployment/control-api --replicas="$RESET_CONTROL_API_REPLICAS" >/dev/null
$KC -n control-plane scale deployment/workflow-recipes --replicas="$RESET_WORKFLOW_REPLICAS" >/dev/null
$KC -n control-plane scale deployment/trace-maintenance-worker --replicas="$RESET_TRACE_REPLICAS" >/dev/null
$KC -n control-plane scale deployment/host-context-controller --replicas="$RESET_HCC_REPLICAS" >/dev/null
$KC -n gfs rollout status deployment/gfsc-writer --timeout=180s >/dev/null
$KC -n gfs rollout status deployment/gfsc-reader --timeout=180s >/dev/null
if [ "$RESET_WORKFLOW_REPLICAS" -gt 0 ]; then
$KC -n control-plane rollout status deployment/workflow-recipes --timeout=180s >/dev/null
fi
if [ "$RESET_TRACE_REPLICAS" -gt 0 ]; then
$KC -n control-plane rollout status deployment/trace-maintenance-worker --timeout=180s >/dev/null
fi
if [ "$RESET_HCC_REPLICAS" -gt 0 ]; then
$KC -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null
fi
ok "Control-api database and GFS roles converged after reset"
else
if [ "$WRITER_RECOVERY_MIGRATIONS_COMPLETE" != true ]; then
ensure_control_postgres_ready
log "Applying control-api database migrations and runtime roles..."
CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \
--overlay "$ACTIVE_MINIKUBE_RENDER_DIR"
CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"
ok "Control-api database migrations and runtime roles applied"
if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then
writer_recovery_state_phase roles-ready
fi
else
log "Skipping migrations and runtime-role provisioning; durable recovery state already completed that boundary"
fi
if [ "$WRITER_RECOVERY" = true ]; then
# Keep every database writer at zero while the full overlay is applied. The overlay
# itself declares one replica for these Deployments, so applying it
# directly would reopen the writer window before the explicit restore.
apply_fenced_recovery_overlay
log "Recovery overlay is applied — restoring control-api before GFS reconciliation"
if ! restore_partial_control_api; then
if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
else
err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
fi
exit 1
fi
if ! control_api_is_ready; then
if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
else
err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
fi
exit 1
fi
log "Control-api/runtime roles are Ready — reconciling GFS after the fenced overlay"
reconcile_existing_gfs_credentials
restore_partial_non_api_writers
ok "Fenced recovery overlay applied after runtime-role readiness; writers restored explicitly"
else
$KC scale deployment/control-api -n control-plane --replicas=1 >/dev/null
$KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null
fi
# On the upgrade path the full overlay may still cut HCC over to the
# split writer/reader templates; reconciling after that lands leaves the
# staged reader credential rollout-pending and fails the final verify.
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/wait-gfsc-secret-references.sh"
# The overlay apply re-declares the base gfs-config with an empty jwt-public-key;
# re-sync it before any reader pod may need to start, otherwise the reconcile
# readiness wait can only time out.
GFS_AUTH_SYNC_ALLOW_STAGED=true \
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \
T2_SKIP_LOCK=true \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
T2_SKIP_LOCK=true \
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
# The preceding staged sync is allowed to wait for this DSN-producing
# reconcile; the strict pass is the durable consumer attestation.
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
if [ "$WRITER_RECOVERY" = true ]; then
writer_recovery_state_clear
fi
ok "GFS credentials reconciled and writer bootstrap verified"
fi
# 6c. Re-apply generated service tokens after kustomize.
# Re-patch the generated inter-service tokens after every overlay apply so the
# consumer Secrets and control-api stay in sync.
log "Re-applying inter-service tokens after kustomize deploy..."
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-inter-service-tokens.sh"
ok "Inter-service tokens re-applied after kustomize deploy"
# 6e. Deploy CRD instances
log "Applying CRD instances..."
$KC apply -f "${PROJECT_DIR}/deploy/overlays/minikube/instances/"
ok "CRD instances applied (Host, Context, CommunicationChannel, GFS, policy)"
# instances-e2e/context-mcpservers.yaml must apply AFTER instances/context.yaml
# so its non-empty mcpServers list wins over the empty default.
if [ "$SEED_PROFILE" = "e2e" ]; then
# airtable-server.yaml names the LOCAL ref clerum/airtable-mcp-server:test,
# and this directory is applied with `kubectl apply -f` -- outside kustomize,
# so the ghcr component never rewrites it -- while HCC forces
# imagePullPolicy=IfNotPresent on minikube. That ref must therefore already
# be in the daemon, and minikube setup no longer acquires it: the registry
# distributes MCP servers and installs them on demand, so airtable-mcp-server
# is deployed_to_minikube:false and is neither built nor pulled by default.
# Build it here, gated on the one branch that consumes it, rather than paying
# for it on every setup.
log "Building the optional Airtable MCP image for the E2E demo instance..."
MINIKUBE_BUILD_AIRTABLE_MCP_IMAGE=true \
bash "${SCRIPT_DIR}/build-images.sh" --only=airtable-mcp-server
ok "airtable-mcp-server built"
log "Applying E2E demo MCP server instances..."
$KC apply -f "${PROJECT_DIR}/deploy/overlays/minikube/instances-e2e/"
ok "E2E instances applied (airtable, mongodb, mongodb-mcp-stack + context1 servers)"
else
log "Skipping demo MCP servers (SEED_PROFILE=minimal) — context1 starts empty."
fi
# 6f. Apply Host model from .env/default E2E model
# The rule lives in scripts/minikube/host-model.sh: an explicit
# CLERUM_MODEL_PROVIDER wins, CLERUM_MODEL_NAME without it is refused, and
# otherwise the provider follows the first key present (openai/gpt-5.4-mini,
# with a warning, when there is none). The pair must then be an enabled
# llm_allowed_models row in this profile's control-postgres, which the
# migrations above created; a refusal stops setup before the Host is applied.
# A no-key run still comes up fully: docs/get-started/quickstart.md documents
# that the agent does not reply until a key is added.
resolve_host_model || exit 1
log "Checking Host model ${RESOLVED_PROVIDER}/${RESOLVED_MODEL} against llm_allowed_models..."
assert_host_model_allowed "$RESOLVED_PROVIDER" "$RESOLVED_MODEL" || exit 1
log "Applying Host model ${RESOLVED_PROVIDER}/${RESOLVED_MODEL} (source: ${CLERUM_MODEL_PROVIDER:+CLERUM_MODEL_PROVIDER}${CLERUM_MODEL_PROVIDER:-auto-detected from .env})..."
cat <<HOSTEOF | $KC apply -f -
apiVersion: clerum.io/v1alpha1
kind: Host
metadata:
name: chatllm
namespace: mcp-host
spec:
host: chatLLM
contextRef: context1
secretRef: chatllm-api-keys
model:
provider: ${RESOLVED_PROVIDER}
name: ${RESOLVED_MODEL}
workflowControl:
scopes:
- workflow:list
- workflow:read
- workflow:trigger
- workflow:approval:resolve
- workflow:approval:decide
channels:
- all-channels
approval:
defaultPolicy: channel_users
channels:
telegram:
enabled: true
HOSTEOF
ok "Host model applied from .env/default E2E model"
# 6g. Post-deploy JWT sync (in case configmap was recreated by kustomize)
log "Re-syncing JWT public key after deploy..."
if $KC get secret rpc-proxy-secrets -n rpc-proxy &>/dev/null; then
$KC get secret rpc-proxy-secrets -n rpc-proxy \
-o jsonpath='{.data.RPC_PROXY_JWT_PUBLIC_KEY}' | base64 -d \
| python3 -c "import sys,subprocess,json; \
key=sys.stdin.read(); \
patch=json.dumps([{'op':'replace','path':'/data/CLERUM_AUTH_JWT_PUBLIC_KEY','value':key}]); \
subprocess.run(['kubectl','--context=${PROFILE}','patch','configmap','mcp-host-config','-n','mcp-host','--type=json','-p',patch],check=True)"
ok "JWT public key re-synced to mcp-host-config"
else
warn "rpc-proxy-secrets still not found — JWT sync skipped"
fi
# 6h. Re-apply generated ConfigMaps that kustomize may have overwritten with placeholders.
# The base manifests contain placeholder values for ConfigMaps that are populated by
# generate-keys.sh. Kustomize apply (Step 6b) overwrites them. Re-apply here to restore.
log "Re-applying generated ConfigMaps (control-api-public-key, clerum-wrc-public-key)..."
bash "${SCRIPT_DIR}/generate-keys.sh" --apply
ok "Generated ConfigMaps re-applied after kustomize deploy"
# 6i. Force rollout restart when images were rebuilt.
# ----------------------------------------------------------------------
# `kubectl apply -k` only triggers a new ReplicaSet when the Deployment
# spec changes. When we rebuild with the same `:test` tag and the
# manifest is byte-identical, pods keep their previous imageID even
# though minikube's Docker daemon now has a fresh image with the same
# tag but a different digest. Result: pods silently run stale code —
# this has caused multiple "I fixed it but the pod didn't change"
# debugging loops (see memory: deploy-gotcha-kustomize-unchanged-stale-pod).
#
# Fix: after every full build+apply, explicitly restart every Deployment
# that consumes a `clerum/*:test` image. `rollout restart` patches a
# kubectl.kubernetes.io/restartedAt annotation on the pod template,
# guaranteeing a new ReplicaSet and a fresh image pull (IfNotPresent
# resolves to the new digest because minikube already has it loaded).
#
# Skipped when --skip-build is passed (nothing new to pick up).
if [ "$SKIP_BUILD" = false ]; then
log "Forcing rollout restart for freshly-built clerum images..."
REFRESH_DEPLOYS=(
# (#273) Static clerum-channel-reader Deployment retired; per-Host
# channel-reader-<host> Deployments come up after the first Host CRD is
# created via the Control UI, not at bootstrap time.
"control-plane:control-api"
"control-plane:control-ui"
"control-plane:host-context-controller"
"control-plane:workflow-recipes"
"mcp-host:chatllm"
"mcp-server:mcp-proxy"
"profiles:external-rest-api"
"profiles:profile-ui"
"rpc-proxy:rpc-proxy"
)
if [ "$SKIP_UIS" = true ]; then
REFRESH_DEPLOYS=(
"control-plane:control-api"
"control-plane:host-context-controller"
"control-plane:workflow-recipes"
"mcp-host:chatllm"
"mcp-server:mcp-proxy"
"profiles:external-rest-api"
"rpc-proxy:rpc-proxy"
)
fi
RESTARTED=0
for entry in "${REFRESH_DEPLOYS[@]}"; do
ns="${entry%%:*}"
name="${entry##*:}"
if $KC get deployment "$name" -n "$ns" >/dev/null 2>&1; then
$KC rollout restart deployment/"$name" -n "$ns" >/dev/null 2>&1 && \
RESTARTED=$((RESTARTED + 1))
fi
done
ok "Restarted ${RESTARTED} deployment(s) to pick up fresh image digests"
fi
# 6j. Validate McpServer envSecret references before waiting for pods.
# If a McpServer CRD points at a non-existent Secret, HCC silently hangs
# and pods never come up — abort early with a clear diagnosis instead of
# letting Step 8 time out on phantom "pod never ready" errors.
if ! validate_mcpserver_secrets; then
err "Aborting setup: fix the missing Secret(s) above and re-run."
exit 1
fi
# ======================================================================
# Step 7: Deploy evenfire-registry side-by-side
# ======================================================================
# evenfire-registry (registry-api + postgres + minio) lives in a sibling
# repo, not this monorepo. The helper resolves the sibling checkout from normal
# primary repos and worktrees, then applies the consumer NetworkPolicies from
# this overlay so control-api/workflow-recipes can reach registry-api.
step_header 7 $TOTAL_STEPS "evenfire-registry side-by-side deploy"
EVENFIRE_DIR_DEFAULT="$(cd "${PROJECT_DIR}/.." && pwd)/evenfire-registry"
EVENFIRE_DIR="${EVENFIRE_REGISTRY_DIR:-${EVENFIRE_DIR_DEFAULT}}"
if [[ -d "${EVENFIRE_DIR}" && -f "${EVENFIRE_DIR}/Dockerfile" ]]; then
log "Found evenfire-registry at ${EVENFIRE_DIR} — deploying..."
if [ -f "${PROJECT_DIR}/scripts/minikube/deploy-evenfire-registry.sh" ]; then
if MINIKUBE_PROFILE="${PROFILE}" EVENFIRE_REGISTRY_DIR="${EVENFIRE_DIR}" \
bash "${PROJECT_DIR}/scripts/minikube/deploy-evenfire-registry.sh"; then
ok "evenfire-registry deployed"
else
warn "evenfire-registry deploy failed — registry calls will fail until you fix this."
warn "Run manually: make minikube-deploy-evenfire-registry"
fi
else
warn "skip: scripts/minikube/deploy-evenfire-registry.sh not present (sibling repo not included in this distribution)"
fi
else
warn "evenfire-registry deploy failed — registry calls will fail until you fix this."
warn "Run manually: MINIKUBE_PROFILE=${PROFILE} make minikube-deploy-evenfire-registry"
warn "If auto-discovery fails, pass EVENFIRE_REGISTRY_DIR=/absolute/path/to/evenfire-registry"
fi
# member-registration-service was extracted to a sibling repo too. Build +
# apply it the same best-effort way so local control-api invitation flows work.
# Skipped silently if the repo is not checked out locally.
MEMBER_DIR_DEFAULT="$(cd "${PROJECT_DIR}/.." && pwd)/evenfire-member-registration"
MEMBER_DIR="${EVENFIRE_MEMBER_REGISTRATION_DIR:-${MEMBER_DIR_DEFAULT}}"
if [[ -d "${MEMBER_DIR}" && -f "${MEMBER_DIR}/Dockerfile" ]]; then
log "Found evenfire-member-registration at ${MEMBER_DIR} — deploying..."
if [ -f "${PROJECT_DIR}/scripts/minikube/deploy-evenfire-member-registration.sh" ]; then
if MINIKUBE_PROFILE="${PROFILE}" EVENFIRE_MEMBER_REGISTRATION_DIR="${MEMBER_DIR}" \
bash "${PROJECT_DIR}/scripts/minikube/deploy-evenfire-member-registration.sh"; then
ok "evenfire-member-registration deployed"
else
warn "evenfire-member-registration deploy failed — invitation/registration calls will fail until you fix this."
warn "Run manually: bash scripts/minikube/deploy-evenfire-member-registration.sh"
fi
else
warn "skip: scripts/minikube/deploy-evenfire-member-registration.sh not present (sibling repo not included in this distribution)"
fi
else
warn "evenfire-member-registration not found at ${MEMBER_DIR}"
warn "control-api invitation flows will fail any registration call until you clone it:"
warn " cd $(dirname "${MEMBER_DIR}") && git clone <evenfire-member-registration-repo> evenfire-member-registration"
warn "Then: bash scripts/minikube/deploy-evenfire-member-registration.sh"
fi
# ======================================================================
# Step 8: Wait + Auto-Recovery
# ======================================================================
step_header 8 $TOTAL_STEPS "Wait + Auto-Recovery"
CORE_DEPLOYS=(
"control-plane:control-api"
"control-plane:host-context-controller"
"profiles:external-rest-api"
"rpc-proxy:rpc-proxy"
"mcp-host:chatllm"
"mcp-server:mcp-proxy"
)
# registry-api is deployed by Step 7 (side-by-side); failure there is a
# warning, not blocking — leave it out of the CORE_DEPLOYS readiness gate
# so a missing evenfire-registry checkout doesn't fail the whole setup.
all_ready=true
for entry in "${CORE_DEPLOYS[@]}"; do
ns="${entry%%:*}"
name="${entry##*:}"
log "Waiting for ${ns}/${name} (180s)..."
if $KC rollout status deployment/"$name" -n "$ns" --timeout=180s 2>/dev/null; then
ok "${ns}/${name} ready"
else
warn "${ns}/${name} not ready — checking for recovery..."
# A stale or late corruption signature is diagnostic evidence only. Storage
# deletion always requires a new explicit --reset-db invocation and UID.
if [ "$name" = "control-api" ] && postgres_has_invalid_checkpoint; then
err "Postgres reports an invalid checkpoint; automatic destructive recovery is disabled"
err "Re-run with --reset-db and CONTROL_DB_RESET_PVC_UID=<exact approved UID>"
fi
err "${ns}/${name} NOT ready"
all_ready=false
fi
done
# ----------------------------------------------------------------------
# gfs serving provisioning (runs once pods are up, so control-api has applied
# migration 0048_gfs_permission_store that creates the gfs_controller role). gfsc fails closed
# without BOTH: (a) its JWT public key in gfs-config, and (b) the gfs_controller
# DB LOGIN + DSN in gfs-controller-db. The inline JWT sync (6g) only covers
# mcp-host-config, so these gfs-specific steps are explicit here. Idempotent.
# FAIL LOUD: when the GFS stack is deployed, a provisioning failure ABORTS the
# setup — continuing would hand the user a cluster whose GFS plane 503s on
# every operation (issue #775).
# ----------------------------------------------------------------------
if $KC get configmap gfs-config -n gfs &>/dev/null; then
log "Provisioning gfs serving (public key → gfs-config + gfs_controller DB login)..."
if ! bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs; then
err "gfs public-key sync FAILED — gfsc cannot verify tokens. Fix and re-run."
exit 1
fi
if CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/scripts/minikube/verify-gfs.sh"; then
ok "gfs serving verified (reader/writer credentials and readiness)"
else
err "gfs DB provisioning FAILED — gfsc /readyz will fail closed and every GFS operation will 503. Fix and re-run."
exit 1
fi
else
log "gfs-config not present — skipping gfs serving provisioning"
fi
# ======================================================================
# Step 9: Seed Registry Catalog
# ======================================================================
# Populates the registry database with the curated catalog of
# MCP servers (local + remote) and workflow recipes. Idempotent — entries
# that already exist return 409 and are skipped. Required for the registry
# E2E tests and the control-ui "Registry Catalog" tab to show entries on
# a fresh deploy.
# ======================================================================
step_header 9 $TOTAL_STEPS "Seed Registry Catalog"
# Registry seed lives in evenfire-registry (sibling repo). If it's checked
# out and exposes a `seed-minikube` make target, run it. Otherwise skip
# loudly — control-ui's "Registry Catalog" tab will be empty but everything
# else still works.
EVENFIRE_DIR_DEFAULT="$(cd "${PROJECT_DIR}/.." && pwd)/evenfire-registry"
EVENFIRE_DIR="${EVENFIRE_REGISTRY_DIR:-${EVENFIRE_DIR_DEFAULT}}"
if [[ ! -d "${EVENFIRE_DIR}" ]]; then
warn "evenfire-registry not found at ${EVENFIRE_DIR} — skipping registry seed"
elif ! $KC get deployment registry-api -n registry &>/dev/null; then
warn "registry-api deployment not found in clerum-test — skipping seed"
elif ! (cd "${EVENFIRE_DIR}" && make -n minikube-seed >/dev/null 2>&1); then
warn "evenfire-registry has no 'minikube-seed' target — skipping seed"
else
log "Running evenfire-registry minikube-seed target..."
if (cd "${EVENFIRE_DIR}" && make minikube-seed 2>&1 | tail -25); then
ok "Registry catalog seeded"
else
warn "Registry seed encountered errors — check output above"
fi
fi
# ======================================================================
# Step 10: Seed Test User
# ======================================================================
step_header 10 $TOTAL_STEPS "Seed Test User"
if [ "$DEFER_BOOTSTRAP_SEED" = true ]; then
# The browser E2E must be the first actor to consume the one-shot
# /api/v1/admin/auth/setup route. Do not invoke seed-test-data.sh here: it
# logs in as the auto-seeded admin and makes that route correctly reject a
# later setup attempt with HTTP 409.
SEED_USER_OK=deferred
SEED_USER_EMAIL=""
log "Deferring bootstrap/admin seed; browser E2E owns /api/v1/admin/auth/setup"
else
# The e2e profile seeds the test identities used by the Desktop and Control UI
# journeys. The minimal profile must not seed anything named test*.
if [ "$SEED_PROFILE" = "e2e" ]; then
SEED_USER_DEFAULT_EMAIL="[email protected]"
SEED_USER_DEFAULT_NAME="Test User"
# ADMIN_PASSWORD was resolved once above. Use the known local test fallback
# only when neither the canonical .env nor the process configured any admin
# alias; never replace a real seeded credential.
if [ -z "${ADMIN_PASSWORD:-}" ]; then
ADMIN_PASSWORD="$(printf '%s%s' 'changeme123' '!')"
fi
else
# Minimal: the Control-UI admin IS the sole Desktop App member — no separate
# seeded user. Point both the admin-bootstrap email and the seeded desktop
# user at the same evenfire-branded address. seed-e2e-data.sh consumes
# /admin/auth/setup before login on a fresh DB, so this identity is created
# atomically with the initial_setup operator link instead of minting a
# second ordinary member/team. `[email protected]` would leak the internal
# code name onto a product surface (the Desktop member list) — evenfire
# branding belongs here (docs/concepts/code-names.md).
ADMIN_EMAIL="$(clerum_canonical_email "${ADMIN_EMAIL:-admin@evenfire.local}")"
SEED_USER_DEFAULT_EMAIL="$ADMIN_EMAIL"
SEED_USER_DEFAULT_NAME="admin"
fi
if [ "$SEED_PROFILE" = "minimal" ]; then
# The minimal quickstart owns the bootstrap identity. Do not let a
# process/.env E2E override silently mint a second ordinary Desktop user;
# only the explicitly named seed override is considered, and the guard
# below still requires it to equal ADMIN_EMAIL.
SEED_USER_EMAIL="$(clerum_canonical_email "${CLERUM_SEED_USER_EMAIL:-${SEED_USER_DEFAULT_EMAIL}}")"
else
SEED_USER_EMAIL="${CLERUM_SEED_USER_EMAIL:-${CLERUM_TEST_USER_EMAIL:-${E2E_DEV_LOGIN_EMAIL:-${SEED_USER_DEFAULT_EMAIL}}}}"
fi
SEED_USER_NAME="${E2E_DEV_LOGIN_NAME:-${SEED_USER_DEFAULT_NAME}}"
if [ "$SEED_PROFILE" = "minimal" ] \
&& ! clerum_minimal_identity_matches "$ADMIN_EMAIL" "$SEED_USER_EMAIL"; then
err "$(clerum_minimal_identity_error "$SEED_USER_EMAIL" "$ADMIN_EMAIL")"
exit 1
fi
log "Seeding test user ${SEED_USER_EMAIL} → agent=chatllm, context=context1"
SEED_USER_OK=true
if CONTEXT="${PROFILE}" ADMIN_PASSWORD="${ADMIN_PASSWORD}" \
SEED_PROFILE="${SEED_PROFILE}" \
ADMIN_EMAIL="${ADMIN_EMAIL:-}" \
E2E_DEV_LOGIN_EMAIL="${SEED_USER_EMAIL}" \
E2E_DEV_LOGIN_NAME="${SEED_USER_NAME}" \
bash "${SCRIPT_DIR}/seed-test-data.sh" 2>&1 | tail -15; then
ok "Test user seeded"
else
SEED_USER_OK=false
if [ "$SEED_PROFILE" = "minimal" ]; then
# This step is the only place that both consumes the governed
# /admin/auth/setup path and creates the owner Desktop identity. The
# minimal seed is setup-first; it refuses to fall back to an ordinary
# /admin/users member when the initial_setup operator link is missing.
# generate-keys.sh bakes a hardcoded bcrypt hash of changeme123! into the
# admin Secret, and control-api/src/db.ts auto-inserts a live `admin` row
# from it on every fresh DB. If this step fails under the default
# (minimal) profile, that publicly-known credential is still live — a
# green summary would ship an install that is both unusable and insecure.
# Abort instead. Setup is idempotent, so re-running after the underlying
# issue is fixed recovers cleanly for every failure except a missing
# initial_setup link on a reused database: control-api stamps
# last_login_at on each admin login and setup only matches a bootstrap
# row whose last_login_at is NULL, so that one needs a DB rebuild. The
# seeder prints the exact recovery command for that case.
err "Test user seed failed under SEED_PROFILE=minimal, aborting. The bootstrap admin password may still be the publicly-known default (see generate-keys.sh) until this step succeeds. Fix the error above and re-run setup. If the seeder reported a missing initial_setup Desktop link, re-run without REUSE_DB/--keep-db so the control DB is rebuilt."
exit 1
else
warn "Test user seed encountered errors — check output above"
fi
fi
fi
# ======================================================================
# Step 11: Seed Workflow Trigger Test Data
# ======================================================================
step_header 11 $TOTAL_STEPS "Seed Workflow Trigger Test Data"
if [ "$SEED_PROFILE" != "e2e" ]; then
log "Skipping workflow-trigger + sandbox-ui fixtures (SEED_PROFILE=minimal)."
else
log "Seeding workflow-trigger fixtures for local E2E recipes..."
if CONTEXT="${PROFILE}" E2E_DEV_LOGIN_EMAIL="${SEED_USER_EMAIL}" \
bash "${SCRIPT_DIR}/seed-workflow-triggers-test-data.sh" 2>&1 | tail -20; then
ok "Workflow-trigger E2E recipes seeded"
else
warn "Workflow-trigger E2E seed encountered errors — desktop/control-ui workflow E2E may fail until fixed"
fi
if [ "$SKIP_UIS" = true ]; then
warn "Skipping sandbox-ui Desktop Apps validation seed (--skip-uis)."
else
log "Seeding sandbox-ui local test app for Desktop Apps validation..."
if CONTEXT="${PROFILE}" E2E_DEV_LOGIN_EMAIL="${SEED_USER_EMAIL}" \
bash "${SCRIPT_DIR}/seed-sandbox-ui-test-data.sh" 2>&1 | tail -25; then
ok "Sandbox-ui local test app seeded"
else
warn "Sandbox-ui test app seed encountered errors — Desktop Apps may not list local sandbox-ui fixtures"
fi
fi
fi
# ======================================================================
# Step 12: Verify
# ======================================================================
step_header 12 $TOTAL_STEPS "Verify"
echo ""
echo -e "${BOLD} Deployment Status${NC}"
echo -e "${BOLD} ─────────────────────────────────────────────────────${NC}"
printf " ${BOLD}%-20s %-35s %-10s${NC}\n" "NAMESPACE" "DEPLOYMENT" "STATUS"
echo -e " ─────────────────────────────────────────────────────"
$KC get deploy -A --no-headers 2>/dev/null | while read -r ns name ready _up_to_date _available _age; do
# ready is like "1/1"
desired="${ready##*/}"
actual="${ready%%/*}"
if [ "$actual" = "$desired" ] && [ "$actual" != "0" ]; then
status="${GREEN}OK${NC}"
else
status="${RED}!! ${ready}${NC}"
fi
printf " %-20s %-35s ${status}\n" "$ns" "$name"
done
echo ""
echo -e "${BOLD}================================================================${NC}"
if [ "${T2_SETUP_HANDOFF_REQUIRED}" = true ]; then
# The ordinary setup summary historically gates only CORE_DEPLOYS and may
# return zero after printing "partially complete". A T2 handoff is stricter:
# reuse the final T2 deployment inventory contract (including additional
# deployed workloads) and make an unready/missing required deployment block
# both publication and setup success.
prior_bootstrap_required="${T2_BOOTSTRAP_REQUIRED}"
prior_plan_mode="${T2_PLAN_MODE}"
T2_BOOTSTRAP_REQUIRED=false
T2_PLAN_MODE=false
if ! t2_deployment_check; then
all_ready=false
fi
T2_BOOTSTRAP_REQUIRED="${prior_bootstrap_required}"
T2_PLAN_MODE="${prior_plan_mode}"
if ! T2_PROJECT_DIR="${T2_PROJECT_DIR}" T2_WORKTREE_ID="${T2_WORKTREE_ID}" \
T2_RUN_ID="${T2_RUN_ID}" T2_PROFILE="${T2_PROFILE}" T2_CONTEXT="${T2_CONTEXT}" \
T2_BRANCH="${T2_BRANCH}" T2_HEAD="${T2_HEAD}" T2_SKIP_LOCK="${T2_SKIP_LOCK}" \
T2_LOCK_KEY="${T2_LOCK_KEY}" T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" \
T2_IMAGE_MANIFEST="${T2_IMAGE_MANIFEST}" \
T2_SETUP_HANDOFF_ROOT="${T2_SETUP_HANDOFF_ROOT}" \
T2_SETUP_HANDOFF_TTL_SECONDS="${T2_SETUP_HANDOFF_TTL_SECONDS}" \
T2_SETUP_HANDOFF_TRANSITION="${T2_SETUP_HANDOFF_TRANSITION}" \
T2_SETUP_HANDOFF_SETUP_COMPLETE="${all_ready}" \
bash "${SCRIPT_DIR}/t2-setup-handoff.sh" create; then
err "T2 strict setup is incomplete; refusing setup success and handoff"
exit 1
fi
ok "Created the one-shot T2 setup-complete handoff"
fi
if [ "$all_ready" = true ]; then
echo -e "${GREEN}${BOLD} Minikube setup complete! All core services ready.${NC}"
else
echo -e "${YELLOW}${BOLD} Setup partially complete. Some services need attention.${NC}"
echo -e " Check: ${CYAN}kubectl --context=${PROFILE} get pods -A${NC}"
fi
echo -e "${BOLD}================================================================${NC}"
echo ""
echo -e " ${BOLD}Next steps:${NC}"
if [ "$SKIP_UIS" = true ]; then
echo -e " ${CYAN}npm run web${NC} # Run web UIs locally"
echo -e " ${YELLOW}Control UI, Profile UI, and Desktop App build steps were skipped.${NC}"
else
echo -e " ${CYAN}make minikube-pf-desktop${NC} # Start port-forwards (Desktop App)"
echo -e " ${CYAN}make minikube-pf-all${NC} # Start port-forwards (Control UI + API)"
echo -e " ${CYAN}cd desktop-app && npm run build && npm start${NC} # Launch desktop app"
fi
echo ""
echo -e " ${BOLD}Already done by setup:${NC}"
# Step 10 either owns the normal seed flow or is deliberately deferred so the
# first-run browser setup can consume /api/v1/admin/auth/setup. Keep the
# summary explicit: deferred is not a successful seed and must never be shown
# as one.
if [ "$DEFER_BOOTSTRAP_SEED" = true ]; then
echo -e " ${YELLOW}•${NC} Bootstrap admin seed deferred — browser E2E owns /api/v1/admin/auth/setup"
echo -e " ${YELLOW}•${NC} Owner user seed deferred — created by the browser setup journey"
else
if [ "${SEED_USER_OK:-true}" = "true" ]; then
echo -e " ${GREEN}✓${NC} JWT keys + admin bootstrap (resolved admin credential)"
else
echo -e " ${RED}✗${NC} Admin bootstrap NOT confirmed — Step 10 (Seed Test User) failed"
fi
if [ "${SEED_PROFILE:-}" = "e2e" ]; then
if [ "${SEED_USER_OK:-true}" = "true" ]; then
echo -e " ${GREEN}✓${NC} Test user seeded (${SEED_USER_EMAIL} → chatllm + context1)"
else
echo -e " ${RED}✗${NC} Test user seed FAILED (${SEED_USER_EMAIL}) — check Step 10 output above"
fi
echo -e " ${GREEN}✓${NC} Workflow-trigger E2E recipes seeded"
else
if [ "${SEED_USER_OK:-true}" = "true" ]; then
echo -e " ${GREEN}✓${NC} Owner user seeded (${SEED_USER_EMAIL} → chatllm + context1)"
else
echo -e " ${RED}✗${NC} Owner user seed FAILED (${SEED_USER_EMAIL}) — check Step 10 output above"
fi
fi
fi
echo -e " ${GREEN}✓${NC} Registry catalog seeded (MCP servers + recipes)"
echo ""