Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
66.8 KB
Raw Blame
#!/usr/bin/env bash
set -u

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck source=scripts/tests/lib/minikube-fixture-repo.sh
source "${REPO_ROOT}/scripts/tests/lib/minikube-fixture-repo.sh"

FAIL=0

pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }

write_test_profile_metadata() {
  local root="$1" profile="clerum-full-setup-fixture"
  local repo="$root/repo" profile_dir="$root/profiles/$profile"
  local branch short_sha
  MINIKUBE_TEST_PROFILE="$profile"
  MINIKUBE_TEST_CONTEXT="$profile"
  minikube_test_fixture_repo_init "$REPO_ROOT" "$root"
  repo="$MINIKUBE_TEST_PROJECT_DIR"
  branch="$MINIKUBE_TEST_BRANCH"
  short_sha="$(git -C "$repo" rev-parse --short=8 HEAD)"
  mkdir -p "$profile_dir"
  printf 'PROFILE=%s\nBRANCH=%s\nSHA_SHORT=%s\nDIRTY=false\nREPO_DIR=%s\n' \
    "$profile" "$branch" "$short_sha" "$repo" >"$profile_dir/profile.env"
  test_loopback_url() { printf 'http://%s:%s' 127.0.0.1 "$1"; }
  cat >"$profile_dir/ports.env" <<EOF_PORTS
PORT_BASE=28117
CONTROL_UI_PORT=28117
PROFILE_UI_PORT=28118
MCP_HOST_PORT=28197
REGISTRY_API_PORT=28202
CONTROL_API_PORT=28207
EXTERNAL_REST_API_PORT=28208
MEMBER_REGISTRATION_SERVICE_PORT=28209
RPC_PROXY_PORT=28211
WORKFLOW_APPROVAL_READER_PORT=28215
CONTROL_UI_URL=$(test_loopback_url 28117)
PROFILE_UI_URL=$(test_loopback_url 28118)
PROFILE_UI_BASE_URL=$(test_loopback_url 28118)
CONTROL_API_URL=$(test_loopback_url 28207)
EXTERNAL_REST_API_URL=$(test_loopback_url 28208)
MEMBER_REGISTRATION_SERVICE_URL=$(test_loopback_url 28209)
RPC_PROXY_URL=$(test_loopback_url 28211)
REGISTRY_API_URL=$(test_loopback_url 28202)
WORKFLOW_APPROVAL_READER_URL=$(test_loopback_url 28215)
MCP_HOST_URL=$(test_loopback_url 28197)
EOF_PORTS
  TEST_PROFILE="$profile"
  TEST_PROFILE_ROOT="$root/profiles"
  TEST_PROFILE_ENV="$profile_dir/profile.env"
  TEST_PORTS_ENV="$profile_dir/ports.env"
  TEST_LOCK_ROOT="$root/locks"
  TEST_PROJECT_DIR="$repo"
}

verify_fixture_host_unchanged() {
  if ! minikube_test_assert_host_unchanged; then
    fail "minikube fixture mutated the host checkout"
    return 1
  fi
}

assert_broken_profile_is_recreated() {
  local tmp log_file setup_log_file
  tmp="$(mktemp -d)"
  log_file="$tmp/ops.log"
  setup_log_file="$tmp/full-setup.log"
  write_test_profile_metadata "$tmp"

  cat > "$tmp/docker" <<'STUB'
#!/usr/bin/env bash
case "${1:-} ${2:-}" in
  "context inspect")
    if [[ "$*" == *'TLSMaterial'* ]]; then
      printf 'unix:///private/tmp/evenfire-test-docker.sock\tfalse\t{}\n'
    else
      printf 'unix:///private/tmp/evenfire-test-docker.sock\n'
    fi
    ;;
  "info ") ;;
esac
exit 0
STUB

  cat > "$tmp/python3" <<'STUB'
#!/usr/bin/env bash
exit 0
STUB

  cat > "$tmp/minikube" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
LOG_FILE="${TEST_LOG_FILE:?}"
STATE_DIR="${TEST_STATE_DIR:?}"
printf '%s\n' "$*" >>"$LOG_FILE"

args=("$@")
if [[ "${1:-}" == "-p" ]]; then
  shift 2 || true
fi
cmd="${1:-}"
shift || true

case "$cmd" in
  status)
    if [[ -f "$STATE_DIR/started" ]]; then
      cat <<'EOF'
clerum-test
type: Control Plane
host: Running
kubelet: Running
apiserver: Running
kubeconfig: Configured
EOF
      exit 0
    fi
    cat <<'EOF'
clerum-test
type: Control Plane
host: Running
kubelet: Stopped
apiserver: Stopped
kubeconfig: Configured
EOF
    exit 2
    ;;
  delete)
    touch "$STATE_DIR/deleted"
    exit 0
    ;;
  start)
    touch "$STATE_DIR/started"
    exit 0
    ;;
esac

exit 0
STUB

  cat > "$tmp/kubectl" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
STATE_DIR="${TEST_STATE_DIR:?}"
args="$*"
if [[ "$args" == *"cluster-info"* ]]; then
  if [[ -f "$STATE_DIR/started" ]]; then
    exit 0
  fi
  exit 1
fi
exit 0
STUB

  cat > "$tmp/start.sh" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
LOG_FILE="${TEST_LOG_FILE:?}"
STATE_DIR="${TEST_STATE_DIR:?}"
printf 'start-helper %s\n' "$*" >>"$LOG_FILE"
touch "$STATE_DIR/started"
exit 0
STUB

  chmod +x "$tmp/docker" "$tmp/python3" "$tmp/minikube" "$tmp/kubectl" "$tmp/start.sh"
  mkdir -p "$tmp/state"

  if PATH="$tmp:$PATH" \
     TEST_LOG_FILE="$log_file" \
     TEST_STATE_DIR="$tmp/state" \
     MINIKUBE_PROFILE="$TEST_PROFILE" \
     T2_PROJECT_DIR="$TEST_PROJECT_DIR" \
     T2_PROFILE_ROOT="$TEST_PROFILE_ROOT" T2_PROFILE_ENV="$TEST_PROFILE_ENV" \
     T2_PORTS_ENV="$TEST_PORTS_ENV" T2_LOCK_ROOT="$TEST_LOCK_ROOT" \
     MINIKUBE_SETUP_EXIT_AFTER_CLUSTER=true \
     MINIKUBE_RECREATE_PROFILE=true \
     CONFIRM_PROFILE="$TEST_PROFILE" \
     MINIKUBE_START_SCRIPT="$tmp/start.sh" \
     bash scripts/minikube/full-setup.sh --skip-build >"$setup_log_file" 2>&1; then
    if grep -q "delete -p $TEST_PROFILE" "$log_file" && grep -q 'start-helper' "$log_file"; then
      pass "full-setup recreates broken minikube profiles before start"
    else
      fail "full-setup did not delete and recreate broken profile"
      cat "$log_file"
    fi
  else
    fail "full-setup failed in broken-profile recovery test"
    cat "$setup_log_file"
  fi

  verify_fixture_host_unchanged
  rm -rf "$tmp"
}

assert_healthy_profile_skips_recreate() {
  local tmp log_file setup_log_file
  tmp="$(mktemp -d)"
  log_file="$tmp/ops.log"
  setup_log_file="$tmp/full-setup.log"
  write_test_profile_metadata "$tmp"

  cat > "$tmp/docker" <<'STUB'
#!/usr/bin/env bash
case "${1:-} ${2:-}" in
  "context inspect")
    if [[ "$*" == *'TLSMaterial'* ]]; then
      printf 'unix:///private/tmp/evenfire-test-docker.sock\tfalse\t{}\n'
    else
      printf 'unix:///private/tmp/evenfire-test-docker.sock\n'
    fi
    ;;
  "info ") ;;
esac
exit 0
STUB

  cat > "$tmp/python3" <<'STUB'
#!/usr/bin/env bash
exit 0
STUB

  cat > "$tmp/minikube" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
LOG_FILE="${TEST_LOG_FILE:?}"
printf '%s\n' "$*" >>"$LOG_FILE"

if [[ "${1:-}" == "-p" ]]; then
  shift 2 || true
fi
cmd="${1:-}"
shift || true

case "$cmd" in
  status)
    cat <<'EOF'
clerum-test
type: Control Plane
host: Running
kubelet: Running
apiserver: Running
kubeconfig: Configured
EOF
    exit 0
    ;;
  delete|start)
    exit 99
    ;;
esac

exit 0
STUB

  cat > "$tmp/kubectl" <<'STUB'
#!/usr/bin/env bash
args="$*"
if [[ "$args" == *"cluster-info"* ]]; then
  exit 0
fi
exit 0
STUB

  cat > "$tmp/start.sh" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
LOG_FILE="${TEST_LOG_FILE:?}"
printf 'start-helper %s\n' "$*" >>"$LOG_FILE"
exit 0
STUB

  chmod +x "$tmp/docker" "$tmp/python3" "$tmp/minikube" "$tmp/kubectl" "$tmp/start.sh"

  if PATH="$tmp:$PATH" \
     TEST_LOG_FILE="$log_file" \
     MINIKUBE_PROFILE="$TEST_PROFILE" \
     T2_PROJECT_DIR="$TEST_PROJECT_DIR" \
     T2_PROFILE_ROOT="$TEST_PROFILE_ROOT" T2_PROFILE_ENV="$TEST_PROFILE_ENV" \
     T2_PORTS_ENV="$TEST_PORTS_ENV" T2_LOCK_ROOT="$TEST_LOCK_ROOT" \
     MINIKUBE_SETUP_EXIT_AFTER_CLUSTER=true \
     MINIKUBE_START_SCRIPT="$tmp/start.sh" \
     bash scripts/minikube/full-setup.sh --skip-build >"$setup_log_file" 2>&1; then
    if ! grep -q 'delete -p clerum-test' "$log_file" && grep -q 'start-helper --validate-only' "$log_file"; then
      pass "full-setup leaves healthy minikube profiles alone"
    else
      fail "full-setup unexpectedly recreated healthy profile"
      cat "$log_file"
    fi
  else
    fail "full-setup failed in healthy-profile test"
    cat "$setup_log_file"
  fi

  verify_fixture_host_unchanged
  rm -rf "$tmp"
}

assert_branch_profile_deploy_dir_is_used() {
  if grep -Fq 'ACTIVE_MINIKUBE_DEPLOY_DIR="${BRANCH_PROFILE_DEPLOY_DIR:-${PROJECT_DIR}/deploy}"' scripts/minikube/full-setup.sh && \
     grep -Fq 'ACTIVE_MINIKUBE_KUSTOMIZE_DIR="${ACTIVE_MINIKUBE_DEPLOY_DIR}/overlays/minikube"' scripts/minikube/full-setup.sh && \
     grep -Fq 'OVERLAY_DIR="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}"' scripts/minikube/full-setup.sh; then
    pass "full-setup applies the branch-profile deploy copy when provided"
  else
    fail "full-setup does not honor BRANCH_PROFILE_DEPLOY_DIR for kustomize apply"
  fi
}

assert_member_registration_hmac_is_patched() {
  if grep -Fq 'MEMBER_REGISTRATION_HMAC="$(resolve_member_registration_hmac)"' deploy/scripts/apply-inter-service-tokens.sh && \
     grep -Fq -- '--arg memberRegistrationHmacKey "$MEMBER_REGISTRATION_HMAC_KEY"' deploy/scripts/apply-inter-service-tokens.sh && \
     grep -Fq '($memberRegistrationHmacKey): $memberRegistrationHmac' deploy/scripts/apply-inter-service-tokens.sh && \
     grep -Fq 'CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET is required when no existing control-api Secret value is present' deploy/scripts/apply-inter-service-tokens.sh; then
    pass "inter-service token patch includes fail-closed member-registration HMAC"
  else
    fail "inter-service token patch omits fail-closed member-registration HMAC"
  fi
}

assert_branch_scoped_minikube_context_is_supported() {
  if grep -Fq 'source "${SCRIPT_DIR}/lib/clerum-minikube-context.sh"' deploy/scripts/apply-inter-service-tokens.sh && \
     grep -Fq 'clerum-*|*minikube*' deploy/scripts/lib/clerum-minikube-context.sh && \
     grep -Fq 'CLERUM_PROJECT_DIR:-}/scripts/minikube/deploy-evenfire-member-registration.sh' deploy/scripts/apply-inter-service-tokens.sh; then
    pass "inter-service tokens recognize branch-scoped Clerum minikube profiles"
  else
    fail "inter-service tokens do not recognize branch-scoped Clerum minikube profiles"
  fi
}

# The extracted production block invokes these stubs and assigns all_ready.
# shellcheck disable=SC2034,SC2329,SC2154
assert_mcp_proxy_bootstrap_wait_is_bounded_and_fail_closed() {
  local block outcome result
  block="$(sed -n '/^CORE_DEPLOYS=(/,/^done$/p' scripts/minikube/full-setup.sh)"
  for outcome in ready unavailable; do
    result="$(
      KC=fixture_kubectl
      proxy_waits=0
      log() { :; }
      ok() { :; }
      warn() { :; }
      err() { :; }
      postgres_has_invalid_checkpoint() { return 1; }
      fixture_kubectl() {
        if [[ "$*" == 'rollout status deployment/mcp-proxy -n mcp-server --timeout=180s' ]]; then
          proxy_waits=$((proxy_waits + 1))
          [[ "$outcome" == ready ]]
          return
        fi
        return 0
      }
      eval "$block"
      printf '%s %s' "$proxy_waits" "$all_ready"
    )"
    if [[ "$outcome" == ready && "$result" != '1 true' ]] || \
       [[ "$outcome" == unavailable && "$result" != '1 false' ]]; then
      fail "mcp-proxy bootstrap wait must run once with a finite deadline and block completion on failure ($outcome: $result)"
      return
    fi
  done
  pass "mcp-proxy bootstrap wait is bounded and failure blocks setup completion"
}

assert_gfs_provisioning_follows_migrations_and_core_readiness() {
  local overlay_line ensure_line migration_line reconcile_after_migration control_ready_line core_block reset_block wal_block
  local reset_boundary reset_overlay reset_pg_ready reset_converge
  local wal_detect wal_error
  overlay_line="$(grep -n 'Kustomize overlay applied' scripts/minikube/full-setup.sh | head -1 | cut -d: -f1)"
  ensure_line="$(grep -n 'ensure_control_postgres_ready' scripts/minikube/full-setup.sh | tail -1 | cut -d: -f1)"
  migration_line="$(grep -n 'deploy/scripts/run-control-api-db-migration.sh' scripts/minikube/full-setup.sh | head -1 | cut -d: -f1)"
  reconcile_after_migration="$(grep -n 'reconcile-gfs-deploy-credentials.sh' scripts/minikube/full-setup.sh | tail -1 | cut -d: -f1)"
  # There are two intentional control-api readiness fences: the re-use path
  # waits before the HCC cutover, and the migration path waits afterwards.
  # Assert the post-migration fence here so adding the bounded re-use wait does
  # not make this ordering check select the earlier guard.
  control_ready_line="$(grep -n 'rollout status deployment/control-api.*timeout=180s' scripts/minikube/full-setup.sh | tail -1 | cut -d: -f1)"
  core_block="$(sed -n '/^CORE_DEPLOYS=(/,/^)/p' scripts/minikube/full-setup.sh)"
  reset_block="$(sed -n '/# 6a. Optional DB reset/,/# 6c. Re-apply generated service tokens/p' scripts/minikube/full-setup.sh)"
  wal_block="$(sed -n '/A stale or late corruption signature/,/err "${ns}\/\${name} NOT ready"/p' scripts/minikube/full-setup.sh)"

  reset_boundary="$(grep -n 'reset-control-db-storage.sh' <<<"$reset_block" | head -1 | cut -d: -f1)"
  reset_overlay="$(grep -n 'Kustomize overlay applied' <<<"$reset_block" | head -1 | cut -d: -f1)"
  reset_pg_ready="$(grep -n 'rollout status deployment/control-postgres' <<<"$reset_block" | head -1 | cut -d: -f1)"
  reset_converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$reset_block" | head -1 | cut -d: -f1)"

  wal_detect="$(grep -n 'postgres_has_invalid_checkpoint' <<<"$wal_block" | head -1 | cut -d: -f1)"
  wal_error="$(grep -n 'automatic destructive recovery is disabled' <<<"$wal_block" | head -1 | cut -d: -f1)"

  if [[ -n "$overlay_line" && -n "$ensure_line" && -n "$migration_line" && -n "$reconcile_after_migration" && -n "$control_ready_line" ]] && \
     [[ "$overlay_line" -lt "$ensure_line" && "$ensure_line" -lt "$migration_line" && "$migration_line" -lt "$control_ready_line" && "$control_ready_line" -lt "$reconcile_after_migration" ]] && \
     ! grep -q 'provision-gfs-db.sh.*\(rotate\|stage\)-' scripts/minikube/full-setup.sh && \
     grep -q 'Fresh bootstrap detected.*GFSC remains fail-closed' scripts/minikube/full-setup.sh && \
     [[ "$reset_boundary" -lt "$reset_overlay" && "$reset_overlay" -lt "$reset_pg_ready" && "$reset_pg_ready" -lt "$reset_converge" ]] && \
     [[ "$wal_detect" -lt "$wal_error" ]] && \
     ! grep -q 'reset-control-db-storage.sh' <<<"$wal_block" && \
     ! grep -q '^recover_control_postgres_wal()' scripts/minikube/full-setup.sh && \
     grep -q 'CONTROL_DB_RESET_PVC_UID is required with --reset-db' scripts/minikube/full-setup.sh && \
     grep -q 'RESET_STORAGE_ARGS.*--expected-pvc-uid' scripts/minikube/full-setup.sh && \
     grep -q 'get configmap control-db-reset-state' <<<"$reset_block" && \
     grep -q 'scale deployment/gfsc-writer --replicas="\$RESET_WRITER_REPLICAS"' <<<"$reset_block" && \
     grep -q 'scale deployment/gfsc-reader --replicas="\$RESET_READER_REPLICAS"' <<<"$reset_block" && \
     grep -q 'scale deployment/host-context-controller --replicas="\$RESET_HCC_REPLICAS"' <<<"$reset_block" && \
     ! grep -q 'delete pvc control-postgres-data' scripts/minikube/full-setup.sh && \
     [[ "$(grep -c 'GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true' scripts/minikube/full-setup.sh)" -ge 2 ]] && \
     [[ "$(grep -c 'scripts/minikube/settle-gfs-reader-rollout.sh' scripts/minikube/full-setup.sh)" -ge 2 ]] && \
     [[ "$(grep -c 'gfs-rollout-shim' scripts/minikube/full-setup.sh)" -ge 2 ]] && \
     [[ "$core_block" != *"gfs-controller"* ]]; then
    pass "full-setup centralizes explicit UID-bound reset and never auto-deletes on WAL logs"
  else
    fail "full-setup GFS/reset ordering can leave partial roles or deadlock on GFS readiness"
  fi
}

assert_full_setup_defaults_to_db_rebuild() {
  if grep -Eq '^RESET_DB=true$' scripts/minikube/full-setup.sh && \
     grep -Fq 'case "${REUSE_DB:-false}" in' scripts/minikube/full-setup.sh && \
     grep -Eq -- '--keep-db\)[[:space:]]+RESET_DB=false' scripts/minikube/full-setup.sh && \
     grep -Fq 'reset-control-db-storage.sh' scripts/minikube/full-setup.sh; then
    pass "full-setup rebuilds the DB by default with a REUSE_DB/--keep-db opt-out"
  else
    fail "full-setup does not default to a clean DB rebuild with a REUSE_DB/--keep-db opt-out"
  fi
}

assert_reuse_db_normalizer_precedes_flag_loop() {
  local normalizer_line flag_loop_line
  normalizer_line="$(grep -Fn 'case "${REUSE_DB:-false}" in' scripts/minikube/full-setup.sh | head -1 | cut -d: -f1)"
  flag_loop_line="$(grep -Fn 'for arg in "$@"; do' scripts/minikube/full-setup.sh | head -1 | cut -d: -f1)"

  if [[ -n "$normalizer_line" && -n "$flag_loop_line" && "$normalizer_line" -lt "$flag_loop_line" ]]; then
    pass "REUSE_DB normalizer runs before the flag-parse loop (flag overrides env)"
  else
    fail "REUSE_DB normalizer does not precede the flag-parse loop — flag>env precedence not enforced"
  fi
}

assert_makefile_passes_reuse_db() {
  if grep -Fq 'REUSE_DB="$(REUSE_DB)"' Makefile; then
    pass "minikube-setup Make target forwards REUSE_DB to full-setup.sh"
  else
    fail "minikube-setup Make target does not forward REUSE_DB to full-setup.sh"
  fi
}

assert_reset_db_flag_backcompat() {
  if grep -Eq -- '--reset-db\)[[:space:]]+RESET_DB=true' scripts/minikube/full-setup.sh; then
    pass "--reset-db flag still forces a DB rebuild (back-compat)"
  else
    fail "--reset-db flag no longer sets RESET_DB=true (back-compat broken)"
  fi
}

assert_skip_build_staleness_find_is_sigpipe_guarded() {
  # full-setup.sh runs `set -euo pipefail`. The --skip-build staleness check
  # pipes `find <source dirs> -newer "$LAST_BUILD_MARKER" | head -1`; when find
  # has matches, head closes the pipe after line 1 and find dies with SIGPIPE
  # (141), which pipefail turns into a fatal `make Error 141`. The pipeline must
  # tolerate that exit status.
  if grep -Fq 'newer "$LAST_BUILD_MARKER" 2>/dev/null | head -1 || true' scripts/minikube/full-setup.sh; then
    pass "skip-build staleness find|head pipeline is SIGPIPE-guarded"
  else
    fail "skip-build staleness find|head is unguarded — SIGPIPE (141) aborts --skip-build"
  fi
}

assert_pipefail_head_guard_prevents_abort() {
  # Deterministic proof the guard is not vacuous: `yes | head -1` always leaves
  # the producer writing when head exits, so under `set -euo pipefail` the bare
  # form aborts (SIGPIPE 141) while the `|| true` form (as applied above) exits 0.
  local bare_rc=0 guarded_rc=0
  bash -c 'set -euo pipefail; v=$(yes | head -1); : "$v"' >/dev/null 2>&1 || bare_rc=$?
  bash -c 'set -euo pipefail; v=$(yes | head -1 || true); : "$v"' >/dev/null 2>&1 || guarded_rc=$?
  if [ "$bare_rc" -ne 0 ] && [ "$guarded_rc" -eq 0 ]; then
    pass "pipefail SIGPIPE guard (|| true) prevents the abort the bare pipeline hits"
  else
    fail "SIGPIPE guard behaved unexpectedly (bare_rc=$bare_rc guarded_rc=$guarded_rc)"
  fi
}

assert_setup_runtime_operations_are_bounded() {
  local setup="scripts/minikube/full-setup.sh"
  if grep -Fq 'run_setup_with_deadline minikube-setup-status' "$setup" && \
     grep -Fq 'run_setup_with_deadline minikube-setup-delete' "$setup" && \
     grep -Fq 'run_setup_with_deadline minikube-setup-start' "$setup" && \
     grep -Fq 'run_setup_with_deadline minikube-setup-validate' "$setup" && \
     grep -Fq 'bash "${SCRIPT_DIR}/docker-cli-env.sh" --check-info' "$setup" && \
     grep -Fq 'if (( status >= 124 )); then' "$setup" && \
     ! grep -Fq 'minikube -p "$PROFILE" status 2>/dev/null || true' "$setup" && \
     ! grep -Eq '(^|[[:space:]])docker info([[:space:]]|$)' "$setup"; then
    pass "full-setup bounds profile lifecycle and isolates its Docker probe"
  else
    fail "full-setup contains an unbounded profile lifecycle or ambient Docker probe"
  fi
}

# full-setup.sh is a 1284-line orchestrator that needs a real cluster to run,
# so these cases read its resolved configuration rather than executing it: they
# source the top of the script with a guard variable set, which stops it before
# the first cluster call. That is the same seam the existing profile cases use,
# applied to variables instead of to minikube invocations.
#
# The stub dir is a safety net, not decoration. Config resolution calls none of
# docker/minikube/kubectl, so if the config-only seam is ever deleted or moved,
# `source full-setup.sh` would otherwise drive a REAL cluster from a unit test.
# The stubs fail loudly instead, which aborts the sourced script at Step 1 and
# leaves the assertion reading an empty value -- a red test, never a green one.
FULL_SETUP_STUB_DIR="$(mktemp -d)"
for _stub in docker minikube kubectl; do
  cat > "$FULL_SETUP_STUB_DIR/$_stub" <<STUB
#!/usr/bin/env bash
echo "STUB: '$_stub' must not be called while resolving full-setup.sh config" >&2
exit 1
STUB
  chmod +x "$FULL_SETUP_STUB_DIR/$_stub"
done
unset _stub
trap 'rm -rf "$FULL_SETUP_STUB_DIR"' EXIT

full_setup_resolves() {
  # full_setup_resolves <var> [env assignments...]
  local var="$1"; shift
  env "$@" PATH="$FULL_SETUP_STUB_DIR:$PATH" MINIKUBE_FULL_SETUP_CONFIG_ONLY=true \
    bash -c 'source "$0" >/dev/null 2>&1; printf "%s" "${'"$var"'}"' \
    "$REPO_ROOT/scripts/minikube/full-setup.sh"
}

# ---------------------------------------------------------------------------
# A PROJECT_DIR whose recorded image manifest the test controls
# ---------------------------------------------------------------------------
#
# full-setup.sh now resolves both the image MODE (on --skip-build) and the ghcr
# TAG from deploy/minikube/.image-manifest.json, and it derives PROJECT_DIR from
# its own location -- so the only way to hand it a chosen manifest is to run a
# COPY of the script from a copied tree.
#
# That copy is not optional even for the cases that want no manifest at all:
# the file is gitignored, so CI never has one and any machine that has run a
# real `make minikube-setup` always does. Running against $REPO_ROOT would make
# "does the banner print the committed pin?" answer differently on a laptop
# than in CI. Every manifest-sensitive case below runs from a copy with the
# manifest either removed or written explicitly.
make_full_setup_copy() {
  # make_full_setup_copy <dir> [manifest-json]
  local d=$1 manifest=${2:-}
  mkdir -p "$d/repo"
  cp -R "$REPO_ROOT/deploy" "$d/repo/deploy"
  cp -R "$REPO_ROOT/scripts" "$d/repo/scripts"
  # `git init`, not decoration. full-setup.sh calls dotenv_canonical_root under
  # `set -e`, and its last statement is `[[ -f .env ]] && printf` -- which
  # returns 1, aborting the caller, when the tree is neither a git repo nor
  # carries a .env. A plain copy is exactly that; a git checkout without a .env
  # (which is what CI runs) takes the branch above it and returns 0. Initialising
  # one here matches CI AND keeps the copy independent of whatever .env the
  # developer's primary checkout happens to hold.
  git init -q "$d/repo"
  rm -f "$d/repo/deploy/minikube/.image-manifest.json"
  if [ -n "$manifest" ]; then
    mkdir -p "$d/repo/deploy/minikube"
    printf '%s' "$manifest" > "$d/repo/deploy/minikube/.image-manifest.json"
  fi
}

# The two fields image-mode.sh reads, in the shape both writers emit.
recorded_manifest() {
  # recorded_manifest <imageSource> <imageTag>
  printf '{\n  "generated": "2026-08-06T00:00:00Z",\n  "profile": "clerum-test",\n  "imageSource": "%s",\n  "imageTag": "%s",\n  "images": {}\n}\n' \
    "$1" "$2"
}

# full_setup_copy_resolves <dir> <var> <flags> [env assignments...]
# <flags> is the (unquoted, test-controlled) flag string handed to the sourced
# script, e.g. --skip-build; pass '' for none.
full_setup_copy_resolves() {
  local d="$1" var="$2" flags="$3"; shift 3
  env "$@" PATH="$FULL_SETUP_STUB_DIR:$PATH" MINIKUBE_FULL_SETUP_CONFIG_ONLY=true \
    bash -c 'source "$0" '"$flags"' >/dev/null 2>&1; printf "%s" "${'"$var"'}"' \
    "$d/repo/scripts/minikube/full-setup.sh"
}

# full_setup_copy_output <dir> <flags> [env assignments...] -- the banner and
# warnings the run prints, which is what an operator actually sees.
full_setup_copy_output() {
  local d="$1" flags="$2"; shift 2
  # $flags is a test-controlled literal, deliberately word-split.
  # shellcheck disable=SC2086
  env "$@" PATH="$FULL_SETUP_STUB_DIR:$PATH" MINIKUBE_FULL_SETUP_CONFIG_ONLY=true \
    bash "$d/repo/scripts/minikube/full-setup.sh" $flags 2>&1
}

assert_ghcr_is_the_default_image_source() {
  local got
  got="$(full_setup_resolves IMAGE_SOURCE)"
  if [ "$got" = "ghcr" ]; then
    pass "IMAGE_SOURCE defaults to ghcr"
  else
    fail "IMAGE_SOURCE defaulted to '${got}', expected 'ghcr'"
  fi
}

assert_image_source_local_is_honoured() {
  local got
  got="$(full_setup_resolves IMAGE_SOURCE IMAGE_SOURCE=local)"
  if [ "$got" = "local" ]; then
    pass "IMAGE_SOURCE=local is honoured"
  else
    fail "IMAGE_SOURCE=local resolved to '${got}'"
  fi
}

assert_bootstrap_seed_deferral_is_opt_in() {
  local got
  got="$(full_setup_resolves DEFER_BOOTSTRAP_SEED)"
  if [ "$got" = "false" ]; then
    pass "bootstrap seed deferral defaults to false"
  else
    fail "bootstrap seed deferral defaulted to '${got}', expected 'false'"
  fi
}

assert_bootstrap_seed_deferral_flag_resolves_for_local_minimal() {
  local d got
  d="$(mktemp -d)"
  make_full_setup_copy "$d"
  got="$(full_setup_copy_resolves "$d" DEFER_BOOTSTRAP_SEED --defer-bootstrap-seed IMAGE_SOURCE=local)"
  if [ "$got" = "true" ]; then
    pass "--defer-bootstrap-seed resolves true for the local minimal browser flow"
  else
    fail "--defer-bootstrap-seed resolved to '${got}', expected 'true'"
  fi
  rm -rf "$d"
}

assert_bootstrap_seed_deferral_rejects_non_local_or_e2e_modes() {
  local d out rc problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d"

  out="$(full_setup_copy_output "$d" --defer-bootstrap-seed IMAGE_SOURCE=ghcr 2>&1)" || rc=$?
  rc="${rc:-0}"
  if [ "$rc" -eq 0 ] || ! grep -q "IMAGE_SOURCE=local" <<< "$out"; then
    problems+="ghcr mode was not rejected; "
  fi

  rc=0
  out="$(full_setup_copy_output "$d" "--defer-bootstrap-seed --seed-profile=e2e" IMAGE_SOURCE=local 2>&1)" || rc=$?
  if [ "$rc" -eq 0 ] || ! grep -q "requires --seed-profile=minimal" <<< "$out"; then
    problems+="e2e seed profile was not rejected; "
  fi

  if [ -z "$problems" ]; then
    pass "bootstrap seed deferral rejects GHCR and e2e fixture modes"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

assert_minimal_seed_is_setup_first_and_link_fail_closed() {
  local seed="$REPO_ROOT/scripts/e2e/seed-e2e-data.sh"
  local dispatch verify_body setup_line login_line verify_line

  # Bound the dispatch and the verifier exactly. A range that ends at the first
  # bare `else` runs past every nested block and swallows most of the file, so
  # presence greps alone still pass when the ordering is reversed or the
  # fail-closed abort is downgraded to a warning.
  dispatch="$(awk '/^# ─── Step 1: Bootstrap before login/,/^fi$/' "$seed")"
  verify_body="$(awk '/^verify_minimal_operator_bootstrap\(\) \{$/,/^\}$/' "$seed")"

  setup_line="$(printf '%s\n' "$dispatch" | grep -n '^  perform_initial_setup$' | head -1 | cut -d: -f1)"
  login_line="$(printf '%s\n' "$dispatch" | grep -n 'login_admin_only' | head -1 | cut -d: -f1)"
  verify_line="$(printf '%s\n' "$dispatch" | grep -n '^  verify_minimal_operator_bootstrap$' | head -1 | cut -d: -f1)"

  if [ -z "$setup_line" ] || [ -z "$login_line" ] || [ -z "$verify_line" ]; then
    fail "minimal seed dispatch must call setup, the login fallback and link verification"
    return
  fi
  # control-api marks last_login_at on every successful admin login, and
  # setupInitialAdminCredentials only matches a bootstrap row whose
  # last_login_at is still NULL. Logging in first destroys setup eligibility
  # permanently, so this ordering is the whole point of the minimal path.
  if [ "$setup_line" -ge "$login_line" ]; then
    fail "minimal seed logs in at line $login_line before consuming setup at line $setup_line; login sets last_login_at and permanently disqualifies /admin/auth/setup"
    return
  fi
  if [ "$verify_line" -le "$login_line" ]; then
    fail "minimal seed must verify the initial_setup link after the login fallback, not before it"
    return
  fi
  if ! printf '%s\n' "$verify_body" | grep -Fq 'if ! clerum_initial_setup_link_matches '; then
    fail "minimal link verification does not gate on the shared initial_setup link contract"
    return
  fi
  # The missing-link branch must abort. Downgrading it to a log leaves the
  # install running as an ordinary unlinked member, which is the exact
  # regression this assertion exists to catch.
  if ! printf '%s\n' "$verify_body" | grep -Eq '^ *die "Minimal bootstrap is incomplete'; then
    fail "minimal link verification does not abort when the initial_setup link is absent"
    return
  fi
  pass "minimal seed consumes setup before login and aborts without an active initial_setup link"
}

assert_minimal_bootstrap_contract_runs_on_system_bash() {
  local contract="$REPO_ROOT/scripts/e2e/minimal-bootstrap-contract.sh"
  local output
  if ! /bin/bash -n "$contract" "$REPO_ROOT/scripts/e2e/seed-e2e-data.sh" "$REPO_ROOT/scripts/minikube/full-setup.sh"; then
    fail "minimal bootstrap scripts are not parseable by the system Bash"
    return
  fi
  # `! cmd` is explicitly exempt from set -e, so a bare `! predicate` line can
  # never fail this suite. refute() runs the predicate inside an `if` and exits
  # non-zero on unexpected success, which set -e does propagate.
  output="$(/bin/bash -c '
    set -e
    source "$1"
    refute() { if "$@"; then printf "unexpected success: %s\n" "$*" >&2; exit 1; fi; }
    [ "$(clerum_canonical_email "[email protected]")" = "[email protected]" ]
    [ "$(clerum_minimal_desktop_email "[email protected]" "" false)" = "[email protected]" ]
    [ "$(clerum_minimal_desktop_email "[email protected]" "[email protected]" true)" = "[email protected]" ]
    clerum_initial_setup_link_matches active initial_setup desktop-id admin-id admin-id
    refute clerum_initial_setup_link_matches revoked initial_setup desktop-id admin-id admin-id
    refute clerum_initial_setup_link_matches active revoked desktop-id admin-id admin-id
    refute clerum_initial_setup_link_matches active initial_setup desktop-id "" admin-id
    refute clerum_initial_setup_link_matches active initial_setup "" admin-id admin-id
    refute clerum_initial_setup_link_matches active initial_setup desktop-id other-admin admin-id
    [ "$(clerum_minimal_setup_outcome 201)" = setup_succeeded ]
    [ "$(clerum_minimal_setup_outcome 409)" = setup_already_consumed ]
    [ "$(clerum_minimal_setup_outcome 401)" = setup_failed ]
    printf ok
  ' bash "$contract")"
  if [ "$output" = "ok" ] && \
     ! grep -Eq '\$\{[A-Za-z_][A-Za-z0-9_]*,,\}' "$REPO_ROOT/scripts/e2e/seed-e2e-data.sh" "$REPO_ROOT/scripts/minikube/full-setup.sh"; then
    pass "minimal bootstrap canonicalization and link contract run on system Bash 3.2"
  else
    fail "minimal bootstrap still contains a Bash-4 lowercase expansion or contract failure"
  fi
}

assert_minimal_seed_rejects_a_divergent_identity() {
  local seed="$REPO_ROOT/scripts/e2e/seed-e2e-data.sh"
  local output rc=0
  output="$(CONTEXT=clerum-test SEED_PROFILE=minimal \
    ADMIN_EMAIL='[email protected]' E2E_DEV_LOGIN_EMAIL='[email protected]' \
    ADMIN_PASSWORD='test-password-only' /bin/bash "$seed" 2>&1)" || rc=$?
  if [ "$rc" -ne 0 ] && grep -Fq 'requires the Desktop identity email' <<<"$output"; then
    pass "minimal seeder rejects a Desktop identity that differs from the bootstrap admin"
  else
    fail "minimal seeder accepted a divergent Desktop identity (rc=$rc output=$output)"
  fi
}

assert_minimal_setup_requires_admin_identity_email() {
  local contract="$REPO_ROOT/scripts/e2e/minimal-bootstrap-contract.sh"
  local output

  # Exercise the decision both call sites now share, rather than grepping for
  # the source line that expresses it. A grep passes on any refactor that keeps
  # the text and changes the meaning.
  if ! output="$(/bin/bash -c '
    set -e
    source "$1"
    refute() { if "$@"; then printf "unexpected success: %s\n" "$*" >&2; exit 1; fi; }
    clerum_minimal_identity_matches [email protected] [email protected]
    refute clerum_minimal_identity_matches [email protected] [email protected]
    refute clerum_minimal_identity_matches [email protected] ""
    refute clerum_minimal_identity_matches "" [email protected]
    refute clerum_minimal_identity_matches "" ""
    case "$(clerum_minimal_identity_error [email protected] [email protected])" in
      *"Desktop identity email ([email protected])"*"bootstrap admin email ([email protected])"*) ;;
      *) exit 1 ;;
    esac
    printf ok
  ' bash "$contract")"; then
    fail "minimal identity guard does not accept the admin identity and reject every divergence"
    return
  fi
  if [ "$output" != "ok" ]; then
    fail "minimal identity guard contract did not complete (output=$output)"
    return
  fi
  # Both enforcement points must route through the shared guard, so a fix in
  # one cannot silently leave the other permissive.
  if ! grep -Fq 'clerum_minimal_identity_matches "$ADMIN_EMAIL" "$SEED_USER_EMAIL"' \
      "$REPO_ROOT/scripts/minikube/full-setup.sh" ||
    ! grep -Fq 'clerum_minimal_identity_matches "$ADMIN_EMAIL" "$DEV_EMAIL"' \
      "$REPO_ROOT/scripts/e2e/seed-e2e-data.sh"; then
    fail "a minimal enforcement point does not use the shared identity guard"
    return
  fi
  pass "minimal setup refuses a second unlinked Desktop identity"
}

assert_an_unknown_image_source_is_a_hard_error() {
  local out rc
  out="$(env PATH="$FULL_SETUP_STUB_DIR:$PATH" IMAGE_SOURCE=gchr \
    MINIKUBE_FULL_SETUP_CONFIG_ONLY=true \
    bash "$REPO_ROOT/scripts/minikube/full-setup.sh" 2>&1)"; rc=$?
  if [ "$rc" -ne 0 ] && grep -qi "IMAGE_SOURCE" <<< "$out"; then
    pass "an unknown IMAGE_SOURCE fails loudly instead of falling back"
  else
    fail "expected a hard error naming IMAGE_SOURCE; got rc=$rc out='$out'"
  fi
}

# Only the RENDER dir moves. The kustomize dir is where the generated
# k8s-api-ip.yaml lands and where the template lives; moving it would break
# minikube-detect-k8s-api-ip.sh and the patchesStrategicMerge reference.
assert_ghcr_mode_moves_only_the_render_dir() {
  local render kustomize
  render="$(full_setup_resolves ACTIVE_MINIKUBE_RENDER_DIR)"
  kustomize="$(full_setup_resolves ACTIVE_MINIKUBE_KUSTOMIZE_DIR)"
  if [[ "$render" == */overlays/minikube-ghcr ]] && [[ "$kustomize" == */overlays/minikube ]]; then
    pass "ghcr mode renders minikube-ghcr while the kustomize dir stays overlays/minikube"
  else
    fail "render='${render}' kustomize='${kustomize}'"
  fi
}

assert_ghcr_mode_with_skip_uis_renders_the_no_uis_ghcr_overlay() {
  local render
  render="$(full_setup_resolves ACTIVE_MINIKUBE_RENDER_DIR MINIKUBE_SKIP_UIS=true)"
  if [[ "$render" == */overlays/minikube-no-uis-ghcr ]]; then
    pass "ghcr + --skip-uis renders minikube-no-uis-ghcr"
  else
    fail "render dir was '${render}'"
  fi
}

assert_local_mode_renders_the_unchanged_overlays() {
  local a b
  a="$(full_setup_resolves ACTIVE_MINIKUBE_RENDER_DIR IMAGE_SOURCE=local)"
  b="$(full_setup_resolves ACTIVE_MINIKUBE_RENDER_DIR IMAGE_SOURCE=local MINIKUBE_SKIP_UIS=true)"
  if [[ "$a" == */overlays/minikube ]] && [[ "$b" == */overlays/minikube-no-uis ]]; then
    pass "local mode renders today's overlays unchanged"
  else
    fail "local render dirs were '${a}' and '${b}'"
  fi
}

# Never `rm -rf "$(dirname "$1")"` unguarded: `dirname ""` is `.`, so an
# assertion that failed to resolve a deploy dir would delete the caller's
# working directory. Only remove a path that is an absolute */deploy outside
# the repo.
remove_override_copy() {
  local deploy_dir="${1:-}"
  case "$deploy_dir" in
    /*/deploy) ;;
    *) return 0 ;;
  esac
  [ "$deploy_dir" = "$REPO_ROOT/deploy" ] && return 0
  rm -rf "$(dirname "$deploy_dir")"
}

# The override must never touch the committed tree. A single in-place `sed` on
# deploy/components/ghcr-images/kustomization.yaml would make the operator lever
# a second writer of the release coordinate.
assert_the_tag_override_copies_the_tree_and_leaves_the_commit_alone() {
  local before after deploy_dir
  before="$(git -C "$REPO_ROOT" status --porcelain deploy/components/ghcr-images | wc -l | tr -d ' ')"
  deploy_dir="$(full_setup_resolves ACTIVE_MINIKUBE_DEPLOY_DIR MINIKUBE_IMAGE_TAG=latest)"
  after="$(git -C "$REPO_ROOT" status --porcelain deploy/components/ghcr-images | wc -l | tr -d ' ')"
  if [ "$before" = "$after" ] && [ -n "$deploy_dir" ] && [[ "$deploy_dir" != "$REPO_ROOT/deploy" ]]; then
    pass "MINIKUBE_IMAGE_TAG renders from a temp copy and leaves the committed component untouched"
  else
    fail "deploy dir was '${deploy_dir}' (repo deploy is $REPO_ROOT/deploy); git status lines before=$before after=$after"
  fi
  remove_override_copy "$deploy_dir"
}

# The whole tree must move: resources: ../minikube and
# deploy/scripts/minikube-detect-k8s-api-ip.sh both resolve relative to the
# deploy root, so copying only the component would break both.
assert_the_tag_override_copy_is_a_whole_deploy_tree_with_the_new_tag() {
  local deploy_dir tag problems=""
  deploy_dir="$(full_setup_resolves ACTIVE_MINIKUBE_DEPLOY_DIR MINIKUBE_IMAGE_TAG=v9.9.9)"
  [ -d "$deploy_dir/overlays/minikube" ] || problems+="no overlays/minikube in the copy; "
  [ -d "$deploy_dir/overlays/minikube-ghcr" ] || problems+="no overlays/minikube-ghcr in the copy; "
  [ -f "$deploy_dir/scripts/minikube-detect-k8s-api-ip.sh" ] || problems+="no deploy/scripts in the copy; "
  # No 2>/dev/null here: if the component is missing from the copy, sed's error
  # belongs in the transcript, not swallowed into an empty-string "mismatch".
  tag="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\).*$/\1/p' \
    "$deploy_dir/components/ghcr-images/kustomization.yaml" | sort -u | tr '\n' ' ')"
  [ "$tag" = "v9.9.9 " ] || problems+="copy's newTag is '${tag}', expected 'v9.9.9 '; "
  if [ -z "$problems" ]; then
    pass "the override copy is a whole deploy tree with every newTag rewritten"
  else
    fail "$problems"
  fi
  remove_override_copy "$deploy_dir"
}

# "Run without --skip-build to rebuild" is wrong advice in ghcr mode: nothing
# was built, and rebuilding is exactly what the default path exists to avoid.
#
# A grep-only check over the OLD wide awk range (from the "Even with
# --skip-build" comment down to the terminating elif/else) is satisfied by
# text that lives entirely in the manifest-ABSENT sibling sub-block
# ("Run without --skip-build, or 'make minikube-pull-images'..."), so it stays
# green even if the manifest-PRESENT ghcr/local arms below are swapped or
# deleted outright (both verified live). The range is narrowed here to just
# the manifest-PRESENT "NEWEST_SRC found" if/else/fi -- anchored on the
# mode-independent `-n "$NEWEST_SRC"` text, with the closing `fi` found by
# depth-counting `; then`/`fi` pairs rather than by line number or
# indentation -- and then the extracted REAL source is `eval`'d with stub
# warn()/log() functions and IMAGE_SOURCE set to each mode in turn. That is
# genuine evaluated behaviour: a swap of the `= ghcr` operand (or a deletion
# of the arms) changes what gets printed for a given mode, not just what text
# is reachable somewhere in the block.
extract_full_setup_staleness_arms_block() {
  local sh="$1"
  awk '
    $0 ~ /-n "\$NEWEST_SRC"/ && !cap { cap = 1; depth = 0 }
    cap {
      print
      line = $0
      gsub(/^[ \t]+|[ \t]+$/, "", line)
      if (line ~ /; *then$/) depth++
      if (line == "fi") { depth--; if (depth == 0) exit }
    }
  ' "$sh"
}

run_full_setup_staleness_block() {
  local mode="$1" block="$2" tmpfile out
  tmpfile="$(mktemp)"
  {
    printf 'warn() { printf "WARN:%%s\\n" "$*"; }\n'
    printf 'log() { printf "LOG:%%s\\n" "$*"; }\n'
    printf '%s\n' "$block"
  } > "$tmpfile"
  out="$(IMAGE_SOURCE="$mode" NEWEST_SRC=fake/newer/file EFFECTIVE_IMAGE_TAG=v1.2.3 bash "$tmpfile" 2>&1)"
  rm -f "$tmpfile"
  printf '%s' "$out"
}

assert_the_skip_build_staleness_advice_is_mode_aware() {
  local sh block ghcr_out local_out problem=""
  sh="$REPO_ROOT/scripts/minikube/full-setup.sh"
  block="$(extract_full_setup_staleness_arms_block "$sh")"
  if [ -z "$block" ]; then
    fail "could not locate the --skip-build staleness NEWEST_SRC arms in $sh"
    return
  fi

  ghcr_out="$(run_full_setup_staleness_block ghcr "$block")"
  local_out="$(run_full_setup_staleness_block local "$block")"

  grep -q 'pre-gate-sync' <<< "$ghcr_out" || problem+="ghcr advice (evaluated) is missing the pre-gate-sync mention; "
  grep -q 'minikube-setup-local' <<< "$ghcr_out" || problem+="ghcr advice (evaluated) is missing the minikube-setup-local mention; "
  grep -qi 'rebuild' <<< "$ghcr_out" && problem+="ghcr advice (evaluated) wrongly says rebuild: $ghcr_out; "

  grep -qi 'rebuild' <<< "$local_out" || problem+="local advice (evaluated) is missing rebuild guidance: $local_out; "
  grep -q 'pre-gate-sync' <<< "$local_out" && problem+="local advice (evaluated) wrongly mentions pre-gate-sync: $local_out; "

  if [ -z "$problem" ]; then
    pass "the --skip-build staleness warning branches on IMAGE_SOURCE (evaluated per mode, not just mentioned)"
  else
    fail "$problem"
  fi
}

# A contributor must be able to tell from the transcript which images the
# cluster is about to run and where the tag came from.
#
# Grepping the source for `print_image_source_banner` and `TAG_ORIGIN` only
# proves those names exist SOMEWHERE in the file -- the function definition
# and the TAG_ORIGIN= assignments satisfy both greps even if the call site
# that actually invokes the banner is deleted (verified live: removing the
# bare `print_image_source_banner` call still leaves both greps green). This
# instead runs the real script through the existing config-only seam (see
# `full_setup_resolves` above) and asserts on the CAPTURED STDOUT the banner
# writes -- observed output, not source text -- so a deleted call site prints
# nothing and the assertion fails.
#
# Runs from a copy with NO recorded manifest (see make_full_setup_copy): the
# tag now falls back to the committed pin only when nothing is recorded, and
# the manifest is gitignored, so against $REPO_ROOT this case would assert the
# pin in CI and the last-pulled tag on a developer machine.
assert_the_banner_prints_the_image_source_tag_and_origin() {
  local d out committed_tag problem=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d"
  out="$(full_setup_copy_output "$d" '')"
  committed_tag="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\).*$/\1/p' \
    "$REPO_ROOT/deploy/components/ghcr-images/kustomization.yaml" | sort -u | head -1)"
  grep -q 'Images: PULLED from ghcr.io/evenfire-ai' <<< "$out" || problem+="banner did not name the image source; "
  [ -n "$committed_tag" ] && grep -q "tag: *${committed_tag}" <<< "$out" || problem+="banner did not print the committed tag '${committed_tag}'; "
  grep -q 'origin: *committed pin in deploy/components/ghcr-images/kustomization.yaml' <<< "$out" || problem+="banner did not name the tag's origin; "
  if [ -z "$problem" ]; then
    pass "the setup prints an image-source banner naming source, tag and origin (observed stdout)"
  else
    fail "$problem got: $out"
  fi
  rm -rf "$d"
}

assert_the_tag_origin_distinguishes_pin_from_override() {
  local d pinned overridden
  d="$(mktemp -d)"
  make_full_setup_copy "$d"
  pinned="$(full_setup_copy_resolves "$d" TAG_ORIGIN '')"
  overridden="$(full_setup_copy_resolves "$d" TAG_ORIGIN '' MINIKUBE_IMAGE_TAG=latest)"
  if grep -qi "pin" <<< "$pinned" && grep -q "MINIKUBE_IMAGE_TAG" <<< "$overridden"; then
    pass "TAG_ORIGIN distinguishes the committed pin from the override"
  else
    fail "TAG_ORIGIN was '${pinned}' (pinned) and '${overridden}' (overridden)"
  fi
  rm -rf "$d"
}

# ---------------------------------------------------------------------------
# THE MODE AND THE TAG FOLLOW THE CLUSTER
# ---------------------------------------------------------------------------

# FIX: --skip-build acquires nothing, so the overlay must follow the images
# already on the node. Taking IMAGE_SOURCE instead rendered the -ghcr overlays
# over a locally built cluster, pointing every Deployment at a ghcr ref nothing
# had pulled. It is not a rare path: `make minikube-setup` ALWAYS passes
# IMAGE_SOURCE=ghcr (the Makefile defaults it), which is exactly the value used
# here.
#
# Semantic mutation coverage: `if false && [ "$SKIP_BUILD" = true ]`, or
# swapping `!=` for `=` in the recorded-vs-requested comparison, or inverting
# image_mode_source's `[ -n "$recorded" ]` -- each one puts the ghcr overlay
# back and this fails.
assert_skip_build_follows_the_recorded_image_source() {
  local d render problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest local "")"
  render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR --skip-build IMAGE_SOURCE=ghcr)"
  [[ "$render" == */overlays/minikube ]] || problems+="a local-recorded cluster rendered '${render}'; "

  # And the mirror image, so the fix cannot be "always render the local
  # overlay": a ghcr-recorded cluster redeployed from an IMAGE_SOURCE=local
  # shell must render the ghcr overlay.
  rm -rf "$d"; d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest ghcr latest)"
  render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR --skip-build IMAGE_SOURCE=local)"
  [[ "$render" == */overlays/minikube-ghcr ]] || problems+="a ghcr-recorded cluster rendered '${render}'; "

  if [ -z "$problems" ]; then
    pass "--skip-build renders the overlay the cluster's recorded images need, in both directions"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# The operator has to be told, or a redeploy that quietly ignores IMAGE_SOURCE
# is indistinguishable from one that honoured it.
assert_skip_build_says_it_is_following_the_cluster() {
  local d out problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest local "")"
  out="$(full_setup_copy_output "$d" --skip-build IMAGE_SOURCE=ghcr)"
  grep -q 'acquired as IMAGE_SOURCE=local' <<< "$out" || problems+="no warning naming the recorded mode; "
  grep -q 'BUILT LOCALLY from source' <<< "$out" || problems+="the banner did not follow the recorded mode; "

  # Silence when there is nothing to say: a matching mode must not warn, or the
  # warning becomes noise everyone learns to ignore.
  rm -rf "$d"; d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest ghcr latest)"
  out="$(full_setup_copy_output "$d" --skip-build IMAGE_SOURCE=ghcr)"
  grep -q 'acquired as IMAGE_SOURCE=' <<< "$out" && problems+="warned even though the modes agree; "

  if [ -z "$problems" ]; then
    pass "--skip-build warns when it overrides IMAGE_SOURCE, and stays quiet when it does not"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# The anti-regression for `make minikube-setup-local` over a pulled cluster: a
# run that ACQUIRES images is about to rewrite the record, so the environment
# still decides. Mutation: drop the `[ "$SKIP_BUILD" = true ]` guard and this
# resolves ghcr from the record, making a mode switch impossible.
assert_an_acquiring_run_still_honours_image_source() {
  local d render problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest ghcr latest)"
  render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR '' IMAGE_SOURCE=local)"
  [[ "$render" == */overlays/minikube ]] || problems+="IMAGE_SOURCE=local over a ghcr record rendered '${render}'; "

  rm -rf "$d"; d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest local "")"
  render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR '' IMAGE_SOURCE=ghcr)"
  [[ "$render" == */overlays/minikube-ghcr ]] || problems+="IMAGE_SOURCE=ghcr over a local record rendered '${render}'; "

  if [ -z "$problems" ]; then
    pass "a run that acquires images still honours IMAGE_SOURCE, so a cluster can change modes"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# The fresh-cluster path: on a FIRST run there is no manifest at all, so the
# env default must still apply -- including under --skip-build. Mutation: make
# image_mode_source ignore its env fallback and both halves break.
assert_a_cluster_with_no_record_keeps_the_env_default() {
  local d ghcr_render local_render problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d"
  ghcr_render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR --skip-build)"
  local_render="$(full_setup_copy_resolves "$d" ACTIVE_MINIKUBE_RENDER_DIR --skip-build IMAGE_SOURCE=local)"
  [[ "$ghcr_render" == */overlays/minikube-ghcr ]] || problems+="default rendered '${ghcr_render}'; "
  [[ "$local_render" == */overlays/minikube ]] || problems+="IMAGE_SOURCE=local rendered '${local_render}'; "
  if [ -z "$problems" ]; then
    pass "with nothing recorded, --skip-build still follows the IMAGE_SOURCE default and override"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# FIX: EFFECTIVE_IMAGE_TAG came from the committed pin and ignored the tag the
# cluster recorded, so after the documented `MINIKUBE_IMAGE_TAG=latest make
# minikube-setup` bootstrap every LATER run reported and acted on v0.6.0
# against a cluster running :latest.
#
# Mutation coverage: swap the precedence in image_mode_ghcr_tag (pin before
# record), or invert its `[ -n "$recorded" ]`, and this reads the pin.
assert_the_effective_tag_comes_from_the_record_not_the_pin() {
  local d tag origin pin problems=""
  d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest ghcr recorded-test-tag)"
  tag="$(full_setup_copy_resolves "$d" EFFECTIVE_IMAGE_TAG '')"
  origin="$(full_setup_copy_resolves "$d" TAG_ORIGIN '')"
  pin="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\).*$/\1/p' \
    "$REPO_ROOT/deploy/components/ghcr-images/kustomization.yaml" | sort -u | head -1)"
  [ "$tag" = "recorded-test-tag" ] || problems+="EFFECTIVE_IMAGE_TAG was '${tag}', expected the recorded tag (pin is '${pin}'); "
  grep -q 'deploy/minikube/.image-manifest.json' <<< "$origin" || problems+="TAG_ORIGIN was '${origin}', which does not name the manifest; "
  if [ -z "$problems" ]; then
    pass "EFFECTIVE_IMAGE_TAG is the tag the cluster recorded, and TAG_ORIGIN names the manifest"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# The floor and the override, so the precedence is pinned from both ends.
assert_the_effective_tag_precedence_is_override_then_record_then_pin() {
  local d pin from_pin from_override problems=""
  pin="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\).*$/\1/p' \
    "$REPO_ROOT/deploy/components/ghcr-images/kustomization.yaml" | sort -u | head -1)"
  d="$(mktemp -d)"
  make_full_setup_copy "$d"
  from_pin="$(full_setup_copy_resolves "$d" EFFECTIVE_IMAGE_TAG '')"
  [ -n "$pin" ] && [ "$from_pin" = "$pin" ] || problems+="with nothing recorded the tag was '${from_pin}', expected the pin '${pin}'; "

  rm -rf "$d"; d="$(mktemp -d)"
  make_full_setup_copy "$d" "$(recorded_manifest ghcr recorded-test-tag)"
  from_override="$(full_setup_copy_resolves "$d" EFFECTIVE_IMAGE_TAG '' MINIKUBE_IMAGE_TAG=v9.9.9)"
  [ "$from_override" = "v9.9.9" ] || problems+="MINIKUBE_IMAGE_TAG did not beat the record; got '${from_override}'; "

  if [ -z "$problems" ]; then
    pass "the effective tag is MINIKUBE_IMAGE_TAG, else the recorded tag, else the committed pin"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# ---------------------------------------------------------------------------
# --skip-uis reaches the PULL path
# ---------------------------------------------------------------------------
#
# The -no-uis-ghcr overlay deletes both UI Deployments, so pulling control-ui
# and profile-ui costs ~470 MiB for images nothing references. The local build
# path has always passed --skip-uis through; the pull path did not.
#
# Step 5's ghcr arm needs a cluster to run, so the REAL arm is extracted and
# eval'd with a stub pull-images.sh that records its argv. That is evaluated
# behaviour, not a grep for the flag's name somewhere in the file: deleting the
# `if [ "$SKIP_UIS" = true ]` guard, or inverting it, changes what the stub is
# called with.
extract_full_setup_pull_invocation() {
  awk '
    /log "Pulling published images/ { cap = 1 }
    cap { print }
    cap && /ok "All published images pulled"/ { exit }
  ' "$1"
}

run_full_setup_pull_invocation() {
  local d=$1 skip_uis=$2 script
  mkdir -p "$d/bin"
  cat > "$d/bin/pull-images.sh" <<'STUB'
#!/usr/bin/env bash
printf 'pull-images.sh %s\n' "$*" >>"${TEST_LOG_FILE:?}"
exit 0
STUB
  chmod +x "$d/bin/pull-images.sh"
  script="$d/pull-arm.sh"
  {
    printf 'set -euo pipefail\n'
    printf 'log() { printf "LOG:%%s\\n" "$*"; }\n'
    printf 'ok() { printf "OK:%%s\\n" "$*"; }\n'
    printf 'SCRIPT_DIR=%q\n' "$d/bin"
    printf 'PROFILE="clerum-test"\n'
    printf 'EFFECTIVE_IMAGE_TAG="v0.0.0-test"\n'
    printf 'SKIP_UIS=%q\n' "$skip_uis"
    extract_full_setup_pull_invocation "$REPO_ROOT/scripts/minikube/full-setup.sh"
  } > "$script"
  TEST_LOG_FILE="$d/ops.log" bash "$script" 2>&1
}

assert_the_pull_path_forwards_skip_uis() {
  local d out rc problems=""
  d="$(mktemp -d)"
  if [ -z "$(extract_full_setup_pull_invocation "$REPO_ROOT/scripts/minikube/full-setup.sh")" ]; then
    fail "could not locate the ghcr pull invocation in full-setup.sh"
    rm -rf "$d"
    return
  fi

  out="$(run_full_setup_pull_invocation "$d" true)"; rc=$?
  [ "$rc" -eq 0 ] || problems+="the extracted pull arm exited ${rc} with SKIP_UIS=true: ${out}; "
  grep -q '^pull-images.sh --skip-uis$' "$d/ops.log" \
    || problems+="SKIP_UIS=true did not forward --skip-uis (log: $(cat "$d/ops.log")); "

  # The complement, so the fix cannot be "always pass --skip-uis". Also proves
  # the empty-array expansion does not abort under `set -u`.
  rm -rf "$d"; d="$(mktemp -d)"
  out="$(run_full_setup_pull_invocation "$d" false)"; rc=$?
  [ "$rc" -eq 0 ] || problems+="the extracted pull arm exited ${rc} with SKIP_UIS=false: ${out}; "
  grep -q '^pull-images.sh $' "$d/ops.log" \
    || problems+="SKIP_UIS=false did not call the puller with no flags (log: $(cat "$d/ops.log")); "

  if [ -z "$problems" ]; then
    pass "the ghcr pull path forwards --skip-uis only when SKIP_UIS is set (evaluated argv)"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

# ---------------------------------------------------------------------------
# The GENERATED k8s-api-ip.yaml under a tag override
# ---------------------------------------------------------------------------
#
# patches/k8s-api-ip.yaml is generated and gitignored; overlays/minikube commits
# only the .template, and overlays/minikube-ghcr renders `../minikube`, which
# patches with it. With MINIKUBE_IMAGE_TAG set, ACTIVE_MINIKUBE_KUSTOMIZE_DIR is
# the mktemp copy, so Step 6b used to write the patch ONLY there and the working
# tree never got one. The next `make minikube-pre-gate-sync` renders a FRESH
# copy of ${PROJECT_DIR}/deploy (image-mode.sh) for the control-api migration,
# BEFORE `make minikube-deploy-all` regenerates the patch, and kustomize dies on
# `evalsymlink failure on .../patches/k8s-api-ip.yaml`.
#
# These cases run the REAL Step 6b block and the REAL
# deploy/scripts/minikube-detect-k8s-api-ip.sh against a stubbed kubectl, then
# assert on the FILES that appear on disk and on how many times the detector
# ran. Inverting the condition (or wrapping it in `if false && ...`) changes
# both, so neither assertion survives a semantic mutation.
extract_full_setup_k8s_api_ip_block() {
  awk '
    /Refreshing minikube K8s API endpoint CIDRs/ { cap = 1 }
    cap { print }
    cap && /Minikube K8s API CIDRs refreshed/ { exit }
  ' "$1"
}

# A throwaway PROJECT_DIR carrying the real detector and the real patch
# template, plus a kubectl stub that answers the one endpoint read the detector
# makes and logs every call so invocations can be counted.
prepare_k8s_api_ip_repo() {
  local d=$1
  mkdir -p "$d/bin" "$d/project/deploy/scripts" "$d/project/deploy/overlays/minikube/patches"
  cp "$REPO_ROOT/deploy/scripts/minikube-detect-k8s-api-ip.sh" "$d/project/deploy/scripts/"
  cp "$REPO_ROOT/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template" \
    "$d/project/deploy/overlays/minikube/patches/"
  cat > "$d/bin/kubectl" <<'STUB'
#!/usr/bin/env bash
printf 'kubectl %s\n' "$*" >>"${TEST_LOG_FILE:?}"
case "$*" in
  *"get endpoints kubernetes"*) echo "10.11.12.13"; exit 0 ;;
esac
exit 0
STUB
  chmod +x "$d/bin/kubectl"
}

# The mktemp copy apply_image_tag_override would have produced.
prepare_k8s_api_ip_override_copy() {
  local d=$1
  mkdir -p "$d/override/deploy/overlays/minikube/patches"
  cp "$REPO_ROOT/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template" \
    "$d/override/deploy/overlays/minikube/patches/"
}

run_k8s_api_ip_block() {
  local d=$1 active=$2 script
  script="$d/step6b.sh"
  {
    printf 'set -euo pipefail\n'
    printf 'log() { printf "LOG:%%s\\n" "$*"; }\n'
    printf 'ok() { printf "OK:%%s\\n" "$*"; }\n'
    printf 'PROFILE="clerum-test"\n'
    printf 'PROJECT_DIR=%q\n' "$d/project"
    printf 'ACTIVE_MINIKUBE_KUSTOMIZE_DIR=%q\n' "$active"
    extract_full_setup_k8s_api_ip_block "$REPO_ROOT/scripts/minikube/full-setup.sh"
  } > "$script"
  PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" bash "$script" 2>&1
}

assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_tree() {
  local d out rc calls problems=""
  d="$(mktemp -d)"
  prepare_k8s_api_ip_repo "$d"
  prepare_k8s_api_ip_override_copy "$d"
  out="$(run_k8s_api_ip_block "$d" "$d/override/deploy/overlays/minikube")"
  rc=$?
  calls="$(grep -c 'get endpoints kubernetes' "$d/ops.log")"
  [ "$rc" -eq 0 ] || problems+="step 6b block exited $rc; "
  grep -q '10.11.12.13/32' "$d/override/deploy/overlays/minikube/patches/k8s-api-ip.yaml" \
    || problems+="the render copy did not get the generated patch; "
  # The whole point: the working tree gets one too, so image-mode.sh's copy and
  # the pre-gate migration render both find it.
  grep -q '10.11.12.13/32' "$d/project/deploy/overlays/minikube/patches/k8s-api-ip.yaml" \
    || problems+="the working tree did not get the generated patch; "
  [ "$calls" = "2" ] || problems+="detector ran ${calls} time(s), expected 2; "
  if [ -z "$problems" ]; then
    pass "a tag override generates k8s-api-ip.yaml in the render copy AND in the working tree"
  else
    fail "$problems out=$out"
  fi
  rm -rf "$d"
}

# The complement, so the fix cannot be "always run it twice": with no override
# the active kustomize dir IS the working tree, and one generation is enough.
assert_the_unoverridden_path_generates_the_api_ip_patch_once() {
  local d out rc calls problems=""
  d="$(mktemp -d)"
  prepare_k8s_api_ip_repo "$d"
  out="$(run_k8s_api_ip_block "$d" "$d/project/deploy/overlays/minikube")"
  rc=$?
  calls="$(grep -c 'get endpoints kubernetes' "$d/ops.log")"
  [ "$rc" -eq 0 ] || problems+="step 6b block exited $rc; "
  grep -q '10.11.12.13/32' "$d/project/deploy/overlays/minikube/patches/k8s-api-ip.yaml" \
    || problems+="the working tree did not get the generated patch; "
  [ "$calls" = "1" ] || problems+="detector ran ${calls} time(s), expected 1; "
  if [ -z "$problems" ]; then
    pass "the unoverridden path generates k8s-api-ip.yaml exactly once, in the working tree"
  else
    fail "$problems out=$out"
  fi
  rm -rf "$d"
}

assert_trace_writer_fence_closes_and_proves_zero_replicas() {
  local fence_block
  fence_block="$(sed -n '/^fence_partial_trace_worker()/,/^}/p' scripts/minikube/full-setup.sh)"
  if grep -Fq -- '--replicas=0' <<<"$fence_block" && \
     grep -Fq "jsonpath={.spec.replicas}" <<<"$fence_block" && \
     grep -Fq 'trace-maintenance-worker did not converge to zero desired replicas' <<<"$fence_block" && \
     grep -Fq 'trace-maintenance-worker pods remain after the zero-replica fence' <<<"$fence_block"; then
    pass "trace-maintenance-worker recovery fence scales to zero and proves no writer pod remains"
  else
    fail "trace-maintenance-worker recovery can be marked fenced without proving a zero-replica state"
  fi
}

assert_trace_writer_fence_is_behaviorally_fail_closed() {
  local d fence_block run_script out rc problems=""
  d="$(mktemp -d)"
  fence_block="$(sed -n '/^fence_partial_trace_worker()/,/^}/p' scripts/minikube/full-setup.sh)"

  mkdir -p "$d/bin"
  cat > "$d/bin/kubectl" <<'STUB'
#!/usr/bin/env bash
set -u
state_dir="${TEST_TRACE_STATE_DIR:?}"
case "$*" in
  *"scale deployment/trace-maintenance-worker --replicas=0"*)
    : >"$state_dir/scaled"
    exit 0
    ;;
  *"get deployment/trace-maintenance-worker"*)
    printf '%s' "${TEST_TRACE_DESIRED:-0}"
    exit 0
    ;;
  *"get pods -l app=trace-maintenance-worker"*)
    if [ -f "$state_dir/waited" ]; then
      exit 0
    fi
    printf '%s' "${TEST_TRACE_PODS:-}"
    exit 0
    ;;
  *"wait --for=delete pod"*)
    : >"$state_dir/waited"
    exit 0
    ;;
esac
exit 0
STUB
  chmod +x "$d/bin/kubectl"

  run_script="$d/run.sh"
  {
    printf 'set -u\n'
    printf 'KC=%q\n' 'kubectl --context=test'
    printf 'PARTIAL_TRACE_REPLICAS=2\n'
    printf 'PARTIAL_TRACE_FENCED=false\n'
    printf 'STATE_DIR=%q\n' "$d"
    printf 'log() { :; }\n'
    printf 'err() { printf "ERR:%%s\\n" "$*" >&2; }\n'
    printf 'writer_recovery_state_phase() { printf "%%s\\n" "$1" >>"$STATE_DIR/phases"; }\n'
    printf '%s\n' "$fence_block"
    printf 'fence_partial_trace_worker\n'
  } >"$run_script"

  out="$(TEST_TRACE_STATE_DIR="$d" TEST_TRACE_DESIRED=0 TEST_TRACE_PODS='pod/trace-0' \
    PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -eq 0 ] || problems+="zero-replica convergence exited $rc; "
  [ -f "$d/scaled" ] || problems+="zero-replica convergence never scaled; "
  grep -Fxq 'trace-fenced' "$d/phases" || problems+="successful fence never persisted trace-fenced; "

  rm -f "$d/scaled" "$d/waited" "$d/phases"
  out="$(TEST_TRACE_STATE_DIR="$d" TEST_TRACE_DESIRED=1 TEST_TRACE_PODS= \
    PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || problems+="nonzero desired replicas were accepted; "
  if [ -f "$d/phases" ] && grep -Fxq 'trace-fenced' "$d/phases"; then
    problems+="trace-fenced was persisted after nonzero convergence; "
  fi
  grep -Fq 'did not converge to zero desired replicas' <<<"$out" \
    || problems+="nonzero convergence did not report the stable failure; "

  if [ -z "$problems" ]; then
    pass "trace-maintenance-worker fence proves convergence and rejects nonzero desired replicas"
  else
    fail "$problems out=$out"
  fi
  rm -rf "$d"
}

assert_interrupted_recovery_resume_restores_all_writer_fences() {
  local resume_block
  resume_block="$(sed -n '/^    api-restored|overlay-applying|overlay-applied)/,/^      ;;$/p' scripts/minikube/full-setup.sh)"
  if grep -Fq 'PARTIAL_HCC_FENCED=true' <<<"$resume_block" && \
     grep -Fq 'PARTIAL_WORKFLOW_FENCED=true' <<<"$resume_block" && \
     grep -Fq 'PARTIAL_TRACE_FENCED=true' <<<"$resume_block" && \
     grep -Fq 'PARTIAL_CONTROL_API_FENCED=true' <<<"$resume_block"; then
    pass "post-migration recovery resumes with all four writer fences active"
  else
    fail "post-migration recovery can omit a writer fence before restoring the journaled replicas"
  fi
}

assert_interrupted_recovery_fence_is_behaviorally_fail_closed() {
  local d helper run_script out rc problems=""
  d="$(mktemp -d)"
  helper="$(sed -n '/^guard_interrupted_writer_deployment()/,/^}/p' scripts/minikube/full-setup.sh)"

  mkdir -p "$d/bin"
  cat > "$d/bin/kubectl" <<'STUB'
#!/usr/bin/env bash
set -u
case "$*" in
  *"scale deployment/"*)
    [ "${TEST_FENCE_MODE:-}" = scale-fail ] && exit 1
    exit 0
    ;;
  *"get deployment/"*)
    printf '%s' "${TEST_FENCE_DESIRED:-0}"
    exit 0
    ;;
  *"get pods -l"*)
    printf '%s' "${TEST_FENCE_PODS:-}"
    exit 0
    ;;
  *"wait --for=delete pod"*)
    [ "${TEST_FENCE_MODE:-}" = wait-fail ] && exit 1
    exit 0
    ;;
esac
exit 0
STUB
  chmod +x "$d/bin/kubectl"

  run_script="$d/run.sh"
  {
    printf 'set -u\n'
    printf 'KC=%q\n' 'kubectl --context=test'
    printf 'log() { :; }\n'
    printf 'err() { printf "ERR:%%s\\n" "$*" >&2; }\n'
    printf '%s\n' "$helper"
    printf "guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller'\n"
  } >"$run_script"

  out="$(TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -eq 0 ] || problems+="healthy zero-replica fence failed; "

  out="$(TEST_FENCE_DESIRED=1 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || problems+="nonzero desired replicas were accepted; "
  grep -Fq 'did not converge to zero desired replicas' <<<"$out" \
    || problems+="nonzero desired replicas did not fail closed; "

  out="$(TEST_FENCE_MODE=scale-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= \
    PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || problems+="scale failure was swallowed; "

  out="$(TEST_FENCE_MODE=wait-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS='pod/hcc-0' \
    PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
  rc=$?
  [ "$rc" -ne 0 ] || problems+="wait failure was swallowed; "

  if [ -z "$problems" ]; then
    pass "interrupted recovery fencing fails closed on scale, convergence, and pod-wait failures"
  else
    fail "$problems"
  fi
  rm -rf "$d"
}

assert_recovery_migrations_are_head_bound() {
  local current_head_checks
  current_head_checks="$(grep -Fc '[ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]' scripts/minikube/full-setup.sh)"
  if grep -Fq 'writer_recovery_state_cli read --include-head' scripts/minikube/full-setup.sh && \
     grep -Fq 'PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD' scripts/minikube/full-setup.sh && \
     [ "$current_head_checks" -ge 2 ]; then
    pass "recovery resumes with writers fenced and reruns migrations after a HEAD change"
  else
    fail "recovery can reuse a historical migration boundary after HEAD changes"
  fi
}

# The guard that keeps this file honest: a case defined but never added to the
# call block below reports nothing at all, which reads as a green run.
assert_every_defined_case_is_invoked() {
  local self defined invoked missing
  self="$REPO_ROOT/scripts/tests/test-minikube-full-setup.sh"
  defined="$(grep -oE '^assert_[a-z_0-9]+\(\) \{' "$self" | sed -E 's/\(\) \{$//' | sort -u)"
  invoked="$(grep -oE '^assert_[a-z_0-9]+$' "$self" | sort -u)"
  missing="$(comm -23 <(printf '%s\n' "$defined") <(printf '%s\n' "$invoked"))"
  if [ -z "$missing" ]; then
    pass "every defined assert_ case is invoked in the call block"
  else
    fail "defined but never invoked: $(printf '%s ' $missing)"
  fi
}

assert_broken_profile_is_recreated
assert_healthy_profile_skips_recreate
assert_branch_profile_deploy_dir_is_used
assert_member_registration_hmac_is_patched
assert_branch_scoped_minikube_context_is_supported
assert_gfs_provisioning_follows_migrations_and_core_readiness
assert_mcp_proxy_bootstrap_wait_is_bounded_and_fail_closed
assert_full_setup_defaults_to_db_rebuild
assert_makefile_passes_reuse_db
assert_reuse_db_normalizer_precedes_flag_loop
assert_reset_db_flag_backcompat
assert_skip_build_staleness_find_is_sigpipe_guarded
assert_pipefail_head_guard_prevents_abort
assert_setup_runtime_operations_are_bounded
assert_ghcr_is_the_default_image_source
assert_image_source_local_is_honoured
assert_bootstrap_seed_deferral_is_opt_in
assert_bootstrap_seed_deferral_flag_resolves_for_local_minimal
assert_bootstrap_seed_deferral_rejects_non_local_or_e2e_modes
assert_minimal_seed_is_setup_first_and_link_fail_closed
assert_minimal_setup_requires_admin_identity_email
assert_minimal_bootstrap_contract_runs_on_system_bash
assert_minimal_seed_rejects_a_divergent_identity
assert_an_unknown_image_source_is_a_hard_error
assert_ghcr_mode_moves_only_the_render_dir
assert_ghcr_mode_with_skip_uis_renders_the_no_uis_ghcr_overlay
assert_local_mode_renders_the_unchanged_overlays
assert_the_tag_override_copies_the_tree_and_leaves_the_commit_alone
assert_the_tag_override_copy_is_a_whole_deploy_tree_with_the_new_tag
assert_the_skip_build_staleness_advice_is_mode_aware
assert_the_banner_prints_the_image_source_tag_and_origin
assert_the_tag_origin_distinguishes_pin_from_override
assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_tree
assert_the_unoverridden_path_generates_the_api_ip_patch_once
assert_trace_writer_fence_closes_and_proves_zero_replicas
assert_trace_writer_fence_is_behaviorally_fail_closed
assert_interrupted_recovery_resume_restores_all_writer_fences
assert_interrupted_recovery_fence_is_behaviorally_fail_closed
assert_recovery_migrations_are_head_bound
assert_skip_build_follows_the_recorded_image_source
assert_skip_build_says_it_is_following_the_cluster
assert_an_acquiring_run_still_honours_image_source
assert_a_cluster_with_no_record_keeps_the_env_default
assert_the_effective_tag_comes_from_the_record_not_the_pin
assert_the_effective_tag_precedence_is_override_then_record_then_pin
assert_the_pull_path_forwards_skip_uis
assert_every_defined_case_is_invoked

exit $FAIL