Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
49.8 KB
Raw Blame
#!/usr/bin/env bash
set -u
FAIL=0

# A ghcr cluster runs RELEASE images. Before this harness existed, the pre-gate
# had no idea: it built clerum/<svc>:test, restarted the Deployment, and the pod
# came back on the SAME release digest, because nothing referenced the image it
# had just built. The gate then passed against code that was never deployed --
# the worst outcome this whole change can produce.
#
# Every case here runs the REAL scripts against PATH stubs for
# docker/minikube/kubectl/make, in a throwaway copy of deploy/ + scripts/ with a
# real git history. Nothing needs a cluster or a network.
#
# The assertions are written to die on INVERTED LOGIC, not on a renamed
# identifier: they observe the docker/make/kubectl calls the scripts actually
# make and the refs those calls carry. Swapping the ghcr and local refs in the
# retag, or wrapping the shadow loop in `if false`, fails them.

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
GHCR_COMPONENT="$REPO_ROOT/deploy/components/ghcr-images/kustomization.yaml"

pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }

# Cleanup is part of fixture isolation and must affect the suite verdict.
cleanup_fixture() {
  local fixture_dir="$1"
  if ! rm -rf -- "$fixture_dir" || [[ -e "$fixture_dir" || -L "$fixture_dir" ]]; then
    fail "temporary fixture cleanup failed: $fixture_dir"
    return 1
  fi
}

PIN_TAG="$(sed -n 's/^[[:space:]]*newTag:[[:space:]]*\([^[:space:]]*\)[[:space:]]*$/\1/p' "$GHCR_COMPONENT" | sort -u)"
if [ -z "$PIN_TAG" ] || [ "$(printf '%s\n' "$PIN_TAG" | wc -l | tr -d ' ')" != "1" ]; then
  echo "FAIL: could not read a single committed pin from $GHCR_COMPONENT (got '$PIN_TAG')"
  exit 1
fi

# Derived from the manifest, deliberately NOT from the code under test.
ghcr_refs_for_selector() {
  local selector="$1" tag="$2"
  node -e '
    import("'"$REPO_ROOT"'/scripts/release/images-manifest.mjs").then(m => {
      for (const i of m.pullInGhcrMode()) {
        if (!i.name.includes(process.argv[1]) && !m.localRef(i).includes(process.argv[1])) continue
        console.log(`ghcr.io/evenfire-ai/${i.name}:${process.argv[2]}`)
      }
    }).catch(e => { console.error(e.message); process.exit(1) })' "$selector" "$tag"
}

local_ref_for_image() {
  node -e '
    import("'"$REPO_ROOT"'/scripts/release/images-manifest.mjs").then(m => {
      const i = m.IMAGES.find(x => x.name === process.argv[1])
      if (!i) { console.error("no such image"); process.exit(1) }
      console.log(m.localRef(i))
    }).catch(e => { console.error(e.message); process.exit(1) })' "$1"
}

make_stubs() {
  local d=$1
  mkdir -p "$d/bin"
  cat > "$d/bin/docker" <<'STUB'
#!/usr/bin/env bash
printf 'docker %s\n' "$*" >>"${TEST_LOG_FILE:?}"
case "${1:-}" in
  context)
    if [[ "${2:-}" == inspect ]]; then
      if [[ "$*" == *SkipTLSVerify* ]]; then
        printf 'unix:///tmp/evenfire-docker.sock\tfalse\t{}\n'
      else
        printf 'unix:///tmp/evenfire-docker.sock\n'
      fi
    fi
    ;;
  inspect)
    # The only inspect the pre-gate makes is the OCI revision-label read that
    # gives it the commit a release image was built from.
    printf '%s\n' "${TEST_IMAGE_REVISION:-}"
    exit 0
    ;;
  tag)
    if [ "${TEST_DOCKER_TAG_FAILS:-0}" = "1" ]; then
      echo "Error response from daemon: simulated retag failure" >&2
      exit 1
    fi
    exit 0
    ;;
esac
exit 0
STUB
  cat > "$d/bin/minikube" <<'STUB'
#!/usr/bin/env bash
printf 'minikube %s\n' "$*" >>"${TEST_LOG_FILE:?}"
case "$*" in
  *docker-env*)
    if [ "${TEST_MINIKUBE_HANG_DOCKER_ENV:-0}" = "1" ]; then
      trap '' TERM
      while :; do sleep 1; done
    fi
    if [ "${TEST_EMPTY_DOCKER_ENV:-0}" = "1" ]; then
      exit 0
    fi
    echo 'export DOCKER_HOST="tcp://127.0.0.1:2376"'
    exit 0
    ;;
esac
exit 0
STUB
  cat > "$d/bin/kubectl" <<'STUB'
#!/usr/bin/env bash
printf 'kubectl %s\n' "$*" >>"${TEST_LOG_FILE:?}"
case "$*" in
  *"get nodes"*) echo "minikube  Ready  control-plane  1d  v1.30.0"; exit 0 ;;
  *"get configmap"*jsonpath*)
    key="$(printf '%s' "$*" | sed -n 's/.*{\.data\.\([A-Za-z]*\)}.*/\1/p')"
    file="${TEST_MARKER_DIR:-/nonexistent}/${key}"
    [ -n "$key" ] || exit 1
    [ -f "$file" ] || exit 1
    cat "$file"
    exit 0
    ;;
esac
exit 0
STUB
  cat > "$d/bin/make" <<'STUB'
#!/usr/bin/env bash
printf 'make %s T2_SKIP_LOCK=%s T2_LOCK_TOKEN_SET=%s\n' \
  "$*" "${T2_SKIP_LOCK:-false}" "$([ -n "${T2_LOCK_TOKEN:-}" ] && printf true || printf false)" \
  >>"${TEST_LOG_FILE:?}"
exit 0
STUB
  chmod +x "$d/bin/docker" "$d/bin/minikube" "$d/bin/kubectl" "$d/bin/make"
}

# A throwaway repo carrying the REAL deploy/ and scripts/, with a real git
# history so incremental_plan's diffs work, and a stubbed build-images.sh so a
# "build" is observable without Docker.
prepare_repo() {
  local d=$1
  make_stubs "$d"
  mkdir -p "$d/repo"
  cp -R "$REPO_ROOT/deploy" "$d/repo/deploy"
  cp -R "$REPO_ROOT/scripts" "$d/repo/scripts"
  rm -rf "$d/repo/deploy/minikube"
  cat > "$d/repo/scripts/minikube/require-t2-mutation-lock.sh" <<'STUB'
#!/usr/bin/env bash
exit 0
STUB
  chmod +x "$d/repo/scripts/minikube/require-t2-mutation-lock.sh"
  # patches/k8s-api-ip.yaml is GENERATED and gitignored (the overlay commits
  # only the .template), so whether the developer's tree has one is incidental.
  # Materialise it deterministically: by the time anything renders, `make
  # minikube-setup` / `make minikube-deploy-all` have written it. Its absence is
  # its own case (assert_a_render_copy_without_the_generated_patch_...), which
  # deletes this file again.
  cp "$d/repo/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template" \
    "$d/repo/deploy/overlays/minikube/patches/k8s-api-ip.yaml"
  cat > "$d/repo/scripts/minikube/build-images.sh" <<'STUB'
#!/usr/bin/env bash
printf 'build-images %s\n' "$*" >>"${TEST_LOG_FILE:?}"
exit 0
STUB
  mkdir -p "$d/repo/control-api/src"
  echo "export const x = 1" > "$d/repo/control-api/src/index.ts"
  (
    cd "$d/repo"
    git init -q .
    # Fixture repositories must not launch background maintenance or inherit
    # filesystem monitors while the suite removes them after each scenario.
    git config maintenance.auto false
    git config gc.auto 0
    git config core.fsmonitor false
    git add -A
    git -c user.email=t@t -c user.name=t commit -qm base
  )
}

write_manifest() {
  local d=$1 body=$2
  mkdir -p "$d/repo/deploy/minikube"
  printf '%s' "$body" > "$d/repo/deploy/minikube/.image-manifest.json"
}

repo_head() { git -C "$1/repo" rev-parse HEAD; }

# Loads pre-gate-incremental.sh the way pre-gate-sync.sh does, in a subshell the
# caller controls, and runs $2 with the callers' contract satisfied.
run_incremental() {
  local d=$1 fn=$2
  (
    set +u
    PROJECT_DIR="$d/repo"
    PROFILE="clerum-test"
    KC="kubectl --context=${PROFILE}"
    CLUSTER_SYNC_STATE_CONFIGMAP="clerum-pre-gate-sync-state"
    FORCE_CLUSTER_SYNC="${FORCE_CLUSTER_SYNC:-false}"
    log() { printf '[pre-gate-sync] %s\n' "$*"; }
    rollout_if_present() { :; }
    # shellcheck source=/dev/null
    . "$PROJECT_DIR/scripts/minikube/pre-gate-incremental.sh"
    eval "$fn"
  ) 2>&1
}

# ---------------------------------------------------------------------------
# The shadow build itself
# ---------------------------------------------------------------------------

# The whole trick: build LOCALLY, then tag that build with the exact ghcr ref
# the Deployment already references, so IfNotPresent picks it up with no
# manifest edit. Building clerum/control-api:test alone leaves the pod on the
# release digest.
assert_a_targeted_ghcr_build_is_retagged_onto_the_running_ghcr_ref() {
  local d out rc want_local want_ghcr
  d="$(mktemp -d)"
  prepare_repo "$d"
  want_local="$(local_ref_for_image control-api)"
  want_ghcr="ghcr.io/evenfire-ai/control-api:${PIN_TAG}"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
    INCREMENTAL_TARGETS_INIT="control-api|control-plane|control-api" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=("control-api|control-plane|control-api"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "build-images --only=control-api" "$d/ops.log" \
     && grep -Fq "docker tag ${want_local} ${want_ghcr}" "$d/ops.log"; then
    pass "a targeted ghcr build is retagged onto the ref the Deployment runs"
  else
    fail "expected 'docker tag ${want_local} ${want_ghcr}'; rc=$rc log=$(cat "$d/ops.log") out=$out"
  fi
  cleanup_fixture "$d"
}

# A SELECTOR IS NOT AN IMAGE NAME. build-images.sh's --only does SUBSTRING
# matching, so --only=workflow builds four images. Resolving the selector to one
# name would shadow one and leave three on their release digests while the
# report claimed the whole selector was covered.
assert_a_multi_image_selector_shadows_every_image_it_builds() {
  local d out rc ref missing="" count
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=("workflow|control-plane|workflow-recipes"); incremental_build_images'
  )"
  rc=$?
  count=0
  while IFS= read -r ref; do
    [ -n "$ref" ] || continue
    count=$((count + 1))
    grep -Fq "docker tag " "$d/ops.log" && grep -Fq " ${ref}" "$d/ops.log" || missing+="${ref} "
  done < <(ghcr_refs_for_selector workflow "$PIN_TAG")
  # An image with no published counterpart carries `-` as its ghcr ref; tagging
  # onto that would put a literal `-` image in the daemon.
  if grep -q 'docker tag .* -$' "$d/ops.log"; then
    missing+="(retagged an unpublished image onto '-') "
  fi
  if [ "$rc" -eq 0 ] && [ "$count" -ge 4 ] && [ -z "$missing" ]; then
    pass "a multi-image selector shadows every image build-images.sh --only builds (${count})"
  else
    fail "selector 'workflow' resolved to ${count} image(s), not shadowed: ${missing:-none}; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The other direction, so the fix cannot be "always retag": a locally built
# cluster runs clerum/* refs, and inventing a ghcr tag there would put an image
# in the daemon that no pod references and no verify expects.
assert_local_mode_never_retags_onto_a_ghcr_ref() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=local IMAGE_TAG="" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=("control-api|control-plane|control-api"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "build-images --only=control-api" "$d/ops.log" \
     && ! grep -q 'docker tag' "$d/ops.log" \
     && ! grep -q 'SHADOWED' <<< "$out"; then
    pass "local mode builds the image and never retags it onto a ghcr ref"
  else
    fail "local mode retagged or reported a shadow; rc=$rc log=$(cat "$d/ops.log") out=$out"
  fi
  cleanup_fixture "$d"
}

# Nobody can act on a shadow they cannot see. The report is the difference
# between "the gate is green" and "the gate is green about these two images".
assert_the_shadow_set_is_reported_with_every_ref() {
  local d out rc want
  d="$(mktemp -d)"
  prepare_repo "$d"
  want="ghcr.io/evenfire-ai/control-api:${PIN_TAG}"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=("control-api|control-plane|control-api"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] && grep -q 'SHADOWED' <<< "$out" && grep -Fq "$want" <<< "$out"; then
    pass "the shadow set is reported explicitly, naming every shadowed ref"
  else
    fail "expected a SHADOWED report naming ${want}; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# With nothing to shadow the report must still run and must say so, or an
# operator reads silence as "my code is deployed".
assert_an_empty_shadow_set_is_reported_as_release_only() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=(); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] && grep -q 'SHADOWED: none' <<< "$out" && grep -Fq "$PIN_TAG" <<< "$out"; then
    pass "an empty shadow set is reported as 'every image is the release build'"
  else
    fail "expected an explicit empty-shadow report naming ${PIN_TAG}; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# A retag that fails leaves the pod on the release digest with a freshly built
# local image nothing references -- exactly the silent-wrong-code state. It has
# to stop the gate.
assert_a_failed_retag_fails_the_pre_gate() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_DOCKER_TAG_FAILS=1 \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
      run_incremental "$d" 'INCREMENTAL_TARGETS=("control-api|control-plane|control-api"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -ne 0 ] && grep -qi 'undeployed\|could not shadow' <<< "$out"; then
    pass "a failed retag stops the pre-gate instead of reporting a build success"
  else
    fail "expected a non-zero exit naming the failed shadow; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# ---------------------------------------------------------------------------
# What INCREMENTAL_FULL_IMAGE_BUILD means in each mode
# ---------------------------------------------------------------------------

# "Build everything" is the wrong recovery in ghcr mode: it reinstates the
# 20-minute build this change exists to remove and leaves the cluster holding
# clerum/* images the ghcr overlay never references.
assert_a_full_sync_in_ghcr_mode_repulls_instead_of_building_everything() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
      IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" T2_LOCK_TOKEN=test-token \
      run_incremental "$d" 'INCREMENTAL_REPULL_ALL=true; INCREMENTAL_TARGETS=("control-api|control-plane|control-api"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "make minikube-pull-images" "$d/ops.log" \
     && grep -Fq 'make minikube-pull-images T2_SKIP_LOCK=true T2_LOCK_TOKEN_SET=true' "$d/ops.log" \
     && ! grep -Fq "make minikube-build-images" "$d/ops.log" \
     && grep -Fq "docker tag $(local_ref_for_image control-api) ghcr.io/evenfire-ai/control-api:${PIN_TAG}" "$d/ops.log"; then
    pass "a ghcr full sync re-pulls the release set, then shadows the changed set on top"
  else
    fail "expected a re-pull followed by the shadow build; rc=$rc log=$(cat "$d/ops.log") out=$out"
  fi
  cleanup_fixture "$d"
}

# Local mode is untouched: a full image build still means build everything.
assert_local_mode_still_builds_everything_on_a_full_image_build() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
      IMAGE_SOURCE=local IMAGE_TAG="" T2_LOCK_TOKEN=test-token \
      run_incremental "$d" 'INCREMENTAL_FULL_IMAGE_BUILD=true; incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "make minikube-build-images" "$d/ops.log" \
     && grep -Fq 'make minikube-build-images T2_SKIP_LOCK=true T2_LOCK_TOKEN_SET=true' "$d/ops.log" \
     && ! grep -Fq "make minikube-pull-images" "$d/ops.log"; then
    pass "local mode still builds every image when the change cannot be targeted"
  else
    fail "expected a full local build and no pull; rc=$rc log=$(cat "$d/ops.log") out=$out"
  fi
  cleanup_fixture "$d"
}

# pre-gate-sync acquires the profile lock itself before it sources this file.
# minikube-build-images and minikube-pull-images wrap their body in
# with-t2-mutation-lock.sh, which acquires the lock unless T2_SKIP_LOCK=true;
# called without the inherited lease they find the lock held by their own
# parent and refuse with PROFILE_BUSY. Both calls must pass the parent's lease,
# exactly as the minikube-deploy-crds / minikube-deploy-all calls do.
assert_full_image_calls_inherit_the_parent_profile_lock() {
  local d out_local out_ghcr rc_local rc_ghcr build_line pull_line
  d="$(mktemp -d)"
  prepare_repo "$d"
  cat > "$d/bin/make" <<'STUB'
#!/usr/bin/env bash
printf 'make %s T2_SKIP_LOCK=%s T2_LOCK_TOKEN=%s\n' "$*" "${T2_SKIP_LOCK:-}" "${T2_LOCK_TOKEN:-}" >>"${TEST_LOG_FILE:?}"
exit 0
STUB
  chmod +x "$d/bin/make"
  # The parent's state after t2_lock_acquire: it owns the lock, so its own
  # T2_SKIP_LOCK is empty and its token is exported.
  out_local="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=local IMAGE_TAG="" T2_SKIP_LOCK="" T2_LOCK_TOKEN="parent-lease-token" \
      run_incremental "$d" 'INCREMENTAL_FULL_IMAGE_BUILD=true; incremental_build_images'
  )"
  rc_local=$?
  out_ghcr="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" T2_SKIP_LOCK="" T2_LOCK_TOKEN="parent-lease-token" \
      run_incremental "$d" 'INCREMENTAL_REPULL_ALL=true; INCREMENTAL_TARGETS=(); incremental_build_images'
  )"
  rc_ghcr=$?
  build_line="$(grep -c '^make minikube-build-images ' "$d/ops.log")"
  pull_line="$(grep -c '^make minikube-pull-images ' "$d/ops.log")"
  # Liveness: each lock-wrapped target was called exactly once. Only then does
  # the lease assertion mean anything.
  if [ "$rc_local" -eq 0 ] && [ "$rc_ghcr" -eq 0 ] \
     && [ "$build_line" = 1 ] && [ "$pull_line" = 1 ] \
     && grep -Fqx "make minikube-build-images T2_SKIP_LOCK=true T2_LOCK_TOKEN=parent-lease-token" "$d/ops.log" \
     && grep -Fqx "make minikube-pull-images T2_SKIP_LOCK=true T2_LOCK_TOKEN=parent-lease-token" "$d/ops.log"; then
    pass "full-image build and re-pull inherit the parent's profile lock"
  else
    fail "expected build/pull to carry T2_SKIP_LOCK=true and the parent token; rc_local=$rc_local rc_ghcr=$rc_ghcr log=$(cat "$d/ops.log") out_local=$out_local out_ghcr=$out_ghcr"
  fi
  cleanup_fixture "$d"
}

# The minikube-verify-images recipe runs `build-images.sh --verify-only` with no
# mutation-lock wrapper, and build-images.sh returns from
# require_inherited_mutation_lease before any lease check when --verify-only is
# set. T2_SKIP_LOCK / T2_LOCK_TOKEN on that call would therefore be ignored and
# would read as if it needed the lease, so it is called without them. This test
# checks the call sites and the Makefile recipe text only (the --verify-only
# flag is present and no mutation-lock wrapper is); it does not run
# build-images.sh, so the script-side skip is not asserted here. If the recipe
# ever gains a lock wrapper, the recipe check below fails and the call must pass
# the lease like the build and pull calls above.
assert_verify_images_is_called_without_the_lock_lease() {
  local d out_local out_ghcr rc_local rc_ghcr verify_calls bare_calls recipe
  recipe="$(awk '/^minikube-verify-images:/{f=1;next} f&&/^[^\t]/{exit} f' "$REPO_ROOT/Makefile")"
  d="$(mktemp -d)"
  prepare_repo "$d"
  cat > "$d/bin/make" <<'STUB'
#!/usr/bin/env bash
printf 'make %s T2_SKIP_LOCK=%s\n' "$*" "${T2_SKIP_LOCK:-}" >>"${TEST_LOG_FILE:?}"
exit 0
STUB
  chmod +x "$d/bin/make"
  out_local="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=local IMAGE_TAG="" T2_SKIP_LOCK="" T2_LOCK_TOKEN="parent-lease-token" \
      run_incremental "$d" 'INCREMENTAL_FULL_IMAGE_BUILD=true; incremental_build_images'
  )"
  rc_local=$?
  out_ghcr="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" T2_SKIP_LOCK="" T2_LOCK_TOKEN="parent-lease-token" \
      run_incremental "$d" 'INCREMENTAL_REPULL_ALL=true; INCREMENTAL_TARGETS=(); incremental_build_images'
  )"
  rc_ghcr=$?
  verify_calls="$(grep -c '^make minikube-verify-images ' "$d/ops.log")"
  bare_calls="$(grep -Fxc 'make minikube-verify-images T2_SKIP_LOCK=' "$d/ops.log")"
  # Liveness: one verify call per path (full build and re-pull), and the recipe
  # is the verify-only build script, before the absence of a lease means anything.
  if [ "$rc_local" -eq 0 ] && [ "$rc_ghcr" -eq 0 ] \
     && [ "$verify_calls" = 2 ] && [ "$bare_calls" = 2 ] \
     && grep -Fq 'build-images.sh --verify-only' <<< "$recipe" \
     && ! grep -Fq 'mutation-lock' <<< "$recipe"; then
    pass "verify-images takes no lock and is called without the lock lease"
  else
    fail "expected two verify-images calls without T2_SKIP_LOCK and a lock-free recipe; rc_local=$rc_local rc_ghcr=$rc_ghcr verify=$verify_calls bare=$bare_calls log=$(cat "$d/ops.log") recipe=$recipe out_local=$out_local out_ghcr=$out_ghcr"
  fi
  cleanup_fixture "$d"
}

# An unmapped runtime path in ghcr mode is the one case the shadow cannot cover:
# there is no image to build. Passing here would gate against undeployed code,
# so it must stop, name the path, and name a remedy that works.
assert_an_unmapped_change_hard_fails_in_ghcr_mode_with_a_remedy() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" \
      run_incremental "$d" 'INCREMENTAL_FULL_IMAGE_BUILD=true; INCREMENTAL_FULL_IMAGE_BUILD_REASON=unmapped; INCREMENTAL_UNMAPPED=("some/unmapped/file.ts"); incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -ne 0 ] \
     && grep -Fq 'some/unmapped/file.ts' <<< "$out" \
     && grep -Fq 'minikube-setup-local' <<< "$out"; then
    pass "an unmapped change in ghcr mode hard-fails, naming the path and a working remedy"
  else
    fail "expected a hard fail naming the unmapped path and minikube-setup-local; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The same rule for the other unrecoverable case: no baseline means the shadow
# set is unknowable, and "shadow nothing" would read as "everything is fine".
assert_an_unresolvable_baseline_hard_fails_in_ghcr_mode() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    IMAGE_SOURCE=ghcr IMAGE_TAG=latest \
      run_incremental "$d" 'INCREMENTAL_FULL_IMAGE_BUILD=true; INCREMENTAL_FULL_IMAGE_BUILD_REASON=no-baseline; incremental_build_images'
  )"
  rc=$?
  if [ "$rc" -ne 0 ] \
     && grep -Fq 'minikube-setup-local' <<< "$out" \
     && grep -q 'baseline' <<< "$out"; then
    pass "an unresolvable release baseline hard-fails instead of gating on release code"
  else
    fail "expected a hard fail naming the missing baseline and a remedy; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# ---------------------------------------------------------------------------
# Where the shadow baseline comes from
# ---------------------------------------------------------------------------

# Published images carry org.opencontainers.image.revision (build-publish.yml),
# so a ghcr cluster with no usable marker still knows exactly which commit its
# release images were built from -- including on the `latest` bootstrap tag,
# which resolves to no git ref at all.
assert_the_release_image_revision_label_supplies_the_missing_baseline() {
  local d out rc base
  d="$(mktemp -d)"
  prepare_repo "$d"
  base="$(repo_head "$d")"
  echo "// changed" >> "$d/repo/control-api/src/index.ts"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_IMAGE_REVISION="$base" \
    TEST_MARKER_DIR="$d/nomarker" \
    IMAGE_SOURCE=ghcr IMAGE_TAG=latest \
      run_incremental "$d" 'incremental_plan; echo "targets=$(incremental_target_summary) full=${INCREMENTAL_FULL_IMAGE_BUILD} repull=${INCREMENTAL_REPULL_ALL}"'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq 'targets=control-api' <<< "$out" \
     && grep -Fq 'full=false' <<< "$out" \
     && grep -Fq 'repull=true' <<< "$out"; then
    pass "the release image's revision label supplies the baseline when no marker exists"
  else
    fail "expected a control-api-only plan from the revision label; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# Without a marker AND without a resolvable release revision there is no
# baseline at all, and the plan must say so rather than compute an empty delta.
assert_no_marker_and_no_revision_label_fails_closed() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  echo "// changed" >> "$d/repo/control-api/src/index.ts"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_IMAGE_REVISION="" \
    TEST_MARKER_DIR="$d/nomarker" \
    IMAGE_SOURCE=ghcr IMAGE_TAG=latest \
      run_incremental "$d" 'incremental_plan; echo "full=${INCREMENTAL_FULL_IMAGE_BUILD} reason=${INCREMENTAL_FULL_IMAGE_BUILD_REASON}"'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq 'full=true' <<< "$out" \
     && grep -Fq 'reason=no-baseline' <<< "$out"; then
    pass "no marker and no revision label fails closed with an explicit reason"
  else
    fail "expected full=true reason=no-baseline; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The baseline read runs inside a command substitution, so a diagnostic printed
# on stdout there would be captured AS the baseline and then diffed against.
# A daemon this pre-gate cannot reach must produce no baseline at all.
assert_an_unreachable_docker_daemon_cannot_become_a_baseline() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  cat > "$d/bin/minikube" <<'STUB'
#!/usr/bin/env bash
printf 'minikube %s\n' "$*" >>"${TEST_LOG_FILE:?}"
echo "minikube: profile not found" >&2
exit 1
STUB
  chmod +x "$d/bin/minikube"
  echo "// changed" >> "$d/repo/control-api/src/index.ts"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_MARKER_DIR="$d/nomarker" \
    IMAGE_SOURCE=ghcr IMAGE_TAG=latest \
      run_incremental "$d" 'incremental_plan; echo "full=${INCREMENTAL_FULL_IMAGE_BUILD} reason=${INCREMENTAL_FULL_IMAGE_BUILD_REASON}"'
  )"
  rc=$?
  if grep -Fq 'full=true' <<< "$out" && grep -Fq 'reason=no-baseline' <<< "$out"; then
    pass "an unreachable Docker daemon produces no baseline instead of a diagnostic-shaped one"
  else
    fail "expected full=true reason=no-baseline from an unreachable daemon; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# A `make minikube-setup` between two pre-gates re-pulls every release image and
# DISCARDS the shadows, while leaving the marker untouched. Trusting the
# marker's gitHead there computes an empty delta and gates on release code.
assert_a_re_acquisition_invalidates_the_marker_baseline_in_ghcr_mode() {
  local d out rc head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  mkdir -p "$d/marker"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "2026-08-06T00:00:00Z" > "$d/marker/imagesGeneratedAt"
  write_manifest "$d" '{"generated":"2026-08-06T09:99:99Z","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_MARKER_DIR="$d/marker" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" IMAGES_GENERATED_AT="2026-08-06T09:99:99Z" \
      run_incremental "$d" 'echo "baseline=[$(incremental_marker_git_head)]"'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] && grep -Fq 'baseline=[]' <<< "$out"; then
    pass "a re-acquisition since the marker invalidates its gitHead baseline in ghcr mode"
  else
    fail "expected an empty baseline after a re-acquisition; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# A local rebuild can also replace image IDs without changing gitHead. The
# incremental planner must not treat that newer acquisition as proof that the
# old deployment is still the image set being tested.
assert_a_re_acquisition_invalidates_the_marker_baseline_in_local_mode() {
  local d out rc head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  mkdir -p "$d/marker"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "2026-08-06T00:00:00Z" > "$d/marker/imagesGeneratedAt"
  write_manifest "$d" '{"generated":"2026-08-06T09:99:99Z","imageSource":"local","imageTag":"","images":{}}'
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_MARKER_DIR="$d/marker" \
    IMAGE_SOURCE=local IMAGE_TAG="" IMAGES_GENERATED_AT="2026-08-06T09:99:99Z" \
      run_incremental "$d" 'echo "baseline=[$(incremental_marker_git_head)]"'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] && grep -Fq 'baseline=[]' <<< "$out"; then
    pass "a newer local image acquisition invalidates its gitHead baseline"
  else
    fail "expected an empty local baseline after a re-acquisition; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The complement, so the fix cannot be "never trust the marker": an untouched
# image set keeps its baseline, which is what makes the incremental path fast.
assert_an_untouched_image_set_keeps_the_marker_baseline() {
  local d out rc head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  mkdir -p "$d/marker"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "2026-08-06T00:00:00Z" > "$d/marker/imagesGeneratedAt"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" TEST_MARKER_DIR="$d/marker" \
    IMAGE_SOURCE=ghcr IMAGE_TAG="$PIN_TAG" IMAGES_GENERATED_AT="2026-08-06T00:00:00Z" \
      run_incremental "$d" 'echo "baseline=[$(incremental_marker_git_head)]"'
  )"
  rc=$?
  if [ "$rc" -eq 0 ] && grep -Fq "baseline=[${head}]" <<< "$out"; then
    pass "an untouched image set keeps the marker's gitHead as the shadow baseline"
  else
    fail "expected baseline=[${head}]; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# ---------------------------------------------------------------------------
# The mode the pre-gate assumes, and the marker that records it
# ---------------------------------------------------------------------------

# Runs pre-gate-sync.sh up to its configuration seam. No cluster call happens
# before that point.
run_pre_gate_config() {
  local d=$1; shift
  (
    cd "$d/repo" || exit 1
    PATH="$d/bin:$PATH" \
    TEST_LOG_FILE="$d/ops.log" \
    TMPDIR="$d/state" \
    PRE_GATE_SYNC_CONFIG_ONLY=true \
    env "$@" bash "$d/repo/scripts/minikube/pre-gate-sync.sh" --gate manual
  ) 2>&1
}

# A cluster set up in one mode and pre-gated in the other must not silently
# agree. What the cluster RUNS decides, and that is recorded by whichever writer
# last acquired images -- not by whatever IMAGE_SOURCE this shell defaults to.
assert_the_pre_gate_adopts_the_recorded_cluster_mode_over_the_environment() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  out="$(run_pre_gate_config "$d" IMAGE_SOURCE=local)"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "imageSource=ghcr" <<< "$out" \
     && grep -Fq "imageTag=${PIN_TAG}" <<< "$out" \
     && grep -Fq "renderDir=${d}/repo/deploy/overlays/minikube-ghcr" <<< "$out"; then
    pass "the pre-gate adopts the mode the cluster's images were acquired in"
  else
    fail "expected the recorded ghcr mode to win over IMAGE_SOURCE=local; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The other direction, so the fix cannot be "always ghcr": a locally built
# cluster must render the local overlay even though ghcr is the default.
assert_a_recorded_local_build_renders_the_local_overlay() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"local","images":{}}'
  out="$(run_pre_gate_config "$d")"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq "imageSource=local" <<< "$out" \
     && grep -Fq "renderDir=${d}/repo/deploy/overlays/minikube" <<< "$out" \
     && ! grep -q "minikube-ghcr" <<< "$out"; then
    pass "a recorded local build renders the local overlay despite the ghcr default"
  else
    fail "expected the local overlay for a recorded local build; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The migration Job EXTRACTS THE CONTROL-API IMAGE from the overlay it renders.
# Rendering the local overlay on a ghcr cluster yields clerum/control-api:test,
# which does not exist there, so the Job ImagePullBackOffs on every ghcr
# pre-gate; rendering the pinned ghcr overlay when the cluster runs an
# overridden tag pulls an image the cluster never had.
assert_the_migration_job_renders_the_overlay_the_cluster_runs() {
  local d out rc hardcoded
  d="$(mktemp -d)"
  prepare_repo "$d"
  hardcoded='--overlay "${PROJECT_DIR}/deploy/overlays/minikube"'
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"latest","images":{}}'
  out="$(run_pre_gate_config "$d")"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -q "renderDir=.*/deploy/overlays/minikube-ghcr" <<< "$out" \
     && grep -Fq 'newTag: latest' "$(grep -o 'renderDir=.*' <<< "$out" | head -1 | cut -d= -f2-)/../../components/ghcr-images/kustomization.yaml" \
     && ! grep -Fq -- "$hardcoded" "$REPO_ROOT/scripts/minikube/pre-gate-sync.sh" \
     && grep -Fq -- '--overlay "${PRE_GATE_RENDER_DIR}"' "$REPO_ROOT/scripts/minikube/pre-gate-sync.sh"; then
    pass "the migration Job renders the overlay (and tag) the cluster actually runs"
  else
    fail "the migration overlay does not follow the cluster's mode/tag; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# Sourcing the script stops at the same seam, which leaves its marker functions
# defined and callable against the stubbed kubectl.
source_pre_gate_marker_fns() {
  local d=$1
  cd "$d/repo" || return 1
  PRE_GATE_SYNC_CONFIG_ONLY=true
  export PRE_GATE_SYNC_CONFIG_ONLY
  # A sourced script sees the SOURCING function's positional parameters, and
  # pre-gate-sync.sh rejects unknown arguments. Clear them first.
  set --
  # shellcheck source=/dev/null
  . "$d/repo/scripts/minikube/pre-gate-sync.sh"
}

# gitHead alone cannot tell a ghcr cluster from a local one, nor v0.6.0 from
# latest. Both belong in the marker or a mode change reads as "in sync".
assert_the_marker_records_the_image_source_and_tag() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g7","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  out="$(
    (
      set +u
      PATH="$d/bin:$PATH"
      TEST_LOG_FILE="$d/ops.log"
      TMPDIR="$d/state"
      export PATH TEST_LOG_FILE TMPDIR
      source_pre_gate_marker_fns "$d" >/dev/null
      persist_cluster_marker fingerprint-a fingerprint-b
    ) 2>&1
  )"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && grep -Fq -- "--from-literal=imageSource=ghcr" "$d/ops.log" \
     && grep -Fq -- "--from-literal=imageTag=${PIN_TAG}" "$d/ops.log" \
     && grep -Fq -- "--from-literal=imagesGeneratedAt=g7" "$d/ops.log" \
     && grep -Fq -- "--from-literal=gitHead=" "$d/ops.log"; then
    pass "the marker records imageSource, imageTag and the image-acquisition stamp beside gitHead"
  else
    fail "the marker does not record the image mode; rc=$rc log=$(cat "$d/ops.log") out=$out"
  fi
  cleanup_fixture "$d"
}

# A marker written by a ghcr pre-gate must not match a local one.
marker_matches_for() {
  local d=$1
  (
    set +u
    PATH="$d/bin:$PATH"
    TEST_LOG_FILE="$d/ops.log"
    TEST_MARKER_DIR="$d/marker"
    TMPDIR="$d/state"
    export PATH TEST_LOG_FILE TEST_MARKER_DIR TMPDIR
    source_pre_gate_marker_fns "$d" >/dev/null
    printf '%s' "$WORKTREE_ID" > "$d/marker/worktreeId"
    cluster_marker_matches fingerprint-a "$WORKTREE_ID"
  ) >/dev/null 2>&1
}

assert_a_mode_change_forces_a_full_resync() {
  local d rc_same rc_diff head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  write_manifest "$d" '{"generated":"g7","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  mkdir -p "$d/marker"
  printf '%s' "fingerprint-a" > "$d/marker/clusterFingerprint"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "g7" > "$d/marker/imagesGeneratedAt"
  printf '%s' "$PIN_TAG" > "$d/marker/imageTag"
  printf '%s' "ghcr" > "$d/marker/imageSource"
  marker_matches_for "$d"; rc_same=$?
  printf '%s' "local" > "$d/marker/imageSource"
  marker_matches_for "$d"; rc_diff=$?
  if [ "$rc_same" -eq 0 ] && [ "$rc_diff" -ne 0 ]; then
    pass "an identical marker matches and a changed imageSource forces a resync"
  else
    fail "expected match=0 and mode-change!=0; got same=$rc_same changed=$rc_diff"
  fi
  cleanup_fixture "$d"
}

# The tag is half the coordinate: v0.6.0 and latest are different clusters.
assert_a_tag_change_forces_a_full_resync() {
  local d rc_same rc_diff head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  write_manifest "$d" '{"generated":"g7","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  mkdir -p "$d/marker"
  printf '%s' "fingerprint-a" > "$d/marker/clusterFingerprint"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "g7" > "$d/marker/imagesGeneratedAt"
  printf '%s' "ghcr" > "$d/marker/imageSource"
  printf '%s' "$PIN_TAG" > "$d/marker/imageTag"
  marker_matches_for "$d"; rc_same=$?
  printf '%s' "some-other-tag" > "$d/marker/imageTag"
  marker_matches_for "$d"; rc_diff=$?
  if [ "$rc_same" -eq 0 ] && [ "$rc_diff" -ne 0 ]; then
    pass "an identical marker matches and a changed image tag forces a resync"
  else
    fail "expected match=0 and tag-change!=0; got same=$rc_same changed=$rc_diff"
  fi
  cleanup_fixture "$d"
}

# A setup between two pre-gates replaces every image and discards the shadows.
# Without this the marker still matches and no sync runs at all.
assert_a_re_acquisition_forces_a_full_resync() {
  local d rc_same rc_diff head
  d="$(mktemp -d)"
  prepare_repo "$d"
  head="$(repo_head "$d")"
  write_manifest "$d" '{"generated":"g8","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  mkdir -p "$d/marker"
  printf '%s' "fingerprint-a" > "$d/marker/clusterFingerprint"
  printf '%s' "$head" > "$d/marker/gitHead"
  printf '%s' "ghcr" > "$d/marker/imageSource"
  printf '%s' "$PIN_TAG" > "$d/marker/imageTag"
  printf '%s' "g8" > "$d/marker/imagesGeneratedAt"
  marker_matches_for "$d"; rc_same=$?
  printf '%s' "g7" > "$d/marker/imagesGeneratedAt"
  marker_matches_for "$d"; rc_diff=$?
  if [ "$rc_same" -eq 0 ] && [ "$rc_diff" -ne 0 ]; then
    pass "a marker stamped before the last image acquisition forces a full resync"
  else
    fail "expected match=0 and stale-acquisition!=0; got same=$rc_same stale=$rc_diff"
  fi
  cleanup_fixture "$d"
}

# ---------------------------------------------------------------------------
# The other mode-crossing path: the full overlay apply
# ---------------------------------------------------------------------------

# `make minikube-deploy-all` hardcoded deploy/overlays/minikube, so a deploy/*
# change on a ghcr cluster applied manifests referencing clerum/*:test tags that
# were never built there -- cluster-wide ImagePullBackOff.
assert_make_deploy_all_renders_the_mode_aware_overlay() {
  local out
  # The public target now wraps its private body with the branch-profile lease;
  # inspect that body for the overlay resolver while the wrapper contract is
  # covered by test-minikube-makefile.sh.
  out="$(cd "$REPO_ROOT" && make -n minikube-deploy-all-body 2>&1)"
  if grep -q 'image-mode.sh --render-dir' <<< "$out" \
     && ! grep -q 'kustomize deploy/overlays/minikube |' <<< "$out"; then
    pass "make minikube-deploy-all resolves the overlay from the cluster's image mode"
  else
    fail "make minikube-deploy-all still hardcodes the local overlay: $out"
  fi
}

# The resolver is the single mechanism both call sites use, so its own
# behaviour is checked directly: a recorded mode wins over the environment.
assert_the_render_dir_resolver_follows_the_recorded_mode() {
  local d ghcr_out local_out
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  ghcr_out="$(IMAGE_SOURCE=local bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  write_manifest "$d" '{"generated":"g1","imageSource":"local","images":{}}'
  local_out="$(bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  if [ "$ghcr_out" = "$d/repo/deploy/overlays/minikube-ghcr" ] \
     && [ "$local_out" = "$d/repo/deploy/overlays/minikube" ]; then
    pass "the render-dir resolver follows the recorded mode, not the environment"
  else
    fail "resolver returned ghcr='$ghcr_out' local='$local_out'"
  fi
  cleanup_fixture "$d"
}

# An overridden tag is not committed anywhere, so rendering the committed
# overlay would apply the PINNED tag to a cluster running the override.
assert_an_overridden_tag_renders_from_a_copy_carrying_that_tag() {
  local d out rendered
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"latest","images":{}}'
  out="$(bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  rendered="${out}/../../components/ghcr-images/kustomization.yaml"
  if [ -d "$out" ] \
     && [ "${out#"$d/repo/deploy"}" = "$out" ] \
     && [ -f "$rendered" ] \
     && grep -q 'newTag: latest' "$rendered" \
     && ! grep -q "newTag: ${PIN_TAG}" "$rendered" \
     && grep -q "newTag: ${PIN_TAG}" "$d/repo/deploy/components/ghcr-images/kustomization.yaml"; then
    pass "an overridden tag renders from a copy carrying it, leaving the committed pin alone"
  else
    fail "expected an out-of-tree render dir pinned to 'latest'; got '$out'"
  fi
  cleanup_fixture "$d"
}

# The copy has to carry the GENERATED patch, not just the committed files.
# overlays/minikube-ghcr renders `resources: ../minikube`, which patches with
# patches/k8s-api-ip.yaml -- gitignored, written by minikube-detect-k8s-api-ip.sh.
assert_a_render_copy_carries_the_generated_api_ip_patch() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"latest","images":{}}'
  out="$(bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  rc=$?
  if [ "$rc" -eq 0 ] \
     && [ -d "$out" ] \
     && [ -f "${out}/../minikube/patches/k8s-api-ip.yaml" ]; then
    pass "the tag-override render copy carries the generated k8s-api-ip.yaml"
  else
    fail "expected the copy to carry patches/k8s-api-ip.yaml; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# Reproduced before the fix: a fresh clone plus
# `MINIKUBE_IMAGE_TAG=latest make minikube-setup` generated the patch only into
# the setup's own mktemp copy, so the next pre-gate rendered a copy without one
# and kustomize failed with `evalsymlink failure on <temp>/patches/
# k8s-api-ip.yaml: no such file or directory` -- a path inside a temp dir, and
# no remedy. Fail before the copy is handed out, naming the file in the
# developer's tree and the command that writes it.
assert_a_render_copy_without_the_generated_api_ip_patch_fails_with_a_remedy() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"latest","images":{}}'
  rm -f "$d/repo/deploy/overlays/minikube/patches/k8s-api-ip.yaml"
  out="$(bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  rc=$?
  if [ "$rc" -ne 0 ] \
     && grep -Fq "$d/repo/deploy/overlays/minikube/patches/k8s-api-ip.yaml" <<< "$out" \
     && grep -Fq 'deploy/scripts/minikube-detect-k8s-api-ip.sh' <<< "$out" \
     && ! grep -q 'evalsymlink' <<< "$out"; then
    pass "a copy missing the generated patch fails naming the file and how to regenerate it"
  else
    fail "expected a named, actionable failure; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# The pinned path renders the committed overlay in place, so it must NOT be
# gated on the generated patch: kustomize is what reports a missing patch there,
# and only the copy can be silently built without one.
assert_the_pinned_render_dir_is_not_gated_on_the_generated_patch() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"'"$PIN_TAG"'","images":{}}'
  rm -f "$d/repo/deploy/overlays/minikube/patches/k8s-api-ip.yaml"
  out="$(bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  rc=$?
  if [ "$rc" -eq 0 ] && [ "$out" = "$d/repo/deploy/overlays/minikube-ghcr" ]; then
    pass "the pinned render dir resolves without the generated patch"
  else
    fail "expected the committed ghcr overlay; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# A mode that is neither ghcr nor local is a typo, and guessing one silently
# picks a cluster shape.
assert_an_unknown_image_source_is_rejected() {
  local d out rc
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(IMAGE_SOURCE=gcr bash "$d/repo/scripts/minikube/image-mode.sh" --render-dir 2>&1)"
  rc=$?
  if [ "$rc" -ne 0 ] && grep -q 'gcr' <<< "$out"; then
    pass "an unknown IMAGE_SOURCE is rejected instead of guessed"
  else
    fail "expected a non-zero exit naming the bad mode; rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

# A shadow build rewrites the image manifest through build-images.sh --only. If
# that run dropped the recorded tag, the next pre-gate would fall back to the
# committed pin and declare the cluster out of sync on every run.
# A shadow build is an --only run of build-images.sh, and that run REWRITES the
# image manifest. Dropping the recorded coordinate there would make the next
# pre-gate fall back to the committed pin, decide the cluster changed tag, and
# force a full resync on every single run.
assert_a_targeted_build_carries_the_recorded_coordinate_forward() {
  local d rc manifest mode tag
  d="$(mktemp -d)"
  prepare_repo "$d"
  # The REAL build-images.sh, not the stub prepare_repo installs.
  cp "$REPO_ROOT/scripts/minikube/build-images.sh" "$d/repo/scripts/minikube/build-images.sh"
  write_manifest "$d" '{"generated":"g1","imageSource":"ghcr","imageTag":"latest","images":{}}'
  (
    cd "$d/repo" || exit 1
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
      T2_PROJECT_DIR="$d/repo" T2_PROFILE=clerum-test T2_CONTEXT=clerum-test \
      MINIKUBE_PROFILE=clerum-test CONTROL_API_REAL_PG_CONTEXT=clerum-test \
      DOCKER_HOST=unix:///tmp/evenfire-docker.sock \
      bash "$d/repo/scripts/minikube/build-images.sh" --only=control-api
  ) >"$d/build.log" 2>&1
  rc=$?
  manifest="$d/repo/deploy/minikube/.image-manifest.json"
  mode="$(IMAGE_SOURCE=local bash "$d/repo/scripts/minikube/image-mode.sh" --image-source 2>&1)"
  tag="$(IMAGE_SOURCE=local bash "$d/repo/scripts/minikube/image-mode.sh" --image-tag 2>&1)"
  if [ "$rc" -eq 0 ] \
     && [ -f "$manifest" ] \
     && [ "$mode" = "ghcr" ] \
     && [ "$tag" = "latest" ]; then
    pass "a targeted build carries the recorded imageSource and imageTag forward"
  else
    fail "after --only the manifest records mode='$mode' tag='$tag' (rc=$rc): $(tail -3 "$d/build.log")"
  fi
  cleanup_fixture "$d"
}

assert_the_touched_scripts_parse() {
  local f bad=""
  for f in scripts/minikube/image-mode.sh scripts/minikube/pre-gate-marker.sh \
           scripts/minikube/pre-gate-sync.sh \
           scripts/minikube/pre-gate-incremental.sh scripts/minikube/build-images.sh \
           scripts/tests/test-minikube-pre-gate-shadow.sh; do
    bash -n "$REPO_ROOT/$f" || bad+="$f "
  done
  if [ -z "$bad" ]; then
    pass "every script this harness touches parses"
  else
    fail "bash -n failed for: $bad"
  fi
}

assert_incremental_runtime_calls_are_bounded() {
  local d out rc=0
  d="$(mktemp -d)"
  prepare_repo "$d"
  out="$(
    PATH="$d/bin:$PATH" TEST_LOG_FILE="$d/ops.log" \
    TEST_MINIKUBE_HANG_DOCKER_ENV=1 \
    INCREMENTAL_RUNTIME_TIMEOUT_SECONDS=1 \
    MINIKUBE_DOCKER_KILL_GRACE_SECONDS=1 \
      run_incremental "$d" 'incremental_use_minikube_docker'
  )" || rc=$?
  if [ "$rc" -ne 0 ] \
     && grep -Fq '[HARNESS_DEADLINE] label=incremental-docker-env event=timeout' <<< "$out"; then
    pass "incremental Docker daemon selection terminates at its explicit deadline"
  else
    fail "incremental docker-env was not bounded (rc=$rc out=$out)"
  fi
  cleanup_fixture "$d"
}

assert_empty_incremental_docker_env_fails_closed() {
  local d out rc=0
  d="$(mktemp -d)"
  prepare_repo "$d"
  export TEST_EMPTY_DOCKER_ENV=1
  out="$(run_incremental "$d" 'incremental_use_minikube_docker')" || rc=$?
  unset TEST_EMPTY_DOCKER_ENV
  if [ "$rc" -ne 0 ] && grep -Fq 'DOCKER_ENV_UNRESOLVED' <<<"$out"; then
    pass "empty incremental minikube docker-env output fails closed"
  else
    fail "incremental docker-env returned success without a Docker host: rc=$rc out=$out"
  fi
  cleanup_fixture "$d"
}

assert_every_defined_case_is_invoked() {
  local self defined invoked missing
  self="$REPO_ROOT/scripts/tests/test-minikube-pre-gate-shadow.sh"
  defined="$(grep -oE '^assert_[a-z_]+\(\) \{' "$self" | sed -E 's/\(\) \{$//' | sort -u)"
  invoked="$(grep -oE '^assert_[a-z_]+$' "$self" | sort -u)"
  missing="$(comm -23 <(printf '%s\n' "$defined") <(printf '%s\n' "$invoked"))"
  if [ -z "$missing" ]; then
    pass "every defined assert_ case is invoked in the call block"
  else
    fail "defined but never invoked: $(printf '%s ' $missing)"
  fi
}

assert_a_targeted_ghcr_build_is_retagged_onto_the_running_ghcr_ref
assert_a_multi_image_selector_shadows_every_image_it_builds
assert_local_mode_never_retags_onto_a_ghcr_ref
assert_the_shadow_set_is_reported_with_every_ref
assert_an_empty_shadow_set_is_reported_as_release_only
assert_a_failed_retag_fails_the_pre_gate
assert_a_full_sync_in_ghcr_mode_repulls_instead_of_building_everything
assert_local_mode_still_builds_everything_on_a_full_image_build
assert_full_image_calls_inherit_the_parent_profile_lock
assert_verify_images_is_called_without_the_lock_lease
assert_an_unmapped_change_hard_fails_in_ghcr_mode_with_a_remedy
assert_an_unresolvable_baseline_hard_fails_in_ghcr_mode
assert_the_release_image_revision_label_supplies_the_missing_baseline
assert_no_marker_and_no_revision_label_fails_closed
assert_an_unreachable_docker_daemon_cannot_become_a_baseline
assert_a_re_acquisition_invalidates_the_marker_baseline_in_ghcr_mode
assert_a_re_acquisition_invalidates_the_marker_baseline_in_local_mode
assert_an_untouched_image_set_keeps_the_marker_baseline
assert_the_pre_gate_adopts_the_recorded_cluster_mode_over_the_environment
assert_a_recorded_local_build_renders_the_local_overlay
assert_the_migration_job_renders_the_overlay_the_cluster_runs
assert_the_marker_records_the_image_source_and_tag
assert_a_mode_change_forces_a_full_resync
assert_a_tag_change_forces_a_full_resync
assert_a_re_acquisition_forces_a_full_resync
assert_make_deploy_all_renders_the_mode_aware_overlay
assert_the_render_dir_resolver_follows_the_recorded_mode
assert_an_overridden_tag_renders_from_a_copy_carrying_that_tag
assert_a_render_copy_carries_the_generated_api_ip_patch
assert_a_render_copy_without_the_generated_api_ip_patch_fails_with_a_remedy
assert_the_pinned_render_dir_is_not_gated_on_the_generated_patch
assert_an_unknown_image_source_is_rejected
assert_a_targeted_build_carries_the_recorded_coordinate_forward
assert_the_touched_scripts_parse
assert_incremental_runtime_calls_are_bounded
assert_empty_incremental_docker_env_fails_closed
assert_every_defined_case_is_invoked

exit $FAIL