Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# ======================================================================
# Pull Published Images into Minikube (IMAGE_SOURCE=ghcr)
# ======================================================================
#
# Pulls every pull_in_ghcr_mode image (deploy/images.json:
# published && deployed_to_minikube) from ghcr.io/evenfire-ai at the pinned
# release tag, in parallel, then aliases each one to its local clerum/* ref.
#
# This is the default path for `make minikube-setup`. Its local-build
# counterpart is scripts/minikube/build-images.sh.
#
# Usage:
# MINIKUBE_PROFILE=<branch-profile> ./scripts/minikube/pull-images.sh [--only=<svc>]
#
# Env:
# MINIKUBE_IMAGE_TAG Override the recorded/committed tag (render-time
# only, never committed). Use `latest` to exercise
# the pull path before a release tag exists.
# MINIKUBE_SKIP_UIS true -> do not pull control-ui/profile-ui, which
# the -no-uis overlays delete anyway (see below).
# MINIKUBE_PROFILE Target branch-owned profile (provided by Make/T2)
# MINIKUBE_MULTI_NODE true -> pull on the host + `minikube image load`
# MINIKUBE_PULL_PARALLELISM Concurrent pulls (default: 6, range: 1-64)
# MINIKUBE_IMAGE_PULL_RETRIES Attempts per image before it is reported
# failed (default: 3, range: 1-10). A transient registry blip
# (one bad pull 24 images into a run) should not
# abort the whole setup. Mirrors the naming and
# defaults of build-images.sh's
# MINIKUBE_BASE_IMAGE_PULL_RETRIES, applied here to
# the ghcr application-image pull instead of
# Dockerfile base images.
# MINIKUBE_IMAGE_PULL_DELAY_SECS Delay between failed attempts (default: 5,
# range: 0-300 seconds).
#
# Two behaviours here are load-bearing; do not "optimize" them away:
#
# 1. It RE-PULLS a tag already present in the daemon. A pre-gate shadow build
# (scripts/minikube/pre-gate-sync.sh) is deliberately tagged with the exact
# ghcr ref so IfNotPresent picks it up. Skipping present tags would let
# that local build survive across setups indefinitely and invisibly, and a
# gate would keep testing code that was never deployed.
#
# 2. It ALIASES each ghcr ref to clerum/<local_name>:<local_tag>. Four
# pull_in_ghcr_mode images have no row in the minikube overlay and are
# never rewritten by the ghcr component -- mcp-host-slim, mcp-host-full,
# mock-mcp-server, mock-stdio-mcp-server. They are consumed under their
# LOCAL names by McpServer CRD instances applied outside the overlay and
# by the E2E scripts under scripts/e2e/ (the minikube registry catalog
# seed publishes mcp-airtable with imageRef clerum/mock-mcp-server:test,
# so that alias is load-bearing for the registry install specs). A
# `docker tag` is a pointer; dropping it breaks those paths in ghcr mode
# only, which is the worst place to find out.
#
# airtable-mcp-server and web-search-mcp used to be in that list. They are
# deployed_to_minikube:false now: the evenfire registry distributes MCP
# servers and installs them on demand, writing the catalog entry's
# fully-qualified imageRef into McpServer.spec.image, so no local alias is
# involved. The one manifest still naming a local MCP-server ref
# (instances-e2e/airtable-server.yaml, SEED_PROFILE=e2e only) is served by
# an opt-in local build instead -- see MINIKUBE_BUILD_AIRTABLE_MCP_IMAGE.
#
# 3. --skip-uis / MINIKUBE_SKIP_UIS drops control-ui and profile-ui from the
# pull list, mirroring build-images.sh's identical filter. The
# -no-uis-ghcr overlay DELETES both Deployments, so pulling them costs
# ~470 MiB of transfer and disk for images no pod will ever reference.
# ======================================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
MINIKUBE_MULTI_NODE="${MINIKUBE_MULTI_NODE:-false}"
PULL_PARALLELISM="${MINIKUBE_PULL_PARALLELISM:-6}"
MINIKUBE_IMAGE_PULL_RETRIES="${MINIKUBE_IMAGE_PULL_RETRIES:-3}"
MINIKUBE_IMAGE_PULL_DELAY_SECS="${MINIKUBE_IMAGE_PULL_DELAY_SECS:-5}"
MINIKUBE_DOCKER_ENV_TIMEOUT_SECONDS="${MINIKUBE_DOCKER_ENV_TIMEOUT_SECONDS:-30}"
GHCR_NAMESPACE="ghcr.io/evenfire-ai"
ONLY_SVC=""
# Same normalizer and same flag name build-images.sh uses, so the two halves of
# `--skip-uis` cannot disagree about what the word means.
SKIP_UIS="${MINIKUBE_SKIP_UIS:-false}"
case "${SKIP_UIS}" in
true | 1 | yes) SKIP_UIS=true ;;
*) SKIP_UIS=false ;;
esac
for arg in "$@"; do
case "$arg" in
--only=*) ONLY_SVC="${arg#--only=}" ;;
--skip-uis) SKIP_UIS=true ;;
esac
done
validate_pull_integer() {
local name="$1" value="$2" minimum="$3" maximum="$4"
if ! [[ "${value}" =~ ^[0-9]+$ ]] ||
(( 10#${value} < minimum || 10#${value} > maximum )); then
err "MINIKUBE_PULL_CONFIG_INVALID: ${name} must be an integer from ${minimum} to ${maximum}"
exit 2
fi
}
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
log() { echo -e "${CYAN}[PULL]${NC} $*"; }
ok() { echo -e "${GREEN} OK${NC} -- $*"; }
warn() { echo -e "${YELLOW} WARN${NC} -- $*"; }
err() { echo -e "${RED} ERROR${NC} -- $*" >&2; }
validate_manifest_image_id() {
local image="$1" image_id="$2"
if [[ -z "${image_id}" || "${image_id}" == "NOT_PULLED" ||
! "${image_id}" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then
err "IMAGE_MANIFEST_INVALID: docker inspect returned an invalid image ID for ${image}"
return 1
fi
}
validate_pull_integer MINIKUBE_PULL_PARALLELISM "${PULL_PARALLELISM}" 1 64
validate_pull_integer MINIKUBE_IMAGE_PULL_RETRIES "${MINIKUBE_IMAGE_PULL_RETRIES}" 1 10
validate_pull_integer MINIKUBE_IMAGE_PULL_DELAY_SECS "${MINIKUBE_IMAGE_PULL_DELAY_SECS}" 0 300
# ---- Resolve the effective tag -------------------------------------------
# THE TAG FOLLOWS THE CLUSTER, NOT JUST THE COMMIT. Reading the committed pin
# here (with MINIKUBE_IMAGE_TAG as the only override) ignored the tag the last
# acquisition actually recorded, so `make minikube-pull-images` on a cluster
# bootstrapped with `MINIKUBE_IMAGE_TAG=latest` went and pulled the pinned
# v0.6.0 -- which is 404 on ghcr until the tag is cut. That command is the
# remedy the docs and this script's own failure text point people at, so it
# failed precisely when someone followed the advice.
#
# image_mode_ghcr_tag applies override -> recorded -> pin, the same precedence
# build-images.sh --verify-only and pre-gate-sync.sh already use. It is called
# rather than image_mode_tag because pulling IS a ghcr acquisition: this script
# records imageSource "ghcr" when it finishes, so a cluster whose current
# record says "local" must still resolve a tag instead of an empty string.
# shellcheck source=scripts/minikube/image-mode.sh
source "${SCRIPT_DIR}/image-mode.sh"
# shellcheck source=scripts/minikube/docker-cli-env.sh
source "${SCRIPT_DIR}/docker-cli-env.sh"
# Pulling and loading images mutates the branch-owned Minikube profile. The
# caller must already hold the canonical T2 mutation lease; this child only
# validates the opaque inherited token and never acquires or releases it.
if [[ -z "${T2_PROJECT_DIR:-}" || -z "${T2_PROFILE:-}" ||
-z "${T2_CONTEXT:-}" || "${T2_PROFILE}" != "${T2_CONTEXT}" ||
"${T2_PROFILE}" != "${PROFILE}" ]]; then
err "PROFILE_LOCK_REQUIRED: pull-images.sh requires an inherited branch-profile mutation lease"
exit 1
fi
T2_PROJECT_DIR="${T2_PROJECT_DIR}" T2_PROFILE="${T2_PROFILE}" \
T2_CONTEXT="${T2_CONTEXT}" T2_SKIP_LOCK=true \
MINIKUBE_PROFILE="${PROFILE}" CONTROL_API_REAL_PG_CONTEXT="${T2_CONTEXT}" \
bash "${SCRIPT_DIR}/require-t2-mutation-lock.sh"
IMAGE_TAG="$(image_mode_ghcr_tag "$PROJECT_DIR")" || exit 1
TAG_ORIGIN="$(image_mode_tag_origin "$PROJECT_DIR")" || exit 1
if [ -z "$IMAGE_TAG" ] || [ -z "$TAG_ORIGIN" ]; then
err "could not resolve a ghcr tag to pull (tag='${IMAGE_TAG}' origin='${TAG_ORIGIN}')"
err "Set MINIKUBE_IMAGE_TAG=<tag> explicitly, or fix deploy/components/ghcr-images/kustomization.yaml."
exit 1
fi
# ---- Resolve the pull list ----------------------------------------------
# Emitted as TAB-separated `name<TAB>localRef` by the ONE manifest reader, so
# this script never re-derives published && deployed_to_minikube itself.
PULL_LIST="$(node -e '
import("'"$PROJECT_DIR"'/scripts/release/images-manifest.mjs").then(m => {
for (const image of m.pullInGhcrMode()) {
process.stdout.write(`${image.name}\t${m.localRef(image)}\n`)
}
}).catch(err => {
process.stderr.write(`images-manifest read failed: ${err.message}\n`)
process.exit(1)
})')"
if [ -z "$PULL_LIST" ]; then
err "the manifest produced zero pull_in_ghcr_mode images; refusing to report success on an empty pull"
exit 1
fi
echo -e "\n${BOLD}=== Pulling Published Images (${IMAGE_TAG}) ===${NC}"
log "tag ${IMAGE_TAG} (${TAG_ORIGIN})"
log "profile ${PROFILE}, parallelism ${PULL_PARALLELISM}"
if [ "$SKIP_UIS" = true ]; then
log "skipping control-ui and profile-ui (--skip-uis); the -no-uis overlay deletes both Deployments"
fi
STATUS_DIR="$(mktemp -d)"
MANIFEST_FILE="${PROJECT_DIR}/deploy/minikube/.image-manifest.json"
MANIFEST_TMP_FILE=""
cleanup() {
local status=$? cleanup_status=0
trap - EXIT INT TERM
rm -rf -- "$STATUS_DIR" || cleanup_status=$?
if [ -n "${MANIFEST_TMP_FILE}" ]; then
rm -f -- "${MANIFEST_TMP_FILE}" || cleanup_status=$?
fi
docker_cli_env_cleanup || cleanup_status=$?
if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
status="$cleanup_status"
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# Resolve an approved local endpoint into an empty task-local Docker config
# before asking Minikube for its daemon endpoint. Every subsequent Docker and
# image-load operation then runs through the same isolated config and deadline
# runner, with ambient auth/context/header variables excluded.
docker_cli_env_prepare false
if [ "$MINIKUBE_MULTI_NODE" = false ]; then
log "Configuring Docker CLI to use minikube's Docker daemon..."
docker_env_status=0
docker_env_output="$(docker_cli_run_public minikube-docker-env \
"$MINIKUBE_DOCKER_ENV_TIMEOUT_SECONDS" \
minikube -p "$PROFILE" docker-env --shell bash)" || docker_env_status=$?
if [ "$docker_env_status" -ne 0 ] || [ -z "$docker_env_output" ]; then
err "DOCKER_ENV_UNRESOLVED: could not resolve Docker environment for minikube '${PROFILE}'"
if [ "$docker_env_status" -ne 0 ]; then
exit "$docker_env_status"
fi
exit 1
fi
eval "$docker_env_output"
unset DOCKER_API_VERSION 2>/dev/null || true
else
log "Multi-node profile: pulling on the host, then 'minikube image load' onto every node"
fi
# ---- Bounded retry for one pull ------------------------------------------
# A transient registry blip (one bad pull 24 images into a run) should not
# read the same as a tag that genuinely does not exist. Mirrors
# build-images.sh's pull_host_image_with_retry: same attempt-count/delay
# shape and the same defaults, applied here to the ghcr application-image
# pull instead of a Dockerfile base image.
#
# Every attempt is a real `docker pull` -- retrying never turns into "skip if
# present"; see header note 1. out_file is truncated on each attempt, so on
# final failure it holds only the last attempt's diagnostic, not a pile-up of
# every prior one.
pull_with_retry() {
local ref="$1" out_file="$2" image_name="$3"
local attempt=1
while [ "$attempt" -le "$MINIKUBE_IMAGE_PULL_RETRIES" ]; do
if docker_cli_run_public "pull-image-${image_name}" \
"$MINIKUBE_DOCKER_PULL_TIMEOUT_SECONDS" docker pull "$ref" >"$out_file" 2>&1; then
return 0
fi
if [ "$attempt" -lt "$MINIKUBE_IMAGE_PULL_RETRIES" ]; then
warn "Pull failed for ${ref} (attempt ${attempt}/${MINIKUBE_IMAGE_PULL_RETRIES}); retrying in ${MINIKUBE_IMAGE_PULL_DELAY_SECS}s"
sleep "$MINIKUBE_IMAGE_PULL_DELAY_SECS"
fi
attempt=$((attempt + 1))
done
return 1
}
# ---- One image ----------------------------------------------------------
pull_one() {
local name="$1" local_ref="$2" slot="$3"
local ghcr_ref="${GHCR_NAMESPACE}/${name}:${IMAGE_TAG}"
# Always pull. Never skip a present tag -- see the header.
if ! pull_with_retry "$ghcr_ref" "${STATUS_DIR}/${slot}.out" "$name"; then
printf '%s' "$ghcr_ref" > "${STATUS_DIR}/${slot}.failed"
return 0
fi
if [ "$MINIKUBE_MULTI_NODE" = true ]; then
docker_cli_run_public "minikube-image-load-${name}" \
"$MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS" \
minikube -p "$PROFILE" image load "$ghcr_ref" >/dev/null 2>&1 || {
printf '%s' "$ghcr_ref" > "${STATUS_DIR}/${slot}.failed"
return 0
}
fi
# The local alias. See header note 2.
if ! docker_cli_run_public "tag-image-${name}" \
"$MINIKUBE_DOCKER_INFO_TIMEOUT_SECONDS" docker tag "$ghcr_ref" "$local_ref" \
>>"${STATUS_DIR}/${slot}.out" 2>&1; then
printf '%s' "$ghcr_ref (alias to ${local_ref})" > "${STATUS_DIR}/${slot}.failed"
return 0
fi
if [ "$MINIKUBE_MULTI_NODE" = true ]; then
if ! docker_cli_run_public "minikube-image-load-alias-${name}" \
"$MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS" \
minikube -p "$PROFILE" image load "$local_ref" >/dev/null 2>&1; then
printf '%s' "$ghcr_ref (alias load to ${local_ref})" > "${STATUS_DIR}/${slot}.failed"
return 0
fi
fi
printf '%s\t%s' "$ghcr_ref" "$local_ref" > "${STATUS_DIR}/${slot}.done"
}
# ---- Parallel fan-out ---------------------------------------------------
slot=0
selected=0
while IFS="$(printf '\t')" read -r name local_ref; do
[ -n "$name" ] || continue
if [ -n "$ONLY_SVC" ] && [[ "$name" != *"$ONLY_SVC"* ]] && [[ "$local_ref" != *"$ONLY_SVC"* ]]; then
continue
fi
# Exact names, not a substring: the -no-uis overlays delete exactly the
# control-ui and profile-ui workloads, and a looser match would silently drop
# any future image whose name happens to contain "ui".
if [ "$SKIP_UIS" = true ]; then
case "$name" in
control-ui | profile-ui) continue ;;
esac
fi
selected=$((selected + 1))
slot=$((slot + 1))
# Bounded concurrency without GNU parallel or `wait -n` (bash 3.2 on macOS).
while [ "$(jobs -pr | wc -l | tr -d ' ')" -ge "$PULL_PARALLELISM" ]; do
sleep 0.2
done
pull_one "$name" "$local_ref" "$slot" &
done <<< "$PULL_LIST"
wait
if [ "$selected" -eq 0 ]; then
err "--only=${ONLY_SVC} matched no pull_in_ghcr_mode image"
exit 1
fi
# ---- Report -------------------------------------------------------------
FAILED=()
FAILED_SLOTS=()
for f in "${STATUS_DIR}"/*.failed; do
[ -e "$f" ] || continue
FAILED+=("$(cat "$f")")
FAILED_SLOTS+=("$(basename "$f" .failed)")
done
DONE_COUNT=0
for f in "${STATUS_DIR}"/*.done; do
[ -e "$f" ] || continue
DONE_COUNT=$((DONE_COUNT + 1))
ok "$(cut -f1 "$f") -> $(cut -f2 "$f")"
done
if [ "${#FAILED[@]}" -gt 0 ]; then
echo ""
err "${#FAILED[@]} of ${selected} image(s) could not be pulled at tag '${IMAGE_TAG}':"
idx=0
for ref in "${FAILED[@]}"; do
err " ${ref}"
# Surface the diagnostic docker already captured instead of discarding it.
# A transient network error and a missing tag both print just the ref
# here; the "why" only exists in the captured output. Trimmed to the last
# few lines -- layer-progress noise can run to hundreds of lines per
# image, and several images can fail at once.
out_file="${STATUS_DIR}/${FAILED_SLOTS[$idx]}.out"
if [ -s "$out_file" ]; then
while IFS= read -r diag_line; do
err " ${diag_line}"
done < <(tail -n 5 "$out_file")
fi
idx=$((idx + 1))
done
echo "" >&2
err "The tag '${IMAGE_TAG}' came from the ${TAG_ORIGIN}."
if [ "$DONE_COUNT" -gt 0 ]; then
# Other images pulled fine at this exact tag, so the tag itself is not in
# question -- this is a per-image failure (registry blip, timeout), not a
# tag that hasn't been promoted yet.
err "${DONE_COUNT} other image(s) pulled fine at '${IMAGE_TAG}', so the tag exists;"
err "the failure(s) above look transient. Retry:"
err " make minikube-setup"
elif [ "$IMAGE_TAG" != "latest" ]; then
err "If '${IMAGE_TAG}' has not been promoted yet (the release-prep commit reaches main"
err "BEFORE the tag is cut and the images are promoted), pull a tag that exists:"
err " MINIKUBE_IMAGE_TAG=latest make minikube-setup"
else
# IMAGE_TAG is already 'latest' and every selected image failed -- do not
# recommend the tag that just failed (verbatim from a real run).
err "Every selected image failed at '${IMAGE_TAG}', which is already the widest tag;"
err "this does not look like an unpromoted tag. Retry, or check registry/network access:"
err " make minikube-setup"
fi
err "Or build everything locally instead:"
err " make minikube-setup-local"
exit 1
fi
# ---- Image manifest -----------------------------------------------------
# Same shape build-images.sh writes. scripts/minikube/full-setup.sh:608 uses
# this file as a `find -newer` staleness marker, and its absence makes the ghcr
# path look like "images were never built".
#
# `imageSource` is not decoration: `build-images.sh --verify-only` reads it back
# to decide whether to verify ghcr.io/evenfire-ai/* or clerum/* refs. Without a
# recorded mode it falls back to the IMAGE_SOURCE env default (ghcr), which is
# how a locally built cluster came to be reported as "25 of 28 images missing".
# This writer only ever pulls, so the value is always "ghcr".
# Resolve and validate every digest before opening the existing manifest. A
# failed inspect is inventory failure, not evidence that an image was absent;
# publishing NOT_PULLED or an empty value would create green evidence for a
# run whose acquisition cannot be verified.
MANIFEST_GHCR_REFS=()
MANIFEST_LOCAL_REFS=()
MANIFEST_SHAS=()
for f in "${STATUS_DIR}"/*.done; do
[ -e "$f" ] || continue
ghcr_ref="$(cut -f1 "$f")"
local_ref="$(cut -f2 "$f")"
manifest_inspect_status=0
sha="$(docker_cli_run_public "inspect-image" \
"$MINIKUBE_DOCKER_INFO_TIMEOUT_SECONDS" \
docker inspect --format='{{.Id}}' "$ghcr_ref")" || manifest_inspect_status=$?
if [ "$manifest_inspect_status" -ne 0 ]; then
err "Could not inspect ${ghcr_ref} while generating the image manifest"
exit "$manifest_inspect_status"
fi
validate_manifest_image_id "$ghcr_ref" "$sha" || exit 1
MANIFEST_GHCR_REFS+=("$ghcr_ref")
MANIFEST_LOCAL_REFS+=("$local_ref")
MANIFEST_SHAS+=("$sha")
done
mkdir -p "$(dirname "$MANIFEST_FILE")"
MANIFEST_TMP_FILE="$(mktemp "${MANIFEST_FILE}.tmp.XXXXXX")" || {
err "IMAGE_MANIFEST_INVALID: could not create a temporary manifest beside ${MANIFEST_FILE}"
exit 1
}
{
echo "{"
echo " \"generated\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","
echo " \"profile\": \"${PROFILE}\","
echo " \"imageSource\": \"ghcr\","
echo " \"imageTag\": \"${IMAGE_TAG}\","
echo " \"images\": {"
first=true
for image_index in "${!MANIFEST_SHAS[@]}"; do
sha="${MANIFEST_SHAS[$image_index]}"
for ref in "${MANIFEST_GHCR_REFS[$image_index]}" "${MANIFEST_LOCAL_REFS[$image_index]}"; do
if [ "$first" = true ]; then first=false; else echo ","; fi
printf ' "%s": "%s"' "$ref" "$sha"
done
done
echo ""
echo " }"
echo "}"
} > "$MANIFEST_TMP_FILE"
if ! node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' \
"$MANIFEST_TMP_FILE"; then
err "IMAGE_MANIFEST_INVALID: generated manifest is not valid JSON"
exit 1
fi
if ! mv -- "$MANIFEST_TMP_FILE" "$MANIFEST_FILE"; then
err "IMAGE_MANIFEST_INVALID: could not publish the validated manifest"
exit 1
fi
MANIFEST_TMP_FILE=""
ok "Manifest: deploy/minikube/.image-manifest.json"
echo ""
echo -e "${GREEN}${BOLD}All ${selected} published image(s) pulled at ${IMAGE_TAG} into minikube '${PROFILE}'.${NC}"