Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# Enforces the "sync to minikube before every gate" rule from the platform E2E plan.
set -euo pipefail
PRE_GATE_SYNC_CONFIG_ONLY="${PRE_GATE_SYNC_CONFIG_ONLY:-}"
T2_SETUP_HANDOFF_EXPECTED="${T2_SETUP_HANDOFF_EXPECTED:-}"
T2_SETUP_HANDOFF_TRANSITION="${T2_SETUP_HANDOFF_TRANSITION:-}"
T2_SETUP_HANDOFF_ROOT="${T2_SETUP_HANDOFF_ROOT:-}"
T2_SETUP_HANDOFF_TTL_SECONDS="${T2_SETUP_HANDOFF_TTL_SECONDS:-}"
if [ -z "$PRE_GATE_SYNC_CONFIG_ONLY" ]; then PRE_GATE_SYNC_CONFIG_ONLY=false; fi
if [ -z "$T2_SETUP_HANDOFF_EXPECTED" ]; then T2_SETUP_HANDOFF_EXPECTED=false; fi
if [ -z "$T2_SETUP_HANDOFF_TTL_SECONDS" ]; then T2_SETUP_HANDOFF_TTL_SECONDS=300; fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
if [ -z "$T2_SETUP_HANDOFF_ROOT" ]; then
T2_SETUP_HANDOFF_ROOT="${PROJECT_DIR}/.local-notes/infra/t2-setup-handoffs"
fi
PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
KC="kubectl --context=${PROFILE}"
# shellcheck source=t2-worktree-id.sh
source "$SCRIPT_DIR/t2-worktree-id.sh"
WORKTREE_ID="$(t2_worktree_id "${PROJECT_DIR}")"
STATE_ROOT="${TMPDIR:-/tmp}/clerum-pre-gate-sync"
STATE_DIR="${STATE_ROOT}/${WORKTREE_ID}"
CLUSTER_SYNC_STATE_CONFIGMAP="${CLERUM_PRE_GATE_SYNC_CONFIGMAP:-clerum-pre-gate-sync-state}"
mkdir -p "${STATE_DIR}"
GATE_NAME="pre-gate"
FORCE_CLUSTER_SYNC=false
FORCE_RESTART=false
SKIP_PORT_FORWARDS=false
while [[ $# -gt 0 ]]; do
case "$1" in
--gate)
GATE_NAME="${2:?missing gate name}"
shift 2
;;
--force-cluster-sync)
FORCE_CLUSTER_SYNC=true
shift
;;
--force-restart)
FORCE_RESTART=true
shift
;;
--skip-port-forwards)
SKIP_PORT_FORWARDS=true
shift
;;
*)
echo "Unknown argument: $1" >&2
exit 1
;;
esac
done
# The pre-gate entrypoint is also a mutating recovery boundary. It must carry
# the same branch/profile identity as the T2 orchestrator, or acquire that
# identity itself when invoked standalone.
T2_PROJECT_DIR="$PROJECT_DIR"
T2_PROFILE="$PROFILE"
T2_CONTEXT="$PROFILE"
T2_GATE_ID="$GATE_NAME"
# shellcheck source=scripts/minikube/t2-common.sh
source "$SCRIPT_DIR/t2-common.sh"
if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
t2_require_commands
t2_repo_metadata
t2_profile_scope
t2_context_check
t2_profile_status
t2_profile_context_identity_check
fi
log() {
printf '[pre-gate-sync] %s\n' "$*"
}
# ---- Which images this cluster runs ---------------------------------------
# THIS HAS TO COME FROM THE CLUSTER, NOT FROM THIS SHELL. A cluster set up with
# ghcr release images and pre-gated with IMAGE_SOURCE=local (or the reverse)
# would otherwise look in sync while every running pod referenced images this
# sync never touched, and the gate would pass against code that is not
# deployed. scripts/minikube/image-mode.sh is the one resolver; it reads the
# mode whichever writer last acquired images recorded, and falls back to the
# environment only for a cluster nothing has built or pulled into yet.
# shellcheck source=scripts/minikube/image-mode.sh
source "${SCRIPT_DIR}/image-mode.sh"
# shellcheck source=scripts/minikube/pre-gate-marker.sh
source "${SCRIPT_DIR}/pre-gate-marker.sh"
if ! IMAGE_SOURCE="$(image_mode_source "${PROJECT_DIR}")"; then
exit 1
fi
if ! IMAGE_TAG="$(image_mode_tag "${PROJECT_DIR}")"; then
exit 1
fi
if ! PRE_GATE_RENDER_DIR="$(image_mode_render_dir "${PROJECT_DIR}")"; then
exit 1
fi
if ! IMAGES_GENERATED_AT="$(image_mode_images_generated_at "${PROJECT_DIR}")"; then
exit 1
fi
export IMAGE_SOURCE
if [[ "${IMAGE_SOURCE}" == "ghcr" ]]; then
# Every `make` sub-invocation below (pull-images, verify-images, deploy-all)
# must resolve the SAME coordinate this run resolved, including an override
# the committed pin does not carry.
export MINIKUBE_IMAGE_TAG="${IMAGE_TAG}"
fi
# shellcheck source=scripts/minikube/pre-gate-runtime.sh
source "${SCRIPT_DIR}/pre-gate-runtime.sh"
# shellcheck source=scripts/minikube/pre-gate-incremental.sh
source "${SCRIPT_DIR}/pre-gate-incremental.sh"
# shellcheck source=scripts/minikube/t2-setup-handoff.sh
source "${SCRIPT_DIR}/t2-setup-handoff.sh"
case "${T2_SETUP_HANDOFF_EXPECTED}" in
true|false) ;;
*) log "ERROR: T2_SETUP_HANDOFF_EXPECTED must be true or false"; exit 1 ;;
esac
# A database/schema migration must not race the eager workflow reconciler. Keep
# the existing replica count so a failed gate restores the branch-owned
# control-plane shape instead of silently leaving reconciliation disabled.
WRC_FENCED=false
WRC_REPLICAS=1
CONTROL_API_FENCED=false
CONTROL_API_REPLICAS=1
fence_workflow_reconciler() {
local deployment_probe
if ! deployment_probe="$(${KC} get deployment/workflow-recipes -n control-plane 2>&1)"; then
if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
log "Workflow reconciler is not present; no rollout fence is required"
return 0
fi
log "ERROR: unable to inspect workflow-recipes before schema migration; refusing to continue without its writer fence"
return 1
fi
WRC_REPLICAS="$(${KC} get deployment/workflow-recipes -n control-plane -o jsonpath='{.spec.replicas}')"
WRC_REPLICAS="${WRC_REPLICAS:-1}"
log "Fencing workflow reconciler at ${WRC_REPLICAS} replica(s) before schema migration"
WRC_FENCED=true
${KC} scale deployment/workflow-recipes -n control-plane --replicas=0 >/dev/null
${KC} rollout status deployment/workflow-recipes -n control-plane --timeout=60s >/dev/null
}
restore_workflow_reconciler() {
if [[ "${WRC_FENCED}" != "true" ]]; then
return 0
fi
set +e
log "Restoring workflow reconciler to ${WRC_REPLICAS} replica(s)"
local scale_rc=0 rollout_rc=0
${KC} scale deployment/workflow-recipes -n control-plane --replicas="${WRC_REPLICAS}" >/dev/null || scale_rc=$?
if [[ "${scale_rc}" -eq 0 ]]; then
${KC} rollout status deployment/workflow-recipes -n control-plane --timeout=120s >/dev/null || rollout_rc=$?
fi
set -e
if [[ "${scale_rc}" -ne 0 || "${rollout_rc}" -ne 0 ]]; then
log "ERROR: workflow reconciler restore failed (scale=${scale_rc}, rollout=${rollout_rc}); leaving the writer fence armed"
return 1
fi
WRC_FENCED=false
}
fence_control_api() {
local deployment_probe
if ! deployment_probe="$(${KC} get deployment/control-api -n control-plane 2>&1)"; then
if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
log "Control API is not present; no writer fence is required"
return 0
fi
log "ERROR: unable to inspect control-api before schema migration; refusing to continue without its writer fence"
return 1
fi
CONTROL_API_REPLICAS="$(${KC} get deployment/control-api -n control-plane -o jsonpath='{.spec.replicas}')"
CONTROL_API_REPLICAS="${CONTROL_API_REPLICAS:-1}"
if [[ ! "${CONTROL_API_REPLICAS}" =~ ^[0-9]+$ ]]; then
log "ERROR: unable to determine control-api replica count; refusing schema-first sync"
return 1
fi
log "Fencing Control API writers at ${CONTROL_API_REPLICAS} replica(s) before schema migration"
CONTROL_API_FENCED=true
${KC} scale deployment/control-api -n control-plane --replicas=0 >/dev/null
${KC} rollout status deployment/control-api -n control-plane --timeout=60s >/dev/null
}
restore_control_api() {
if [[ "${CONTROL_API_FENCED}" != "true" ]]; then
return 0
fi
set +e
log "Restoring Control API to ${CONTROL_API_REPLICAS} replica(s)"
local scale_rc=0 rollout_rc=0
${KC} scale deployment/control-api -n control-plane --replicas="${CONTROL_API_REPLICAS}" >/dev/null || scale_rc=$?
if [[ "${scale_rc}" -eq 0 ]]; then
${KC} rollout status deployment/control-api -n control-plane --timeout=120s >/dev/null || rollout_rc=$?
fi
set -e
if [[ "${scale_rc}" -ne 0 || "${rollout_rc}" -ne 0 ]]; then
log "ERROR: Control API restore failed (scale=${scale_rc}, rollout=${rollout_rc}); leaving the writer fence armed"
return 1
fi
CONTROL_API_FENCED=false
}
restore_pre_gate_writers() {
local status=0
# Restore Control API first so it can serve the final readiness checks; WRC
# remains last because it is the consumer that must stay stopped until every
# migration, writer restart, and legacy-policy gate has completed.
restore_control_api || status=1
restore_workflow_reconciler || status=1
return "$status"
}
finalize_pre_gate_sync() {
local status=$? restore_status=0
# An EXIT trap's return value does not replace the script's exit status. Exit
# explicitly so a failed writer restore cannot turn a successful sync into a
# green exact-head attestation. Disable the trap first to avoid recursion.
trap - EXIT
if ! restore_pre_gate_writers; then
restore_status=1
fi
if [[ "${status}" -eq 0 && "${restore_status}" -ne 0 ]]; then
status=1
fi
# t2_lock_release returns the status passed to it; pass zero so an existing
# non-zero script status is preserved rather than misclassified as a lock
# cleanup failure.
t2_lock_release 0 || status=1
exit "${status}"
}
commit_cluster_sync_state() {
local cluster_fingerprint="$1" infra_fingerprint="$2"
# The exact-head marker is a certification artifact, not merely progress.
# Restore every fenced writer before publishing it so a cleanup failure can
# never leave a marker that claims the cluster is ready at this HEAD.
restore_pre_gate_writers || return 1
persist_cluster_marker "${cluster_fingerprint}" "${infra_fingerprint}" || return 1
persist_state cluster "${cluster_fingerprint}" || return 1
persist_state infra "${infra_fingerprint}" || return 1
}
if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
t2_mutation_lock
fi
export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV \
T2_SKIP_LOCK T2_LOCK_TOKEN
trap finalize_pre_gate_sync EXIT
fingerprint_dir() {
pre_gate_marker_fingerprint_dir "${PROJECT_DIR}" "$1"
}
state_file_for() {
local key="$1"
local safe_key
safe_key="$(echo "${key}" | tr '/.' '__')"
echo "${STATE_DIR}/${safe_key}.sha"
}
has_changed() {
local key="$1"
local current="$2"
local state_file
state_file="$(state_file_for "${key}")"
if [[ ! -f "${state_file}" ]]; then
return 0
fi
[[ "$(cat "${state_file}")" != "${current}" ]]
}
persist_state() {
local key="$1"
local value="$2"
printf '%s' "${value}" >"$(state_file_for "${key}")"
}
cluster_marker_value() {
local field="$1" output
if output="$(${KC} get configmap "${CLUSTER_SYNC_STATE_CONFIGMAP}" -n control-plane \
-o "jsonpath={.data.${field}}" 2>&1)"; then
printf '%s' "${output}"
return 0
fi
if [[ "${output}" == *NotFound* || "${output}" == *"not found"* ]]; then
printf ''
return 0
fi
log "WARNING: unable to inspect ${CLUSTER_SYNC_STATE_CONFIGMAP}.${field}; treating marker as stale: ${output}" >&2
printf ''
}
cluster_marker_matches() {
local expected_cluster_fingerprint="$1"
local expected_worktree_id="$2"
local expected_git_head actual_cluster_fingerprint actual_worktree_id actual_git_head
local actual_image_source actual_image_tag actual_images_generated_at
expected_git_head="$(git -C "${PROJECT_DIR}" rev-parse --verify HEAD 2>/dev/null || true)"
actual_cluster_fingerprint="$(cluster_marker_value clusterFingerprint)"
actual_worktree_id="$(cluster_marker_value worktreeId)"
actual_git_head="$(cluster_marker_value gitHead)"
# The image coordinate is part of "in sync". gitHead cannot tell a ghcr
# cluster from a local one, nor v0.6.0 from latest, and the acquisition stamp
# is what catches a `make minikube-setup` between two pre-gates: that re-pulls
# every release image and discards any shadow build while leaving this marker
# untouched, so without it no sync would run and the gate would silently test
# release code.
actual_image_source="$(cluster_marker_value imageSource)"
actual_image_tag="$(cluster_marker_value imageTag)"
actual_images_generated_at="$(cluster_marker_value imagesGeneratedAt)"
[[ "${actual_cluster_fingerprint}" == "${expected_cluster_fingerprint}" ]] &&
[[ "${actual_worktree_id}" == "${expected_worktree_id}" ]] &&
[[ -n "${expected_git_head}" && "${actual_git_head}" == "${expected_git_head}" ]] &&
[[ "${actual_image_source}" == "${IMAGE_SOURCE}" ]] &&
[[ "${actual_image_tag}" == "${IMAGE_TAG}" ]] &&
[[ "${actual_images_generated_at}" == "${IMAGES_GENERATED_AT}" ]]
}
persist_cluster_marker() {
local cluster_fingerprint="$1"
local infra_fingerprint="$2"
local git_head updated_at images_generated_at
git_head="$(git -C "${PROJECT_DIR}" rev-parse --verify HEAD 2>/dev/null || printf 'unknown')"
updated_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
# Re-read rather than reusing the value from startup: this run may have
# re-pulled or rebuilt images, which rewrites the manifest.
if ! images_generated_at="$(image_mode_images_generated_at "${PROJECT_DIR}")"; then
log "ERROR: cannot read the image manifest to stamp the cluster sync marker"
return 1
fi
${KC} create configmap "${CLUSTER_SYNC_STATE_CONFIGMAP}" -n control-plane \
--from-literal=clusterFingerprint="${cluster_fingerprint}" \
--from-literal=infraFingerprint="${infra_fingerprint}" \
--from-literal=worktreeId="${WORKTREE_ID}" \
--from-literal=gitHead="${git_head}" \
--from-literal=imageSource="${IMAGE_SOURCE}" \
--from-literal=imageTag="${IMAGE_TAG}" \
--from-literal=imagesGeneratedAt="${images_generated_at}" \
--from-literal=gate="${GATE_NAME}" \
--from-literal=updatedAt="${updated_at}" \
--dry-run=client \
-o yaml | ${KC} apply -f - >/dev/null
}
run_if_changed() {
local dir="$1"
local cmd="$2"
local fingerprint
fingerprint="$(fingerprint_dir "${dir}")"
if has_changed "pkg:${dir}" "${fingerprint}"; then
log "Running targeted unit tests for ${dir}"
(
cd "${PROJECT_DIR}/${dir}"
eval "${cmd}"
)
persist_state "pkg:${dir}" "${fingerprint}"
else
log "No package changes detected for ${dir}"
fi
}
ensure_artifact() {
local dir="$1"
local artifact="$2"
local cmd="$3"
if [[ -e "${PROJECT_DIR}/${dir}/${artifact}" ]]; then
return 0
fi
log "Building ${dir} because ${artifact} is missing"
(
cd "${PROJECT_DIR}/${dir}"
eval "${cmd}"
)
}
sync_mcp_host_auth_key() {
local source_probe target_probe
if ! source_probe="$(${KC} get secret rpc-proxy-secrets -n rpc-proxy 2>&1)"; then
if [[ "${source_probe}" == *NotFound* || "${source_probe}" == *"not found"* ]]; then
log "Skipping mcp-host auth key sync (rpc-proxy-secrets not found)"
return 0
fi
log "ERROR: unable to inspect rpc-proxy/rpc-proxy-secrets before MCP auth sync"
return 1
fi
if ! target_probe="$(${KC} get configmap mcp-host-config -n mcp-host 2>&1)"; then
if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then
log "Skipping mcp-host auth key sync (mcp-host-config not found)"
return 0
fi
log "ERROR: unable to inspect mcp-host/mcp-host-config before MCP auth sync"
return 1
fi
bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --skip-gfs --require-mcp
}
sync_gfs_auth_key() {
local allow_staged="${1:-false}"
GFS_AUTH_SYNC_ALLOW_STAGED="${allow_staged}" \
bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --require-gfs
}
settle_gfs_reader_rollout() {
T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
CONTEXT="$T2_CONTEXT" \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
}
reconcile_gfs_credentials() {
PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
CONTEXT="$T2_CONTEXT" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
}
converge_gfs_reader_after_restore() {
# A reader whose runtime role was NOLOGIN keeps serving 503 from its
# already-started pod even after the role is restored. Converging it here
# keeps the repair inside the single orchestrated run instead of a manual
# `kubectl rollout restart deploy/gfsc-reader` side quest.
local deployment=gfsc-reader deployment_probe desired ready
if ! deployment_probe="$(${KC} get deployment "${deployment}" -n gfs 2>&1)"; then
if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
return 0
fi
log "ERROR: unable to inspect gfs/${deployment} before credential convergence"
return 1
fi
if ! desired="$(${KC} get deployment "${deployment}" -n gfs -o jsonpath='{.spec.replicas}' 2>/dev/null)"; then
log "ERROR: unable to read desired replicas for gfs/${deployment}"
return 1
fi
if [[ ! "${desired}" =~ ^[0-9]+$ ]]; then
log "ERROR: desired replicas for gfs/${deployment} is not numeric"
return 1
fi
if [[ "${desired}" == "0" ]]; then
return 0
fi
if ! ready="$(${KC} get deployment "${deployment}" -n gfs -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; then
log "ERROR: unable to read Ready replicas for gfs/${deployment}"
return 1
fi
# Kubernetes omits status.readyReplicas while a Deployment has zero Ready
# pods. An empty successful read is a real 0; an API failure above remains
# unknown and fails closed.
ready="${ready:-0}"
if [[ ! "${ready}" =~ ^[0-9]+$ ]]; then
log "ERROR: Ready replicas for gfs/${deployment} is not numeric"
return 1
fi
if [[ "${ready}" == "${desired}" ]]; then
return 0
fi
log "Restarting gfs/${deployment} after credential restore (${ready:-0}/${desired} Ready)"
rollout_restart_with_retry gfs "${deployment}"
# HCC's gfsReconciler now preserves restartedAt. Delete live unready reader
# pods once so they re-read the restored Secret without waiting out
# CrashLoopBackOff, then judge readiness directly (leftover ReplicaSets can
# still make a generation-based rollout status wait the wrong revision).
local pod_rows pod_name pod_ready pod_deleting
if ! pod_rows="$(${KC} get pods -n gfs -l 'app=gfs-controller,clerum.io/gfsc-role=reader' -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \
2>&1)"; then
if [[ "$pod_rows" == *NotFound* || "$pod_rows" == *"not found"* ]]; then
pod_rows=""
else
log "ERROR: unable to inspect gfs reader pods before convergence: $pod_rows"
return 1
fi
fi
while IFS='|' read -r pod_name pod_ready pod_deleting; do
[ -n "$pod_name" ] || continue
[ -z "$pod_deleting" ] || continue
[ "$pod_ready" != True ] || continue
if ! ${KC} delete pod "$pod_name" -n gfs --ignore-not-found --wait=false >/dev/null 2>&1; then
log "ERROR: unable to delete unready gfs reader pod $pod_name"
return 1
fi
done <<<"$pod_rows"
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/wait-gfs-reader-ready.sh"
}
provision_gfs_serving() {
if [[ "${GATE_NAME}" != "minikube-t2" ]]; then
# Other platform security gates must retain their read/verify contract.
# GFS credential recovery is a T2 transition because it can patch Secrets,
# restart GFSC, delete pods, and resume an abandoned rollout claim.
log "Skipping GFS serving mutation before ${GATE_NAME}; only minikube-t2 owns GFS recovery"
sync_mcp_host_auth_key
return 0
fi
if ${KC} get configmap gfs-config -n gfs >/dev/null 2>&1; then
local reader_probe
if ! reader_probe="$(${KC} get deployment gfsc-reader -n gfs 2>&1)"; then
if [[ "${reader_probe}" == *NotFound* || "${reader_probe}" == *"not found"* ]]; then
log "Skipping gfs serving provisioning (gfsc-reader not found — GFS reader is not deployed)"
sync_mcp_host_auth_key
return 0
fi
log "ERROR: unable to inspect gfs/gfsc-reader before credential recovery"
return 1
fi
log "Provisioning gfs serving before ${GATE_NAME}"
# FAIL LOUD: with the GFS stack deployed, a broken gfs_controller credential
# means every GFS operation 503s (issue #775). Continuing would burn the
# whole gate run on a cluster that cannot pass.
# GFS_RESTORE_ACTIVE_NOLOGIN=true: a role-reset T1 suite can leave the
# cluster-global reader role NOLOGIN. On this local branch-owned profile
# the committed Secret DSN is the source of truth, so the recovery helper
# may restore the role from it — the same contract the GFS T1 gate already
# applies on exit. The helper still fails loud when the restored
# credential cannot authenticate; no password is invented here.
# GFS_RECOVER_ABANDONED_STATE: a dead prior setup can leave
# rollout-running; this sync holds the T2 lock, so resume that claim.
# An interrupted prior setup can leave gfs-config.jwt-public-key empty
# (the overlay re-applies the base ConfigMap with an empty value); a
# reader pod cannot start without it and any readiness wait would only
# time out. Re-sync it first; this is a no-op when the key matches.
sync_gfs_auth_key true
# Settle a Ready reader first so reconcile does not restart it and
# race HCC's gfsReconciler during kubectl rollout status. If reconcile
# still needs a reader rollout, the gfs-rollout-shim PATH prefix makes
# that wait judge readiness instead of a leftover ReplicaSet generation.
if ! settle_gfs_reader_rollout; then
log "ERROR: unable to settle the branch-owned GFS reader rollout before credential reconciliation"
exit 1
fi
if ! reconcile_gfs_credentials; then
log "ERROR: gfs DB provisioning FAILED — gfsc cannot authorize any operation. Aborting ${GATE_NAME} pre-gate sync."
exit 1
fi
# The first staged sync may have deferred active-pod proof because this
# reconcile call is the producer of gfs-controller-db. Re-run strict proof
# after it so the durable marker is never published before consumers have
# the key and a usable DSN.
if ! sync_gfs_auth_key; then
log "ERROR: gfs auth-key convergence failed after credential reconciliation"
return 1
fi
if ! converge_gfs_reader_after_restore; then
log "ERROR: gfs reader did not converge after credential restoration"
return 1
fi
else
log "Skipping gfs serving provisioning (gfs-config not found — GFS stack not deployed)"
sync_mcp_host_auth_key
fi
}
assert_no_legacy_prompt_bridge_grants() {
# Target-aware promptBridge authorization is fail-closed for legacy rows.
# Inventory only policies that require migration review; intentionally
# disabled/revoking grants are already fenced and must not block an upgrade.
# This check deliberately does not infer a provider, model, slot, or order;
# migration remains an explicit operator-reviewed action.
local legacy_count query
# A full image sync restarts the database deployment without waiting for the
# whole namespace. Wait for the exact Postgres deployment here before
# kubectl exec; otherwise the deployment resolver can select a completed
# migration Job during the restart window and produce a false gate failure.
rollout_if_present control-plane control-postgres
query="
SELECT count(*)::int
FROM plugin_workload_sdk_grants
WHERE capability_family = 'promptBridge'
AND policy_state = 'legacy_unreviewed';
"
# Keep the command after `kubectl exec --` on the same continued shell
# command. A bare newline after `--` makes kubectl receive no command and
# then runs `psql` on the host, producing a misleading local-socket error.
legacy_count="$(${KC} exec -n control-plane deployment/control-postgres -- \
psql -U postgres -d profiles -Atqc "${query}")"
legacy_count="$(printf '%s' "${legacy_count}" | tr -d '[:space:]')"
if [[ ! "${legacy_count}" =~ ^[0-9]+$ ]]; then
log "ERROR: unable to inventory legacy promptBridge grants; refusing target-aware rollout"
return 1
fi
if [[ "${legacy_count}" != "0" ]]; then
log "ERROR: ${legacy_count} legacy promptBridge grant(s) require operator-reviewed target migration before mcp-host rollout"
return 1
fi
log "Legacy promptBridge grant inventory is empty"
}
# Test seam: everything above resolves configuration and defines functions
# only, with no cluster or network call. `PRE_GATE_SYNC_CONFIG_ONLY=true` stops
# here so scripts/tests/test-minikube-pre-gate-shadow.sh can read the resolved
# image coordinate, and (when it sources this file) exercise the marker
# functions, without a cluster. It must stay immediately before the first
# cluster call.
if [[ "${PRE_GATE_SYNC_CONFIG_ONLY:-false}" == "true" ]]; then
printf 'imageSource=%s\n' "${IMAGE_SOURCE}"
printf 'imageTag=%s\n' "${IMAGE_TAG}"
printf 'renderDir=%s\n' "${PRE_GATE_RENDER_DIR}"
printf 'imagesGeneratedAt=%s\n' "${IMAGES_GENERATED_AT}"
# `return` when sourced, `exit` when executed.
return 0 2>/dev/null || exit 0
fi
log "Evaluating sync requirements for ${GATE_NAME}"
log "Cluster images: ${IMAGE_SOURCE}${IMAGE_TAG:+ (${IMAGE_TAG})}; overlay $(basename "${PRE_GATE_RENDER_DIR}")"
preflight_host_lifecycle_probe
if ! cluster_fingerprint="$(pre_gate_marker_cluster_fingerprint "${PROJECT_DIR}")"; then
log "ERROR: unable to compute the cluster fingerprint; refusing to sync or stamp the marker"
exit 1
fi
if ! infra_fingerprint="$(pre_gate_marker_infra_fingerprint "${PROJECT_DIR}")"; then
log "ERROR: unable to compute the infrastructure fingerprint; refusing to sync or stamp the marker"
exit 1
fi
cluster_changed=false
infra_changed=false
if [[ "${FORCE_CLUSTER_SYNC}" == "true" ]] || has_changed cluster "${cluster_fingerprint}"; then
cluster_changed=true
elif ! cluster_marker_matches "${cluster_fingerprint}" "${WORKTREE_ID}"; then
log "Cluster sync marker is missing or belongs to a different worktree/fingerprint"
cluster_changed=true
fi
if [[ "${FORCE_RESTART}" == "true" ]] || has_changed infra "${infra_fingerprint}"; then
infra_changed=true
fi
setup_handoff_result=not-expected
if [[ "${T2_SETUP_HANDOFF_EXPECTED}" == "true" ]]; then
if [[ "${GATE_NAME}" != "minikube-t2" || "${T2_SKIP_LOCK}" != "true" ||
"${IMAGE_SOURCE}" != "local" ]]; then
log "ERROR: setup-complete handoff requires the inherited minikube-t2 lease and local image mode"
exit 1
fi
if T2_PROJECT_DIR="${T2_PROJECT_DIR}" T2_WORKTREE_ID="${T2_WORKTREE_ID}" \
T2_RUN_ID="${T2_RUN_ID}" T2_PROFILE="${T2_PROFILE}" T2_CONTEXT="${T2_CONTEXT}" \
T2_BRANCH="${T2_BRANCH}" T2_HEAD="${T2_HEAD}" T2_SKIP_LOCK="${T2_SKIP_LOCK}" \
T2_LOCK_KEY="${T2_LOCK_KEY}" T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" \
T2_IMAGE_MANIFEST="${T2_IMAGE_MANIFEST}" \
T2_SETUP_HANDOFF_ROOT="${T2_SETUP_HANDOFF_ROOT}" \
T2_SETUP_HANDOFF_TTL_SECONDS="${T2_SETUP_HANDOFF_TTL_SECONDS}" \
T2_SETUP_HANDOFF_TRANSITION="${T2_SETUP_HANDOFF_TRANSITION}" \
bash "${SCRIPT_DIR}/t2-setup-handoff.sh" consume -- \
make --no-print-directory minikube-verify-images; then
setup_handoff_result=consumed
log "Consumed the verified one-shot setup-complete handoff"
else
setup_handoff_result=rejected
cluster_changed=true
log "Setup-complete handoff was not accepted; forcing the safe full reconcile"
fi
fi
run_if_changed packages/workflow-runtime-core "npm test && npm run build"
ensure_artifact packages/workflow-runtime-core dist/index.js "npm run build"
# @clerum/network-policy-core is a file: dependency of workflow-recipes and
# host-context-controller (issue #299). Run its node:test suite when it changes.
run_if_changed packages/network-policy-core "npm test"
run_if_changed control-api "npm test"
run_if_changed external-rest-api "npm test"
run_if_changed rpc-proxy "npm test"
run_if_changed mcp-host "npm test"
run_if_changed host-context-controller "npm test"
run_if_changed workflow-recipes "npm test"
run_if_changed packages/workflow-sdk "npm test"
run_if_changed workflow-approval-request-reader "npm test"
run_if_changed control-ui "npm test"
run_if_changed desktop-app "npm test"
if [[ "${cluster_changed}" == "true" ]]; then
incremental_plan
if [[ "${T2_SETUP_HANDOFF_EXPECTED}" == "true" ]]; then
if ! t2_setup_handoff_apply_plan_result "${setup_handoff_result}"; then
log "ERROR: setup-complete handoff reached an ambiguous plan state"
exit 1
fi
if [[ "${setup_handoff_result}" == consumed ]]; then
log "Reusing the attested images/deployment from the completed full setup"
fi
fi
log "Cluster sync plan before ${GATE_NAME}: images=$(incremental_target_summary), full-image-build=${INCREMENTAL_FULL_IMAGE_BUILD}, full-deployment=${INCREMENTAL_FULL_DEPLOYMENT}"
if [[ "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ||
"${INCREMENTAL_FULL_DEPLOYMENT}" == "true" ]]; then
(
cd "${PROJECT_DIR}"
gfs_crd_mutation=false
[[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_crd_mutation=true
T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" \
MINIKUBE_GFS_MUTATION="${gfs_crd_mutation}" make minikube-deploy-crds
)
if [[ "${GATE_NAME}" == "minikube-t2" ]]; then
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
else
log "Skipping GFS writer Secret mutation before ${GATE_NAME}; only minikube-t2 owns GFS recovery"
fi
writer_dsn="$(${KC} -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"
if [[ -n "${writer_dsn}" ]]; then
# Do not reconcile the GFS roles before the migration window. The
# current control-api image may add a new least-privilege projection
# (0095 currently adds lifecycle/link columns); provision-gfs-db.sh
# correctly refuses a role that does not have that projection yet. The
# post-migration provision_gfs_serving call below is the authoritative
# reconciliation point after schema and runtime roles have converged.
log "Upgrade path — deferring GFS credential reconciliation until after schema migration"
else
log "Fresh bootstrap — reader staging deferred until post-migration convergence; GFSC remains fail-closed"
fi
fi
if incremental_requires_database_reconcile; then
fence_workflow_reconciler
fence_control_api
fi
incremental_build_images
if incremental_requires_database_reconcile; then
rollout_if_present control-plane control-postgres
# The migration script EXTRACTS THE CONTROL-API IMAGE from the overlay it
# renders. Hardcoding the local overlay here yielded clerum/control-api:test
# on a ghcr cluster -- an image that does not exist there, so the migration
# Job ImagePullBackOffs; and rendering the pinned ghcr overlay on a cluster
# running an overridden tag would run a release nobody pulled. Follow the
# coordinate the cluster actually runs, which is also the ref the shadow
# build above just retagged.
CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \
--overlay "${PRE_GATE_RENDER_DIR}"
CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"
# The GFS DSN authentication probe deliberately runs inside control-api so
# the connection material stays on stdin and never enters argv or the host.
# Control API was fenced only for the schema migration/writer window; the
# runtime roles now exist, so restore it before GFS provisioning makes that
# probe. The workflow reconciler remains fenced until the complete
# schema/credential/overlay sequence has converged.
restore_control_api
# The base manifest no longer declares connection-string (provisioning-owned
# key), so deploy-all cannot clobber it. Provisioning must still run AFTER
# control-api migrations (0048_gfs_permission_store creates the least-privilege gfs_controller
# role) for fresh profiles and to converge any stale credential before the
# rest of the gate observes service readiness.
provision_gfs_serving
fi
if ! incremental_requires_database_reconcile; then
# No schema work is planned, but a role-reset T1 run may still have left
# the cluster-global GFS runtime roles NOLOGIN while every image and
# manifest matched. The T2 transition owns this idempotent convergence;
# security-gate invocations return through the non-mutating guard above.
provision_gfs_serving
fi
# Schema/control-plane first: only after migrations, runtime roles, and
# inventory gates have converged may the new consumer workloads be applied or
# restarted. The workflow reconciler is fenced above for this window.
if [[ "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ||
"${INCREMENTAL_FULL_DEPLOYMENT}" == "true" ]]; then
(
cd "${PROJECT_DIR}"
make minikube-apply-secrets
gfs_mutation=false
[[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_mutation=true
T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" MINIKUBE_GFS_MUTATION="${gfs_mutation}" \
make minikube-deploy-all
if [[ "${FORCE_RESTART}" == "true" || "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ]]; then
make minikube-restart-all
fi
)
fi
# The schema-first window is over: all desired manifests/images are now in
# place, so restore the old replica count only after the old writer was fully
# stopped and migrations completed. Recreate strategy on the Deployment makes
# ordinary image/config rollouts obey the same no-overlap contract.
restore_control_api
# Release invariant: every path that can restart/serve mcp-host proves that
# no legacy grant is routable, not only the migration path.
assert_no_legacy_prompt_bridge_grants
ensure_evenfire_registry
incremental_restart_targets
# All three nginx gateway configs are mounted through subPath. Kubernetes
# updates the ConfigMaps but not the files in existing pods, so a deployment
# sync must roll all three before any SDK or NP-08 runtime assertion. The
# rpc gateway carries the redeem error_page mapping (#720).
if [[ "${INCREMENTAL_FULL_DEPLOYMENT}" == "true" ||
"${FORCE_RESTART}" == "true" ]]; then
rollout_restart_with_retry control-plane nginx-workflow-approval-gateway
rollout_if_present control-plane nginx-workflow-approval-gateway
rollout_restart_with_retry control-plane control-api-rpc-gateway
rollout_if_present control-plane control-api-rpc-gateway
rollout_restart_with_retry control-plane host-context-controller-api-gateway
rollout_if_present control-plane host-context-controller-api-gateway
fi
assert_workflow_gateway_prompt_bridge_finalization_route
assert_hcc_gateway_np08_routes
rollout_if_present control-plane host-context-controller
rollout_if_present control-plane workflow-recipes
rollout_if_present control-plane control-ui
rollout_if_present profiles external-rest-api
rollout_if_present rpc-proxy rpc-proxy
rollout_namespace_deployments mcp-host
rollout_if_present channels clerum-channel-reader
rollout_if_present channels clerum-workflow-approval-request-reader
if gate_needs_registry; then
rollout_if_present registry registry-api
fi
incremental_verify_gfs_if_required
log "Cluster status after sync"
${KC} get deploy -A --no-headers 2>/dev/null | grep -v kube-system || true
${KC} get sts -A --no-headers 2>/dev/null || true
if [[ "${SKIP_PORT_FORWARDS}" != "true" ]]; then
"${SCRIPT_DIR}/pf-all-stack.sh"
fi
# Full deploy may regenerate ignored, profile-specific inputs such as the
# Kubernetes API-IP patch. Recompute after that generation and before
# stamping the marker; otherwise a strict consumer can correctly reject a
# marker that describes the pre-generation tree.
if ! cluster_fingerprint="$(pre_gate_marker_cluster_fingerprint "${PROJECT_DIR}")"; then
log "ERROR: unable to recompute the cluster fingerprint after deployment; refusing to stamp the marker"
exit 1
fi
if ! infra_fingerprint="$(pre_gate_marker_infra_fingerprint "${PROJECT_DIR}")"; then
log "ERROR: unable to recompute the infrastructure fingerprint after deployment; refusing to stamp the marker"
exit 1
fi
commit_cluster_sync_state "${cluster_fingerprint}" "${infra_fingerprint}"
log "Pre-gate cluster sync complete"
else
log "No cluster sync required before ${GATE_NAME}"
# Images and manifests match, yet the GFS runtime credentials can still be
# broken (a role-reset T1 leaves the reader NOLOGIN without changing any
# fingerprint). The T2 fast path repairs them in this single orchestrated
# run; non-T2 gates are fenced out by provision_gfs_serving.
provision_gfs_serving
assert_no_legacy_prompt_bridge_grants
ensure_evenfire_registry
rollout_if_present control-plane nginx-workflow-approval-gateway
assert_workflow_gateway_prompt_bridge_finalization_route
rollout_if_present control-plane host-context-controller-api-gateway
assert_hcc_gateway_np08_routes
${KC} get deploy -A --no-headers 2>/dev/null | grep -v kube-system || true
fi