Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: harden PR 416 rollout and validation gates

Alfredo Lopez1bc24dccbdeb
diff --git a/Makefile b/Makefile
index c725d91df..51aec064c 100644
--- a/Makefile
+++ b/Makefile
@@ -155,12 +155,17 @@ minikube-stop: ## Stop minikube cluster
# pulls ~25 published images instead of building 28 from source. Build
# everything locally with `make minikube-setup-local` (or IMAGE_SOURCE=local).
IMAGE_SOURCE ?= ghcr
-MINIKUBE_GFS_MUTATION ?= true
+# GFS Secret/role mutation is owned by the canonical T2 lease. Callers that
+# intentionally run the full T2 transition set this to true and pass the
+# inherited opaque T2_LOCK_TOKEN; ordinary deploys render/filter GFS resources.
+MINIKUBE_GFS_MUTATION ?= false
.PHONY: minikube-setup
minikube-setup: ## Clean install from scratch, PULLING published images (IMAGE_SOURCE=local or `make minikube-setup-local` builds instead). Rebuilds the DB; REUSE_DB=true keeps it. SKIP_UIS=true omits Control/Profile UI. Runs 'prereqs' first (SKIP_PREREQS=true to bypass). Needs ADMIN_PASSWORD in .env.
@if [ "$(SKIP_PREREQS)" != "true" ]; then $(MAKE) --no-print-directory prereqs; fi
- @MINIKUBE_SKIP_UIS="$(SKIP_UIS)" MINIKUBE_SEED_PROFILE="$(SEED_PROFILE)" REUSE_DB="$(REUSE_DB)" \
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ MINIKUBE_SKIP_UIS="$(SKIP_UIS)" MINIKUBE_SEED_PROFILE="$(SEED_PROFILE)" REUSE_DB="$(REUSE_DB)" \
IMAGE_SOURCE="$(IMAGE_SOURCE)" MINIKUBE_IMAGE_TAG="$(MINIKUBE_IMAGE_TAG)" \
scripts/minikube/full-setup.sh $(ARGS)
@@ -224,6 +229,9 @@ minikube-detect-k8s-api-ip: ## Patch overlays/minikube/patches/k8s-api-ip.yaml w
.PHONY: minikube-deploy-all
minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
@$(MAKE) --no-print-directory minikube-detect-k8s-api-ip
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ] && [ -z "$(T2_LOCK_TOKEN)" ]; then \
+ echo "[minikube-deploy-all] refusing GFS mutation without the owning T2 profile lock" >&2; exit 1; \
+ fi
@# Upgrade path: adopt/validate writer and stage reader before HCC cutover.
@if [ "$(MINIKUBE_GFS_MUTATION)" != "true" ]; then echo "[minikube-deploy-all] GFS mutation disabled for this non-T2 sync"; fi
@if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
@@ -235,7 +243,7 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \
echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \
}; \
- CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
else \
echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \
fi; \
@@ -286,7 +294,7 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
if [ -n "$$gfs_config_probe" ]; then \
if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \
CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/wait-gfsc-secret-references.sh; \
- CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
else \
echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \
fi; \
@@ -316,8 +324,17 @@ minikube-restart-all: ## Restart all Clerum deployments
.PHONY: minikube-deploy-crds
minikube-deploy-crds: ## Install/upgrade CRDs via Helm chart + apply CRD YAML (idempotent)
kubectl --context=$(MINIKUBE_PROFILE) apply -f deploy/base/namespaces.yaml
- helm upgrade --install --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds
- kubectl --context=$(MINIKUBE_PROFILE) apply -f ./charts/clerum-crds/crds/
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "false" ]; then \
+ echo "[minikube-deploy-crds] GFS CRD mutation disabled for this non-T2 gate"; \
+ helm upgrade --install --skip-crds --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds; \
+ for crd in ./charts/clerum-crds/crds/*.yaml; do \
+ case "$$crd" in *globalfilesystem.yaml) continue ;; esac; \
+ kubectl --context=$(MINIKUBE_PROFILE) apply -f "$$crd"; \
+ done; \
+ else \
+ helm upgrade --install --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds; \
+ kubectl --context=$(MINIKUBE_PROFILE) apply -f ./charts/clerum-crds/crds/; \
+ fi
.PHONY: minikube-deploy-service
minikube-deploy-service: ## Rebuild single image + rollout restart deployment (usage: make minikube-deploy-service SVC=mcp-host NS=mcp-host [DEPLOYMENT=chatllm])
@@ -388,14 +405,20 @@ minikube-apply-namespaces: ## Create all namespaces
.PHONY: minikube-sync-auth-key
minikube-sync-auth-key: ## Sync JWT public key from rpc-proxy-secrets into runtime ConfigMaps when drift exists
+ @if [ -z "$(T2_LOCK_TOKEN)" ]; then \
+ echo "[minikube-sync-auth-key] refusing ConfigMap mutation without the owning T2 profile lock" >&2; exit 1; \
+ fi
@if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE); \
else \
+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE) --skip-gfs --require-mcp; \
fi
.PHONY: minikube-sync-auth-key-if-present
minikube-sync-auth-key-if-present: ## Sync JWT public key only when minikube auth resources already exist
+ @if [ -z "$(T2_LOCK_TOKEN)" ]; then echo "Skipping auth key sync (canonical T2 profile lock is not held)."; exit 0; fi
@if ! $(KC) get secret rpc-proxy-secrets -n rpc-proxy >/dev/null 2>&1; then \
echo "Skipping auth key sync (rpc-proxy-secrets not found yet)."; \
elif ! $(KC) get configmap mcp-host-config -n mcp-host >/dev/null 2>&1; then \
diff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
index 76f6cd279..cab7d63a6 100755
--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh
+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
@@ -6,12 +6,13 @@ CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
GFS_NS="${GFS_NS:-gfs}"
set +u
-T2_PROJECT_DIR="$T2_PROJECT_DIR"
-T2_PROFILE="$T2_PROFILE"
-T2_CONTEXT="$T2_CONTEXT"
-MINIKUBE_PROFILE="$MINIKUBE_PROFILE"
-CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
-T2_LOCK_TOKEN="$T2_LOCK_TOKEN"
+T2_PROJECT_DIR="${T2_PROJECT_DIR:-}"
+T2_PROFILE="${T2_PROFILE:-}"
+T2_CONTEXT="${T2_CONTEXT:-}"
+MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-}"
+CONTROL_API_REAL_PG_CONTEXT="${CONTROL_API_REAL_PG_CONTEXT:-}"
+T2_LOCK_TOKEN="${T2_LOCK_TOKEN:-}"
+T2_SKIP_LOCK="${T2_SKIP_LOCK:-}"
set -u
if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fi
if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi
@@ -26,14 +27,30 @@ remote_context_allowed() {
[ -n "${ALLOWED_CONTEXTS:-}" ] || return 1
IFS=',' read -r -a allowed_contexts <<<"${ALLOWED_CONTEXTS}"
for allowed_context in "${allowed_contexts[@]}"; do
- if [ "$allowed_context" = "$CONTEXT" ]; then
- return 0
- fi
+ [ "$allowed_context" = "$CONTEXT" ] && return 0
done
return 1
}
-if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; then
+minikube_profile_exists() {
+ local status
+ status="$(minikube -p "$CONTEXT" status 2>/dev/null || true)"
+ [ -n "$status" ] &&
+ [[ "$status" != *"does not exist"* && "$status" != *"not found"* && "$status" != *Nonexistent* ]]
+}
+
+# The sibling infra deploy path owns its explicit non-local context decision.
+# A local Minikube profile, even if a caller supplies the legacy remote flag,
+# must still use the branch-owned T2 lease below.
+REMOTE_RECONCILE=false
+if remote_context_allowed && ! minikube_profile_exists; then
+ REMOTE_RECONCILE=true
+fi
+
+if [ "$REMOTE_RECONCILE" != true ]; then
+ # Every local invocation is a profile-owned mutating boundary. A caller may
+ # either inherit the opaque lease from T2 or acquire one here; context names
+ # and caller-provided allowlists are not local authorization.
# shellcheck source=scripts/minikube/t2-common.sh
source "$ROOT/scripts/minikube/t2-common.sh"
if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
@@ -69,23 +86,19 @@ if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; then
trap 'handle_reconcile_signal TERM' TERM
t2_repo_metadata
t2_profile_scope
+ t2_profile_status
+ if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
+ T2_NEXT_COMMAND="start and bootstrap the branch-owned profile, then retry GFS reconciliation"
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "branch-owned Minikube profile is not bootstrapped: $T2_PROFILE"
+ fi
t2_context_check
+ t2_profile_context_identity_check
t2_mutation_lock
- # The verified profile context is authoritative. Do not let a caller keep
- # a second CONTEXT value after the fence has been acquired.
+ # The verified profile context is authoritative. Do not let a caller keep a
+ # second CONTEXT value after the fence has been acquired.
CONTEXT="$T2_CONTEXT"
else
- # Remote reconciliation is authorized by the explicit workflow decision and
- # exact context membership, never by a provider/name convention. This also
- # supports self-hosted dev contexts whose names are not gke_*.
- if ! remote_context_allowed; then
- if [ -z "${GFS_REMOTE_RECONCILE_AUTHORIZED:-}" ] && [ -z "${ALLOWED_CONTEXTS:-}" ]; then
- printf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2
- exit 1
- fi
- printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2
- exit 1
- fi
+ printf '[reconcile-gfs-deploy] using the explicitly authorized non-local context: %s\n' "$CONTEXT" >&2
fi
CONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
index 0a17bd1eb..414d48ab8 100755
--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -216,15 +216,27 @@ EOF
}
delete_synthetic_fleet() {
- local failed=0 kind
+ local failed=0 kind host_query host
+ if ! host_query="$(kctl get host -A \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
+ -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)"; then
+ printf 'HCC cleanup: unable to enumerate synthetic Host names for run %s; refusing broad child deletion\n' "$RUN_ID" >&2
+ return 1
+ fi
for kind in mcpserver context host; do
- kctl delete "$kind" -A -l e2e.clerum.io/suite=hcc-watch-churn \
+ kctl delete "$kind" -A \
+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \
--ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 || failed=1
done
kctl delete secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found >/dev/null 2>&1 || failed=1
- # HCC-managed runtime left behind by the fleet Hosts.
- kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \
- -A -l "clerum.io/managed-by=host-context-controller" --ignore-not-found >/dev/null 2>&1 || true
+ # HCC-managed runtime left behind by these exact synthetic Hosts. Never use
+ # the cluster-wide managed-by label: legitimate platform Hosts can own
+ # resources with that label while this cleanup is running.
+ while IFS= read -r host; do
+ [ -n "$host" ] || continue
+ kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \
+ -A -l "clerum.io/host=${host}" --ignore-not-found >/dev/null 2>&1 || failed=1
+ done <<<"$host_query"
return "$failed"
}
diff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.sh
index aa1b550f7..a4dc4fbc8 100755
--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh
+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh
@@ -244,9 +244,18 @@ assert_botched_outage() {
else
fail "could not list HCC pod UIDs — refusing to treat a query failure as old-pod termination"
fi
+ deployment_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.metadata.deletionTimestamp}{"\t"}{.metadata.ownerReferences[?(@.kind=="ReplicaSet")].name}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].image}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
+ replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" -v image="$BOTCHED_IMAGE" \
+ '$2 != old && $2 != "" && $4 == "" && $3 != "Failed" && $3 != "Succeeded" && $5 != "" && $6 == image && $7 ~ /ErrImagePull|ImagePullBackOff/ { print; exit }' <<<"$stuck_pod_rows")"
+ replacement_owner="$(cut -f5 <<<"$replacement_row")"
+ replacement_owner_revision=""
+ if [ -n "$replacement_owner" ]; then
+ replacement_owner_revision="$(kctl get rs "$replacement_owner" -n "$HCC_NS" \
+ -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
+ fi
case "$flavor" in
recovery)
@@ -278,7 +287,11 @@ assert_botched_outage() {
fail "$image_fail"
[ "$old_uid_live" = 0 ] && ok "$old_pod_ok" || fail "$old_pod_fail"
if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"
+ if [ -n "$deployment_revision" ] && [ "$replacement_owner_revision" = "$deployment_revision" ]; then
+ ok "replacement pod is a live current-revision pod pinned by the injected image failure: ${replacement_row}"
+ else
+ fail "replacement pod owner revision ${replacement_owner_revision:-unset} does not match Deployment revision ${deployment_revision:-unset}"
+ fi
else
fail "$replacement_fail"
fi
diff --git a/scripts/e2e/gfs-real-pg-minikube-gate.sh b/scripts/e2e/gfs-real-pg-minikube-gate.sh
index e743a3afa..73e147b90 100755
--- a/scripts/e2e/gfs-real-pg-minikube-gate.sh
+++ b/scripts/e2e/gfs-real-pg-minikube-gate.sh
@@ -1,10 +1,11 @@
#!/usr/bin/env bash
-# Run the real-Postgres T1 suites against the control-postgres instance in an
-# explicitly validated, branch-owned Minikube profile.
+# Run the GFS real-Postgres T1 suites with an isolated PostgreSQL instance while
+# validating the target branch-owned Minikube profile and restoring its GFS
+# runtime roles on exit.
#
# This runner is deliberately separate from CI: CI supplies its own ephemeral
-# Postgres DSN, while this gate obtains a short-lived localhost port-forward to
-# the selected Minikube service. It never discovers or accepts a GKE context.
+# Postgres DSN, while this gate starts a disposable local PostgreSQL container.
+# It never mutates the shared control-postgres database used by the profile.
set -euo pipefail
set +x
@@ -13,10 +14,26 @@ PROJECT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
CONTEXT="${CONTEXT:-${MINIKUBE_PROFILE:-}}"
SYNC_CONFIGMAP="${CLERUM_PRE_GATE_SYNC_CONFIGMAP:-clerum-pre-gate-sync-state}"
CONTROL_NS="${CONTROL_NS:-control-plane}"
-PG_SERVICE="${PG_SERVICE:-control-postgres}"
-PG_SECRET="${PG_SECRET:-control-postgres}"
TIMEOUT="${TIMEOUT:-120}"
-PORTS_ENV="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${CONTEXT}/ports.env}"
+ISOLATED_PG_IMAGE="${ISOLATED_PG_IMAGE:-postgres:16-alpine}"
+ISOLATED_HOST='127.0.0.1'
+
+# Reuse the canonical T2 ownership, context-identity, marker, and mutation
+# lock contract. This standalone GFS lane is still a T1 helper, but it mutates
+# cluster-global GFS roles and therefore must have the same branch-owned safety
+# boundary as the main T2 orchestrator.
+T2_PROJECT_DIR="${PROJECT_DIR}"
+MINIKUBE_PROFILE="${CONTEXT}"
+T2_PROFILE="${CONTEXT}"
+T2_CONTEXT="${CONTEXT}"
+T2_PROFILE_ROOT="${T2_PROFILE_ROOT:-${CLERUM_PROFILE_CACHE_ROOT:-${HOME}/.cache/clerum/minikube-profiles}}"
+T2_PROFILE_ENV="${T2_PROFILE_ENV:-${T2_PROFILE_ROOT}/${CONTEXT}/profile.env}"
+T2_PORTS_ENV="${T2_PORTS_ENV:-${CLERUM_PROFILE_PORTS_ENV:-${T2_PROFILE_ROOT}/${CONTEXT}/ports.env}}"
+T2_MARKER_NAME="${SYNC_CONFIGMAP}"
+T2_CONTROL_NAMESPACE="${CONTROL_NS}"
+export T2_PROJECT_DIR MINIKUBE_PROFILE T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT \
+ T2_PROFILE_ENV T2_PORTS_ENV T2_MARKER_NAME T2_CONTROL_NAMESPACE
+source "${PROJECT_DIR}/scripts/minikube/t2-common.sh"
die() {
printf '[gfs-real-pg-minikube] ERROR: %s\n' "$*" >&2
@@ -34,68 +51,21 @@ require_command() {
done
}
-is_branch_profile() {
- [[ "${1}" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]
-}
-
-reject_unsafe_context() {
- [[ -n "${CONTEXT}" ]] || die 'CONTEXT or MINIKUBE_PROFILE is required'
- case "${CONTEXT}" in
- gke_*) die "refusing non-Minikube or production context: ${CONTEXT}" ;;
- *prod*) die "refusing non-Minikube or production context: ${CONTEXT}" ;;
- esac
- if [[ "${CONTEXT}" != "clerum-test" ]] && ! is_branch_profile "${CONTEXT}"; then
- die "context is not an allowed local/branch profile: ${CONTEXT}"
+verify_branch_gate() {
+ t2_require_commands
+ t2_repo_metadata
+ t2_profile_scope
+ # Profile status is checked before context identity so a missing/stopped
+ # profile produces the supported bootstrap transition instead of an opaque
+ # kube-context failure.
+ t2_profile_status
+ if [ "${T2_BOOTSTRAP_REQUIRED}" = true ]; then
+ die "branch-owned Minikube profile is not bootstrapped: ${T2_PROFILE}"
fi
-}
-
-verify_profile_ownership() {
- if ! is_branch_profile "${CONTEXT}"; then
- return 0
- fi
- [[ -f "${PORTS_ENV}" ]] || die "branch profile ports.env is missing: ${PORTS_ENV}"
-
- local profile_name profile_repo profile_dir
- profile_dir="${PORTS_ENV%/ports.env}"
- profile_name="$(awk -F= '$1 == "PROFILE" { print substr($0, index($0, "=") + 1); exit }' "${profile_dir}/profile.env" 2>/dev/null || true)"
- profile_repo="$(awk -F= '$1 == "REPO_DIR" { print substr($0, index($0, "=") + 1); exit }' "${profile_dir}/profile.env" 2>/dev/null || true)"
- [[ "${profile_name}" == "${CONTEXT}" ]] || die "ports.env/profile marker belongs to '${profile_name:-unknown}', not ${CONTEXT}"
- [[ -n "${profile_repo}" ]] || die 'profile marker has no REPO_DIR'
- [[ "$(cd -- "${profile_repo}" 2>/dev/null && pwd -P)" == "${PROJECT_DIR}" ]] || \
- die "profile marker is owned by another worktree: ${profile_repo}"
-}
-
-verify_clean_and_sync_marker() {
- local head worktree_id marker_json
- [[ -z "$(git -C "${PROJECT_DIR}" status --porcelain)" ]] || \
- die 'worktree is dirty; run pre-gate-sync after committing or cleanly restoring changes'
- head="$(git -C "${PROJECT_DIR}" rev-parse --verify HEAD)"
- worktree_id="$(printf '%s' "${PROJECT_DIR}" | shasum | awk '{print $1}')"
- marker_json="$(kc -n "${CONTROL_NS}" get configmap "${SYNC_CONFIGMAP}" -o json 2>/dev/null)" || \
- die "validated pre-gate marker is missing: ${CONTROL_NS}/${SYNC_CONFIGMAP}"
-
- python3 - "${worktree_id}" "${head}" "${marker_json}" <<'PY'
-import json
-import sys
-
-expected_worktree, expected_head, marker_json = sys.argv[1:]
-payload = json.loads(marker_json)
-data = payload.get("data") or {}
-if not data.get("clusterFingerprint"):
- raise SystemExit("pre-gate marker has no cluster fingerprint")
-if data.get("worktreeId") != expected_worktree:
- raise SystemExit("pre-gate marker belongs to another worktree")
-if data.get("gitHead") != expected_head:
- raise SystemExit("pre-gate marker does not match the current HEAD")
-PY
-}
-
-secret_value() {
- local key="$1" encoded
- encoded="$(kc -n "${CONTROL_NS}" get secret "${PG_SECRET}" -o "jsonpath={.data.${key}}" 2>/dev/null)" || \
- die "cannot read ${CONTROL_NS}/${PG_SECRET}.${key}"
- [[ -n "${encoded}" ]] || die "${CONTROL_NS}/${PG_SECRET}.${key} is empty"
- printf '%s' "${encoded}" | python3 -c 'import base64,sys; print(base64.b64decode(sys.stdin.read(), validate=True).decode(), end="")'
+ t2_context_check
+ t2_profile_context_identity_check
+ t2_mutation_lock
+ t2_marker_check
}
wait_for_tcp() {
@@ -131,6 +101,64 @@ path.write_text(text)
PY
}
+PROFILE_PG_FORWARD_PID=""
+
+stop_profile_postgres_forward() {
+ [[ -n "${PROFILE_PG_FORWARD_PID}" ]] || return 0
+ if ! kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ PROFILE_PG_FORWARD_PID=''
+ return 0
+ fi
+ local command_line
+ command_line="$(ps -p "${PROFILE_PG_FORWARD_PID}" -o command= 2>/dev/null || true)"
+ if [[ "${command_line}" != *port-forward* || "${command_line}" != *svc/control-postgres* ]]; then
+ printf '[gfs-real-pg-minikube] ERROR: profile PostgreSQL forward PID is not the expected child; refusing cleanup success\n' >&2
+ return 1
+ fi
+ if ! kill "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1 && kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ return 1
+ fi
+ if kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ if wait "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ :
+ fi
+ fi
+ if kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ return 1
+ fi
+ PROFILE_PG_FORWARD_PID=''
+}
+
+verify_profile_postgres() {
+ kc -n "${CONTROL_NS}" rollout status deployment/control-postgres --timeout="${TIMEOUT}s" >/dev/null 2>&1 || \
+ die 'branch-profile control-postgres did not become Ready'
+
+ local key encoded profile_port
+ for key in POSTGRES_USER POSTGRES_PASSWORD POSTGRES_DB; do
+ encoded="$(kc -n "${CONTROL_NS}" get secret control-postgres -o "jsonpath={.data.${key}}" 2>/dev/null || true)"
+ [[ -n "${encoded}" ]] || die "branch-profile control-postgres Secret is missing ${key}"
+ printf '%s' "${encoded}" | python3 -c 'import base64,sys; base64.b64decode(sys.stdin.buffer.read(), validate=True)' >/dev/null 2>&1 || \
+ die "branch-profile control-postgres Secret key is not valid base64: ${key}"
+ done
+
+ profile_port="$(choose_local_port)"
+ kubectl --context="${CONTEXT}" -n "${CONTROL_NS}" port-forward \
+ --address=127.0.0.1 svc/control-postgres "${profile_port}:5432" \
+ >"${TMP_DIR}/profile-postgres-port-forward.log" 2>&1 &
+ PROFILE_PG_FORWARD_PID=$!
+ if ! wait_for_tcp "${profile_port}"; then
+ sanitize_file "${TMP_DIR}/profile-postgres-port-forward.log"
+ cat "${TMP_DIR}/profile-postgres-port-forward.log" >&2 || true
+ die 'branch-profile control-postgres port-forward did not become reachable'
+ fi
+ if ! kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then
+ sanitize_file "${TMP_DIR}/profile-postgres-port-forward.log"
+ cat "${TMP_DIR}/profile-postgres-port-forward.log" >&2 || true
+ die 'branch-profile control-postgres port-forward exited before the GFS lane started'
+ fi
+ stop_profile_postgres_forward || die 'could not cleanly close the branch-profile PostgreSQL precondition forward'
+}
+
real_postgres_suite_files() {
local package="$1"
case "${package}" in
@@ -218,22 +246,11 @@ PY
printf '[gfs-real-pg-minikube] PASS %s (%s suites, no skips)\n' "${package}" "${suite_count}"
}
-require_command git kubectl npm python3 shasum
-reject_unsafe_context
-verify_profile_ownership
-verify_clean_and_sync_marker
-kc -n "${CONTROL_NS}" get service "${PG_SERVICE}" >/dev/null || die "${CONTROL_NS}/${PG_SERVICE} service is missing"
-kc -n "${CONTROL_NS}" rollout status deployment/control-postgres --timeout="${TIMEOUT}s" >/dev/null || \
- die 'control-postgres is not Ready in the validated profile'
-
-PG_USER="$(secret_value POSTGRES_USER)"
-PG_PASSWORD="$(secret_value POSTGRES_PASSWORD)"
-PG_DATABASE="$(secret_value POSTGRES_DB)"
-[[ -n "${PG_USER}" && -n "${PG_PASSWORD}" && -n "${PG_DATABASE}" ]] || die 'control-postgres secret is incomplete'
-
TMP_DIR="$(mktemp -d)"
-PORT_FORWARD_PID=''
-LOCAL_PORT=''
+ISOLATED_CONTAINER=""
+ISOLATED_PORT=''
+ADMIN_DSN=''
+GFS_RESTORE_REQUIRED=false
restore_gfs_runtime_credentials() {
# The reader-role real-Postgres suite exercises the production role names,
# which are cluster-global even though its fixture database is temporary.
@@ -243,34 +260,32 @@ restore_gfs_runtime_credentials() {
printf '[gfs-real-pg-minikube] ERROR: required gfs-controller-db Secret is missing or unreadable; refusing to finish with GFS credentials unreconciled\n' >&2
return 1
fi
- if ! GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="${CONTEXT}" \
+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" \
+ T2_PROJECT_DIR="${T2_PROJECT_DIR}" T2_PROFILE="${T2_PROFILE}" T2_CONTEXT="${T2_CONTEXT}" \
+ T2_PROFILE_ROOT="${T2_PROFILE_ROOT}" T2_PROFILE_ENV="${T2_PROFILE_ENV}" T2_PORTS_ENV="${T2_PORTS_ENV}" \
+ GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="${CONTEXT}" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then
printf '[gfs-real-pg-minikube] ERROR: failed to restore branch-profile GFS credentials\n' >&2
return 1
fi
}
-stop_port_forward() {
- if [[ -z "${PORT_FORWARD_PID}" ]] || ! kill -0 "${PORT_FORWARD_PID}" >/dev/null 2>&1; then
- PORT_FORWARD_PID=''
+stop_isolated_postgres() {
+ [[ -n "${ISOLATED_CONTAINER}" ]] || return 0
+ if docker rm -f "${ISOLATED_CONTAINER}" >/dev/null 2>&1; then
+ ISOLATED_CONTAINER=''
return 0
fi
- local command_line
- command_line="$(ps -p "${PORT_FORWARD_PID}" -o command= 2>/dev/null || true)"
- if [[ "${command_line}" == *'port-forward'* &&
- "${command_line}" == *"svc/${PG_SERVICE}"* &&
- "${command_line}" == *"${LOCAL_PORT}:5432"* ]]; then
- kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true
- wait "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true
- fi
- PORT_FORWARD_PID=''
+ return 1
}
cleanup() {
local status=$?
- stop_port_forward
- if ! restore_gfs_runtime_credentials; then
- status=1
- fi
- rm -rf "${TMP_DIR}"
+ trap - EXIT
+ trap '' INT TERM
+ if ! stop_profile_postgres_forward; then status=1; fi
+ if ! stop_isolated_postgres; then status=1; fi
+ if [ "${GFS_RESTORE_REQUIRED}" = true ] && ! restore_gfs_runtime_credentials; then status=1; fi
+ if ! rm -rf "${TMP_DIR}"; then status=1; fi
+ if ! t2_lock_release "${status}"; then status=1; fi
exit "${status}"
}
trap cleanup EXIT
@@ -286,39 +301,32 @@ sock.close()
PY
}
-port_forward_ready=false
-for _attempt in 1 2 3 4 5; do
- LOCAL_PORT="$(choose_local_port)"
- kubectl --context="${CONTEXT}" -n "${CONTROL_NS}" port-forward \
- --address=127.0.0.1 "svc/${PG_SERVICE}" "${LOCAL_PORT}:5432" \
- >"${TMP_DIR}/port-forward.log" 2>&1 &
- PORT_FORWARD_PID=$!
- if wait_for_tcp "${LOCAL_PORT}"; then
- port_forward_ready=true
+require_command docker npm
+docker info >/dev/null 2>&1 || die 'Docker is required for the isolated GFS real-Postgres lane'
+verify_branch_gate
+verify_profile_postgres
+
+ISOLATED_PORT="$(choose_local_port)"
+ISOLATED_CONTAINER="evenfire-gfs-t1-pg-$$"
+docker run -d --rm --name "${ISOLATED_CONTAINER}" \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_DB=postgres \
+ -e POSTGRES_HOST_AUTH_METHOD=trust \
+ -p "127.0.0.1:${ISOLATED_PORT}:5432" \
+ "${ISOLATED_PG_IMAGE}" >/dev/null || die 'isolated GFS PostgreSQL container failed to start'
+wait_for_tcp "${ISOLATED_PORT}" || die 'isolated GFS PostgreSQL did not become reachable'
+deadline=$((SECONDS + TIMEOUT))
+while (( SECONDS < deadline )); do
+ if docker exec "${ISOLATED_CONTAINER}" pg_isready -U postgres >/dev/null 2>&1; then
break
fi
- stop_port_forward
+ sleep 1
done
-[[ "${port_forward_ready}" == true ]] || {
- cat "${TMP_DIR}/port-forward.log" >&2 || true
- die 'Minikube PostgreSQL port-forward did not become reachable'
-}
-
-ADMIN_DSN="$(printf '%s\0%s\0%s' "${PG_USER}" "${PG_PASSWORD}" "${LOCAL_PORT}" | python3 -c '
-from urllib.parse import quote
-import sys
-
-user, password, port = sys.stdin.buffer.read().split(b"\0")[:3]
-user = user.decode()
-password = password.decode()
-port = port.decode()
-print("postgresql://{}:{}@127.0.0.1:{}/postgres".format(
- quote(user, safe=""), quote(password, safe=""), port
-), end="")
-')"
-unset PG_USER PG_PASSWORD PG_DATABASE
-[[ -n "${ADMIN_DSN}" ]] || die 'constructed Minikube PostgreSQL admin DSN is empty'
+docker exec "${ISOLATED_CONTAINER}" pg_isready -U postgres >/dev/null 2>&1 ||
+ die 'isolated GFS PostgreSQL did not become ready'
+ADMIN_DSN="$(printf 'postgresql://postgres@%s:%s/postgres' "${ISOLATED_HOST}" "${ISOLATED_PORT}")"
+GFS_RESTORE_REQUIRED=true
run_suite control-api
run_suite gfs-controller
printf '[gfs-real-pg-minikube] T1 PASS: real PostgreSQL executed against validated Minikube profile %s\n' "${CONTEXT}"
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.sh
index fb37ef25e..4d980dd6f 100755
--- a/scripts/e2e/minikube-real-postgres.sh
+++ b/scripts/e2e/minikube-real-postgres.sh
@@ -11,8 +11,10 @@ source "$PROJECT_DIR/scripts/minikube/t2-common.sh"
PG_NAMESPACE="$T2_CONTROL_NAMESPACE"
PG_SERVICE=control-postgres
PG_SECRET=control-postgres
-# Suites that DROP/ALTER cluster-global roles cannot share live control-postgres
-# (#412). They run against a throwaway Postgres 16 that matches CI.
+# Only suites that DROP/ALTER cluster-global roles cannot share live
+# control-postgres (#412). They run against a throwaway PostgreSQL 16 that
+# matches CI; the remaining control-api suites must exercise the validated
+# branch profile database.
T1_ISOLATED_PG_IMAGE="${T1_ISOLATED_PG_IMAGE:-postgres:16-alpine}"
T1_TIMEOUT="$T2_TIMEOUT_SECONDS"
T1_TMP_ROOT="$T2_TMP_ROOT"
@@ -52,10 +54,37 @@ die_t1() {
}
stop_isolated_postgres() {
- if [ -n "$ISOLATED_CONTAINER" ]; then
- docker rm -f "$ISOLATED_CONTAINER" >/dev/null 2>&1 || true
+ [ -n "$ISOLATED_CONTAINER" ] || return 0
+ if docker rm -f "$ISOLATED_CONTAINER" >/dev/null 2>&1; then
ISOLATED_CONTAINER=""
+ return 0
fi
+ return 1
+}
+
+stop_control_postgres_forward() {
+ [ -n "$PORT_FORWARD_PID" ] || return 0
+ if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
+ return 0
+ fi
+ local command_line
+ command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"
+ if [[ "$command_line" != *port-forward* || "$command_line" != *svc/control-postgres* ]]; then
+ printf '[minikube-t1] ERROR: T1 port-forward PID is not the expected control-postgres child; refusing to report cleanup success\n' >&2
+ return 1
+ fi
+ if ! kill "$PORT_FORWARD_PID" >/dev/null 2>&1 && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
+ return 1
+ fi
+ if kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
+ if wait "$PORT_FORWARD_PID" >/dev/null 2>&1; then
+ :
+ fi
+ fi
+ if kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
+ return 1
+ fi
+ return 0
}
restore_gfs_runtime_credentials() {
@@ -82,7 +111,8 @@ restore_gfs_runtime_credentials() {
T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
CONTEXT="$T2_CONTEXT" \
bash "$PROJECT_DIR/scripts/minikube/settle-gfs-reader-rollout.sh"; then
- printf '[minikube-t1] WARN: could not settle a leftover gfsc-reader rollout claim before restore\n' >&2
+ printf '[minikube-t1] ERROR: could not settle a leftover gfsc-reader rollout claim before restore; refusing to reconcile credentials\n' >&2
+ return 1
fi
if ! PATH="$PROJECT_DIR/scripts/minikube/gfs-rollout-shim:$PATH" \
T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
@@ -118,15 +148,12 @@ cleanup_t1() {
# interrupt the lock release and leave a half-restored profile behind.
trap - EXIT
trap '' INT TERM
- if [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
- local command_line
- command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"
- if [[ "$command_line" == *port-forward* && "$command_line" == *svc/control-postgres* ]]; then
- kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
- wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
- fi
+ if ! stop_control_postgres_forward; then
+ status=1
+ fi
+ if ! stop_isolated_postgres; then
+ status=1
fi
- stop_isolated_postgres
if ! restore_gfs_runtime_credentials; then
status=1
fi
@@ -222,7 +249,11 @@ list_real_pg_files() {
is_isolated_control_api_file() {
local path="$1"
case "$path" in
- "$PROJECT_DIR"/control-api/test/*realPostgres*.test.ts)
+ "$PROJECT_DIR"/control-api/*realPostgres*.test.ts)
+ # Every control-api real-Postgres suite calls initDb. That migration
+ # path can reconcile cluster-global GFS roles, so keeping any of these
+ # suites on the branch profile database can poison the runtime even when
+ # the fixture database itself is temporary.
return 0
;;
*)
@@ -232,19 +263,15 @@ is_isolated_control_api_file() {
}
require_isolated_control_api_files() {
- local path count=0
- while IFS= read -r path; do
- [ -n "$path" ] || continue
- count=$((count + 1))
- is_isolated_control_api_file "$path" || {
- T1_NEXT_COMMAND='route every control-api Real PostgreSQL suite through the disposable PostgreSQL instance'
- die_t1 ZERO_TESTS_EXECUTED "control-api Real PostgreSQL suite is not isolated: $path"
+ local path
+ for path in \
+ "$PROJECT_DIR/control-api/test/db.realPostgresMigration.integration.test.ts" \
+ "$PROJECT_DIR/control-api/test/gfsReaderRole.realPostgres.integration.test.ts"; do
+ [ -f "$path" ] || {
+ T1_NEXT_COMMAND='restore the role-reset Real PostgreSQL suite files, then re-run T1'
+ die_t1 ZERO_TESTS_EXECUTED "required isolated control-api suite is missing: $path"
}
- done < <(list_real_pg_files control-api)
- if [ "$count" -le 0 ]; then
- T1_NEXT_COMMAND='restore the control-api Real PostgreSQL suite files, then re-run T1'
- die_t1 ZERO_TESTS_EXECUTED 'no control-api Real PostgreSQL suites were found'
- fi
+ done
}
start_isolated_postgres() {
@@ -341,13 +368,10 @@ run_suite() {
# comparable to the number of physical *realPostgres*.test.ts files above.
# Validate the reporter's own total/passed suite counts instead; the file
# discovery count remains a separate zero-selection guard.
- if ! stats="$(python3 - "$json_file" "$expected_files" <<'PY'
+ if ! stats="$(python3 - "$json_file" <<'PY'
import json
import sys
result = json.loads(open(sys.argv[1]).read())
-expected = int(sys.argv[2])
-files = result.get("testResults") or []
-passed_files = sum(1 for item in files if item.get("status") == "passed")
total_suites = int(result.get("numTotalTestSuites") or 0)
passed_suites = int(result.get("numPassedTestSuites") or 0)
failed_suites = int(result.get("numFailedTestSuites") or 0)
@@ -357,9 +381,8 @@ pending_files = int(result.get("numPendingTestSuites") or 0)
pending_tests = int(result.get("numPendingTests") or 0)
total_tests = int(result.get("numTotalTests") or 0)
success = bool(result.get("success"))
-print(total_suites, passed_suites, failed_suites, passed_files, passed_tests,
- failed_tests, pending_files, pending_tests, total_tests, int(success),
- expected, len(files))
+print(total_suites, passed_suites, failed_suites, passed_tests, failed_tests,
+ pending_files, pending_tests, total_tests, int(success))
PY
)"; then
cat "$log_file" >&2 || true
@@ -367,15 +390,13 @@ PY
T1_NEXT_COMMAND='inspect the Vitest reporter output, then re-run T1'
die_t1 ZERO_TESTS_EXECUTED "Real PostgreSQL reporter was unreadable for $package ($lane)"
fi
- read -r total_suites passed_suites failed_suites passed_files passed_tests \
- failed_tests pending_files pending_tests total_tests success expected \
- reported_files <<< "$stats"
+ read -r total_suites passed_suites failed_suites passed_tests failed_tests \
+ pending_files pending_tests total_tests success <<< "$stats"
T1_TOTAL_TESTS=$((T1_TOTAL_TESTS + total_tests))
T1_PASSED_TESTS=$((T1_PASSED_TESTS + passed_tests))
T1_PENDING_TESTS=$((T1_PENDING_TESTS + pending_tests))
if [ "$success" -ne 1 ] || [ "$total_suites" -le 0 ] || \
[ "$passed_suites" -ne "$total_suites" ] || [ "$failed_suites" -ne 0 ] || \
- [ "$passed_files" -ne "$expected" ] || [ "$reported_files" -ne "$expected" ] || \
[ "$failed_tests" -ne 0 ]; then
T1_NEXT_COMMAND='repair the failed or incomplete Real PostgreSQL lane, then re-run T1'
die_t1 REAL_PG_SUITE_FAILED "Real PostgreSQL reporter did not pass every suite in $package ($lane)"
@@ -393,7 +414,7 @@ PY
die_t1 REAL_PG_SUITE_FAILED "Vitest process exited $suite_status for $package ($lane)"
fi
printf '[minikube-t1] PASS %s %s files=%s tests=%s skipped=0\n' \
- "$package" "$lane" "$passed_files" "$passed_tests"
+ "$package" "$lane" "$expected_files" "$passed_tests"
}
main() {
@@ -408,6 +429,7 @@ main() {
die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'local Real PostgreSQL requires an explicit Minikube context'
fi
t2_profile_status
+ t2_profile_context_identity_check
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T1_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'the selected profile is not bootstrapped'
@@ -472,7 +494,6 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.dec
require_isolated_control_api_files
start_isolated_postgres
T1_GFS_RESTORE_REQUIRED=true
- run_suite control-api shared "$ADMIN_DSN"
run_suite control-api isolated "$ISOLATED_DSN"
run_suite gfs-controller shared "$ADMIN_DSN"
unset PG_USER PG_PASSWORD PG_DATABASE T1_REDACT_PASSWORD
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index a7f661fb1..edfb9654f 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -339,6 +339,17 @@ export ADMIN_PASSWORD
PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
KC="kubectl --context=${PROFILE}"
+# Full setup is a mutating profile transition as well. It acquires the same
+# branch-owned lease used by T2 when run standalone, or validates the inherited
+# lease when called by the T2 orchestrator. This is initialized before the
+# config-only seam but acquired only after the profile is started and reachable.
+T2_PROJECT_DIR="${T2_PROJECT_DIR:-${PROJECT_DIR}}"
+T2_PROFILE="${T2_PROFILE:-${PROFILE}}"
+T2_CONTEXT="${T2_CONTEXT:-${PROFILE}}"
+T2_GATE_ID="${T2_GATE_ID:-minikube-setup}"
+T2_SKIP_LOCK="${T2_SKIP_LOCK:-false}"
+# shellcheck source=scripts/minikube/t2-common.sh
+source "${SCRIPT_DIR}/t2-common.sh"
TOTAL_STEPS=12
# MINIKUBE_IMAGE_TAG overrides the committed pin AT RENDER TIME ONLY.
#
@@ -652,6 +663,48 @@ fi
ok "Cluster '${PROFILE}' is reachable"
maybe_exit_after_cluster_step
+SETUP_LOCK_CLEANUP_DONE=false
+cleanup_setup_lock() {
+ local status="${1:-$?}" cleanup_status=0
+ if [ "$SETUP_LOCK_CLEANUP_DONE" = true ]; then
+ return "$status"
+ fi
+ SETUP_LOCK_CLEANUP_DONE=true
+ trap - EXIT
+ trap '' INT TERM
+ t2_lock_release "$status" || cleanup_status=$?
+ if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
+ status="$cleanup_status"
+ fi
+ return "$status"
+}
+handle_setup_signal() {
+ local signal="$1" status
+ case "$signal" in
+ INT) status=130 ;;
+ TERM) status=143 ;;
+ *) status=1 ;;
+ esac
+ cleanup_setup_lock "$status" || status=$?
+ exit "$status"
+}
+handle_setup_exit() {
+ local status=$?
+ cleanup_setup_lock "$status" || status=$?
+ exit "$status"
+}
+trap handle_setup_exit EXIT
+trap 'handle_setup_signal INT' INT
+trap 'handle_setup_signal TERM' TERM
+t2_require_commands
+t2_repo_metadata
+t2_profile_scope
+t2_profile_status
+t2_context_check
+t2_profile_context_identity_check
+t2_mutation_lock
+export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_SKIP_LOCK T2_LOCK_TOKEN
+
# ======================================================================
# Step 3: Namespaces + CRDs
# ======================================================================
@@ -990,19 +1043,23 @@ else
# wiped by the overlay re-apply). Re-sync it from the live platform
# Secret before reconcile so a reader pod can actually start; this is a
# no-op when the key already matches.
- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
+ GFS_AUTH_SYNC_ALLOW_STAGED=true \
+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
# If gfsc-reader is already Ready, settle the leftover claim first so
# reconcile does not rollout restart and race HCC's gfsReconciler.
# The gfs-rollout-shim PATH prefix makes any reader rollout wait inside
# reconcile judge readiness instead of the template generation HCC keeps
# rewriting (it strips the restartedAt annotation, so a generation wait
# loops until timeout).
- CONTEXT="${PROFILE}" \
+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
+ # A staged sync may have deferred active-pod proof until the reconciler
+ # created the reader DSN. Complete that proof before the overlay is applied.
+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
else
log "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"
fi
@@ -1067,13 +1124,17 @@ else
# The overlay apply above re-declared the base gfs-config with an empty
# jwt-public-key; re-sync it before any reader pod may need to start,
# otherwise the reconcile readiness wait can only time out.
- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
- CONTEXT="${PROFILE}" \
+ GFS_AUTH_SYNC_ALLOW_STAGED=true \
+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
+ # The preceding staged sync is allowed to wait for this DSN-producing
+ # reconcile; the strict pass is the durable consumer attestation.
+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
ok "GFS credentials reconciled and writer bootstrap verified"
fi
diff --git a/scripts/minikube/gfs-rollout-shim/kubectl b/scripts/minikube/gfs-rollout-shim/kubectl
index 8d34ed686..43c55f0b8 100755
--- a/scripts/minikube/gfs-rollout-shim/kubectl
+++ b/scripts/minikube/gfs-rollout-shim/kubectl
@@ -69,6 +69,7 @@ done
# explicit short timeout would hide a dead reader and invalidate the caller's
# fail-fast contract.
if ! timeout_seconds="$(python3 - "$timeout_raw" <<'PY'
+import math
import re
import sys
@@ -77,15 +78,17 @@ timeout_raw = sys.argv[1]
def parse_duration(raw, default, label):
if raw == "":
return default
- match = re.fullmatch(r"[1-9][0-9]{0,4}([sm]?)", raw)
- if not match:
- raise SystemExit(f"invalid {label}: expected a positive integer with optional s/m suffix")
- value = int(raw[:-1] if raw[-1:] in {"s", "m"} else raw)
- if raw.endswith("m"):
- value *= 60
- if value < 1 or value > 86400:
+ if re.fullmatch(r"[1-9][0-9]*", raw):
+ total = float(raw)
+ else:
+ tokens = re.findall(r"([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h)", raw)
+ if not tokens or "".join(number + unit for number, unit in tokens) != raw:
+ raise SystemExit(f"invalid {label}: expected a positive Go-style duration")
+ factors = {"ns": 1e-9, "us": 1e-6, "µs": 1e-6, "ms": 1e-3, "s": 1.0, "m": 60.0, "h": 3600.0}
+ total = sum(float(number) * factors[unit] for number, unit in tokens)
+ if total <= 0 or total > 86400:
raise SystemExit(f"invalid {label}: value must be between 1 and 86400 seconds")
- return value
+ return max(1, math.ceil(total))
print(parse_duration(timeout_raw, 600, "rollout timeout"))
PY
diff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.sh
index 625acefba..907f59c23 100755
--- a/scripts/minikube/pf-all-stack.sh
+++ b/scripts/minikube/pf-all-stack.sh
@@ -11,6 +11,7 @@ PIDFILES=()
PIDFILE_SERVICES=()
SAFE_PROFILE="${PROFILE//[^A-Za-z0-9_.-]/_}"
HAS_PROFILE_OWNED_PORTS=false
+STARTUP_COMPLETE=false
if [[ ! "$PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then
echo "ERROR: invalid Minikube profile identifier: $PROFILE" >&2
exit 1
@@ -69,6 +70,16 @@ elif [[ "${PROFILE}" =~ ^clerum-(codex|detached)- ]]; then
exit 1
fi
+is_branch_profile() {
+ [[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]
+}
+
+if [[ ! -f "${BRANCH_PROFILE_PORTS_ENV}" ]] && is_branch_profile; then
+ echo "ERROR: missing branch-scoped port cache for minikube profile: ${PROFILE}" >&2
+ echo "Expected ${BRANCH_PROFILE_PORTS_ENV}" >&2
+ exit 1
+fi
+
if [[ -z "${PROFILE_UI_PORT:-}" && -n "${PORT_BASE:-}" ]]; then
PROFILE_UI_PORT=$((PORT_BASE + 1))
fi
@@ -99,7 +110,7 @@ require_random_port_for_branch_profile() {
local name="$1"
local value="$2"
local default_port="$3"
- if [[ "${HAS_PROFILE_OWNED_PORTS}" != "true" && ! "${PROFILE}" =~ ^clerum-(codex|detached)- ]]; then
+ if [[ "${HAS_PROFILE_OWNED_PORTS}" != "true" ]] && ! is_branch_profile; then
return 0
fi
if [[ "${value}" == "${default_port}" ]]; then
@@ -119,7 +130,7 @@ require_random_port_for_branch_profile WORKFLOW_APPROVAL_READER_PORT "${WORKFLOW
require_random_port_for_branch_profile MCP_HOST_PORT "${MCP_HOST_PORT}" 8080
cleanup() {
- if [[ "${HOLD}" != "true" ]]; then
+ if [[ "${HOLD}" != "true" && "${STARTUP_COMPLETE}" == "true" ]]; then
return 0
fi
local index
@@ -131,7 +142,7 @@ cleanup() {
kill_owned_pidfile() {
local pidfile="$1"
local service="$2"
- local pid command_line
+ local pid command_line expected_start actual_start
[[ -f "$pidfile" ]] || return 0
pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"
if [[ ! "$pid" =~ ^[1-9][0-9]*$ ]]; then
@@ -149,10 +160,29 @@ kill_owned_pidfile() {
echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the $PROFILE $service port-forward" >&2
return 1
fi
+ expected_start="$(sed -n 's/^PROCESS_START=//p' "$pidfile" 2>/dev/null | head -1 || true)"
+ if [[ -z "$expected_start" ]]; then
+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature is missing" >&2
+ return 1
+ fi
+ if [[ "$expected_start" != unavailable ]]; then
+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ if [[ -z "$actual_start" || "$actual_start" != "$expected_start" ]]; then
+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature changed" >&2
+ return 1
+ fi
+ fi
kill "$pid" 2>/dev/null || true
rm -f -- "$pidfile"
}
+write_pidfile() {
+ local pidfile="$1" pid="$2" process_start
+ process_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [ -n "$process_start" ] || process_start=unavailable
+ printf '%s\nPROCESS_START=%s\n' "$pid" "$process_start" >"$pidfile"
+}
+
start_pf() {
local name="$1"
local namespace="$2"
@@ -169,8 +199,8 @@ start_pf() {
kill_owned_pidfile "$profile_pidfile" "$service"
nohup "${KC[@]}" -n "${namespace}" port-forward --address=127.0.0.1 "svc/${service}" "${ports}" >"${log}" 2>&1 </dev/null &
- echo $! >"${pidfile}"
- echo $! >"${profile_pidfile}"
+ write_pidfile "${pidfile}" "$!"
+ write_pidfile "${profile_pidfile}" "$!"
PIDS+=("$!")
PIDFILES+=("$pidfile" "$profile_pidfile")
PIDFILE_SERVICES+=("$service" "$service")
@@ -238,6 +268,11 @@ start_optional_deployment_pf() {
start_optional_pf "${name}" "${namespace}" "${service}" "${ports}" "${health_url}"
}
+# Install the cleanup boundary before the first background process is started.
+# A later start_pf failure must not strand forwards that were already launched.
+trap cleanup EXIT
+trap 'cleanup; exit 0' INT TERM
+
echo "=== Starting gate port-forwards (${PROFILE}) ==="
start_pf control-ui control-plane control-ui "${CONTROL_UI_PORT}:3000" "http://127.0.0.1:${CONTROL_UI_PORT}"
start_optional_pf profile-ui profiles profile-ui "${PROFILE_UI_PORT}:3001" "http://127.0.0.1:${PROFILE_UI_PORT}"
@@ -253,11 +288,10 @@ elif "${KC[@]}" get svc mcp-host -n mcp-host >/dev/null 2>&1; then
start_pf mcp-host mcp-host mcp-host "${MCP_HOST_PORT}:8080" "http://127.0.0.1:${MCP_HOST_PORT}/v1/runtime/health"
fi
+STARTUP_COMPLETE=true
echo "=== Port-forwards refreshed ==="
if [[ "${HOLD}" == "true" ]]; then
- trap cleanup EXIT
- trap 'cleanup; exit 0' INT TERM
echo "=== Holding port-forwards open; press Ctrl-C to stop ==="
while true; do
sleep 3600 &
diff --git a/scripts/minikube/pf-control-stack.sh b/scripts/minikube/pf-control-stack.sh
index f6e42d7f2..e5b0110f1 100755
--- a/scripts/minikube/pf-control-stack.sh
+++ b/scripts/minikube/pf-control-stack.sh
@@ -4,38 +4,77 @@
# - control-api on :8090
# Writes PIDs to /tmp/pf-*.pid for later cleanup.
-set -eo pipefail
+set -euo pipefail
+
+PROFILE="${MINIKUBE_PROFILE:-${KUBECONTEXT:-}}"
+if [ -z "$PROFILE" ]; then
+ echo 'ERROR: MINIKUBE_PROFILE or KUBECONTEXT is required; refusing a context-less port-forward' >&2
+ exit 1
+fi
+KC=(kubectl --context="$PROFILE")
+SAFE_PROFILE="$(printf '%s' "$PROFILE" | tr -c 'A-Za-z0-9_.-' '_')"
+CONTROL_UI_PORT="${CONTROL_UI_PORT:-3000}"
+CONTROL_API_PORT="${CONTROL_API_PORT:-8090}"
+LOCALHOST="localhost"
+if [[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]] && {
+ [ "$CONTROL_UI_PORT" = 3000 ] || [ "$CONTROL_API_PORT" = 8090 ];
+}; then
+ echo "ERROR: branch-owned profile $PROFILE requires profile-owned CONTROL_UI_PORT and CONTROL_API_PORT" >&2
+ exit 1
+fi
start_pf() {
local name=$1 ns=$2 svc=$3 ports=$4
- local log=/tmp/pf-$name.log
- local pidfile=/tmp/pf-$name.pid
+ local log=/tmp/pf-${SAFE_PROFILE}-${name}.log
+ local pidfile=/tmp/pf-${SAFE_PROFILE}-${name}.pid
+ local pid expected_start actual_start process_start
- # Kill previous
+ # Kill previous only when both the command identity and process-start
+ # signature match; a recycled PID must never be treated as ours.
if [[ -f "$pidfile" ]]; then
- kill "$(cat "$pidfile")" 2>/dev/null || true
+ pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"
+ if [[ "$pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$pid" 2>/dev/null; then
+ command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)"
+ expected_start="$(sed -n 's/^PROCESS_START=//p' "$pidfile" 2>/dev/null | head -1 || true)"
+ if [[ "$command_line" != *port-forward* || "$command_line" != *"svc/$svc"* ||
+ "$command_line" != *"$PROFILE"* || -z "$expected_start" ]]; then
+ echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the signed $PROFILE $svc port-forward" >&2
+ exit 1
+ fi
+ if [[ "$expected_start" != unavailable ]]; then
+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [[ -n "$actual_start" && "$actual_start" == "$expected_start" ]] || {
+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature changed" >&2
+ exit 1
+ }
+ fi
+ kill "$pid" 2>/dev/null || true
+ fi
rm -f "$pidfile"
fi
- kubectl -n "$ns" port-forward "svc/$svc" "$ports" > "$log" 2>&1 &
- echo $! > "$pidfile"
+ "${KC[@]}" -n "$ns" port-forward --address=127.0.0.1 "svc/$svc" "$ports" > "$log" 2>&1 &
+ pid=$!
+ process_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [ -n "$process_start" ] || process_start=unavailable
+ printf '%s\nPROCESS_START=%s\n' "$pid" "$process_start" > "$pidfile"
echo " $name: pid=$(cat "$pidfile") ns=$ns svc=$svc ports=$ports"
}
echo "=== Starting port-forwards ==="
-start_pf control-ui control-plane control-ui 3000:3000
-start_pf control-api control-plane control-api 8090:8090
+start_pf control-ui control-plane control-ui "${CONTROL_UI_PORT}:3000"
+start_pf control-api control-plane control-api "${CONTROL_API_PORT}:8090"
echo "=== Waiting 5s for tunnels to warm up ==="
sleep 5
echo "=== Health checks ==="
-if curl -sf -m 3 http://localhost:3000 >/dev/null 2>&1; then
+if curl -sf -m 3 "http://${LOCALHOST}:${CONTROL_UI_PORT}" >/dev/null 2>&1; then
echo " control-ui: UP"
else
echo " control-ui: DOWN (may take longer, retry in a few seconds)"
fi
-if curl -sf -m 3 http://localhost:8090/health >/dev/null 2>&1; then
+if curl -sf -m 3 "http://${LOCALHOST}:${CONTROL_API_PORT}/health" >/dev/null 2>&1; then
echo " control-api: UP"
else
echo " control-api: DOWN (may take longer, retry in a few seconds)"
diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.sh
index d189ec3f4..747917c83 100755
--- a/scripts/minikube/pre-gate-sync.sh
+++ b/scripts/minikube/pre-gate-sync.sh
@@ -62,6 +62,8 @@ if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
t2_repo_metadata
t2_profile_scope
t2_context_check
+ t2_profile_status
+ t2_profile_context_identity_check
fi
log() {
@@ -237,6 +239,8 @@ commit_cluster_sync_state() {
if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
t2_mutation_lock
fi
+export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV \
+ T2_SKIP_LOCK T2_LOCK_TOKEN
trap finalize_pre_gate_sync EXIT
fingerprint_dir() {
@@ -384,7 +388,9 @@ sync_mcp_host_auth_key() {
}
sync_gfs_auth_key() {
- bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --require-gfs
+ local allow_staged="${1:-false}"
+ GFS_AUTH_SYNC_ALLOW_STAGED="${allow_staged}" \
+ bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --require-gfs
}
settle_gfs_reader_rollout() {
@@ -454,14 +460,24 @@ converge_gfs_reader_after_restore() {
# restored Secret without waiting out CrashLoopBackOff, then judge
# readiness directly.
local pod_rows pod_name pod_ready pod_deleting
- pod_rows="$(${KC} get pods -n gfs -l 'app=gfs-controller,clerum.io/gfsc-role=reader' -o \
+ if ! pod_rows="$(${KC} get pods -n gfs -l 'app=gfs-controller,clerum.io/gfsc-role=reader' -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \
- 2>/dev/null || true)"
+ 2>&1)"; then
+ if [[ "$pod_rows" == *NotFound* || "$pod_rows" == *"not found"* ]]; then
+ pod_rows=""
+ else
+ log "ERROR: unable to inspect gfs reader pods before convergence: $pod_rows"
+ return 1
+ fi
+ fi
while IFS='|' read -r pod_name pod_ready pod_deleting; do
[ -n "$pod_name" ] || continue
[ -z "$pod_deleting" ] || continue
[ "$pod_ready" != True ] || continue
- ${KC} delete pod "$pod_name" -n gfs --wait=false >/dev/null 2>&1 || true
+ if ! ${KC} delete pod "$pod_name" -n gfs --ignore-not-found --wait=false >/dev/null 2>&1; then
+ log "ERROR: unable to delete unready gfs reader pod $pod_name"
+ return 1
+ fi
done <<<"$pod_rows"
CONTEXT="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/wait-gfs-reader-ready.sh"
@@ -503,7 +519,7 @@ provision_gfs_serving() {
# (the overlay re-applies the base ConfigMap with an empty value); a
# reader pod cannot start without it and any readiness wait would only
# time out. Re-sync it first; this is a no-op when the key matches.
- sync_gfs_auth_key
+ sync_gfs_auth_key true
# Settle a Ready reader first so reconcile does not restart it and
# race HCC's gfsReconciler during kubectl rollout status. If reconcile
# still needs a reader rollout, the gfs-rollout-shim PATH prefix makes
@@ -517,6 +533,14 @@ provision_gfs_serving() {
log "ERROR: gfs DB provisioning FAILED — gfsc cannot authorize any operation. Aborting ${GATE_NAME} pre-gate sync."
exit 1
fi
+ # The first staged sync may have deferred active-pod proof because this
+ # reconcile call is the producer of gfs-controller-db. Re-run strict proof
+ # after it so the durable marker is never published before consumers have
+ # the key and a usable DSN.
+ if ! sync_gfs_auth_key; then
+ log "ERROR: gfs auth-key convergence failed after credential reconciliation"
+ return 1
+ fi
if ! converge_gfs_reader_after_restore; then
log "ERROR: gfs reader did not converge after credential restoration"
return 1
@@ -628,7 +652,9 @@ if [[ "${cluster_changed}" == "true" ]]; then
"${INCREMENTAL_FULL_DEPLOYMENT}" == "true" ]]; then
(
cd "${PROJECT_DIR}"
- make minikube-deploy-crds
+ gfs_crd_mutation=false
+ [[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_crd_mutation=true
+ MINIKUBE_GFS_MUTATION="${gfs_crd_mutation}" make minikube-deploy-crds
)
if [[ "${GATE_NAME}" == "minikube-t2" ]]; then
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
diff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.sh
index 0d27d5976..4b0a839f2 100755
--- a/scripts/minikube/settle-gfs-reader-rollout.sh
+++ b/scripts/minikube/settle-gfs-reader-rollout.sh
@@ -32,15 +32,38 @@ source "$ROOT/scripts/minikube/t2-common.sh"
if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
T2_CONTEXT="$T2_CONTEXT"
CONTEXT="$T2_CONTEXT"
+SETTLE_T2_MANAGED=false
+[ -n "$T2_LOCK_TOKEN" ] && SETTLE_T2_MANAGED=true
GFS_NS="${GFS_NS:-gfs}"
DEPLOY="${GFS_READER_DEPLOYMENT:-gfsc-reader}"
SECRET="${GFS_READER_DB_SECRET:-gfs-controller-reader-db}"
SELECTOR="${GFS_READER_SELECTOR:-app=gfs-controller,clerum.io/gfsc-role=reader}"
+SCALE_DOWN_TIMEOUT_SECONDS="${GFS_READER_SCALE_DOWN_TIMEOUT_SECONDS:-60}"
+SCALE_DOWN_POLL_SECONDS="${GFS_READER_SCALE_DOWN_POLL_SECONDS:-1}"
log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }
die() { log "ERROR: $*"; exit 1; }
kc() { kubectl --context="$T2_CONTEXT" "$@"; }
+wait_for_scaled_rs() {
+ local rs_name="$1" deadline desired pod_rows live
+ deadline=$((SECONDS + SCALE_DOWN_TIMEOUT_SECONDS))
+ while [ "$SECONDS" -lt "$deadline" ]; do
+ desired="$(kc -n "$GFS_NS" get rs "$rs_name" -o jsonpath='{.spec.replicas}' 2>&1)" ||
+ die "unable to verify ReplicaSet ${rs_name} scale-down: ${desired}"
+ [[ "$desired" =~ ^[0-9]+$ ]] || die "ReplicaSet ${rs_name} returned a non-numeric replica count"
+ pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \
+ 'jsonpath={range .items[*]}{.metadata.ownerReferences[0].name}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' 2>&1)" ||
+ die "unable to inspect pods while waiting for ReplicaSet ${rs_name} scale-down: ${pod_rows}"
+ live="$(awk -F'|' -v rs="$rs_name" '$1 == rs && $2 == "" { n++ } END { print n+0 }' <<<"$pod_rows")"
+ if [ "$desired" = 0 ] && [ "${live:-0}" -eq 0 ]; then
+ return 0
+ fi
+ sleep "$SCALE_DOWN_POLL_SECONDS"
+ done
+ die "ReplicaSet ${rs_name} did not settle at zero replicas before credential proof"
+}
+
SETTLE_CLEANUP_DONE=false
cleanup_settle() {
@@ -51,7 +74,9 @@ cleanup_settle() {
SETTLE_CLEANUP_DONE=true
trap - EXIT
trap '' INT TERM
- t2_lock_release "$status" || status=$?
+ if [ "$SETTLE_T2_MANAGED" = true ]; then
+ t2_lock_release "$status" || status=$?
+ fi
return "$status"
}
handle_settle_signal() {
@@ -72,10 +97,22 @@ handle_settle_exit() {
trap handle_settle_exit EXIT
trap 'handle_settle_signal INT' INT
trap 'handle_settle_signal TERM' TERM
-t2_repo_metadata
-t2_profile_scope
-t2_context_check
-t2_mutation_lock
+if [ "$SETTLE_T2_MANAGED" = true ]; then
+ t2_repo_metadata
+ t2_profile_scope
+ t2_context_check
+ t2_mutation_lock
+else
+ [ "${GFS_READER_ROLLOUT_AUTHORIZED:-false}" = true ] ||
+ die 'non-T2 reader settlement requires explicit GFS_READER_ROLLOUT_AUTHORIZED=true'
+ authorized=false
+ IFS=',' read -r -a allowed_contexts <<<"${ALLOWED_CONTEXTS:-}"
+ for allowed_context in "${allowed_contexts[@]}"; do
+ [ "$allowed_context" = "$CONTEXT" ] && authorized=true
+ done
+ [ "$authorized" = true ] ||
+ die "non-T2 reader settlement context is not in the explicit ALLOWED_CONTEXTS list: $CONTEXT"
+fi
# The proof library is sourced only after the profile fence is established. It
# reads the committed DSN in memory and never prints it.
@@ -115,6 +152,7 @@ fi
# Desired Ready is met from here on: leftovers below serve nothing.
current_rev="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
-o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
+scaled_rs_names=()
rs_rows="$(kc -n "$GFS_NS" get rs -l "$SELECTOR" -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.spec.replicas}{"|"}{.status.readyReplicas}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' \
2>/dev/null)"
@@ -127,8 +165,13 @@ while IFS='|' read -r rs_name rs_owner rs_replicas rs_ready rs_rev; do
[ -n "$current_rev" ] && [ "$rs_rev" = "$current_rev" ] && continue
log "scaling leftover unready ReplicaSet ${rs_name} to 0 (revision ${rs_rev:-unknown}, current ${current_rev:-unknown})"
kc -n "$GFS_NS" scale rs "$rs_name" --replicas=0 >/dev/null
+ scaled_rs_names+=("$rs_name")
done <<<"$rs_rows"
+for rs_name in "${scaled_rs_names[@]}"; do
+ wait_for_scaled_rs "$rs_name"
+done
+
pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.containerStatuses[0].state.waiting.reason}{"\n"}{end}' \
2>/dev/null)"
diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.sh
index 47f0fec87..5a742287a 100755
--- a/scripts/minikube/sync-auth-key.sh
+++ b/scripts/minikube/sync-auth-key.sh
@@ -50,6 +50,39 @@ if [[ "${REQUIRE_GFS}" == "true" && "${SYNC_GFS}" != "true" ]]; then
exit 2
fi
+log() { printf '[sync-auth-key] %s\n' "$*"; }
+die() {
+ log "ERROR: $*" >&2
+ exit 1
+}
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+T2_PROJECT_DIR="${T2_PROJECT_DIR:-${ROOT}}"
+T2_PROFILE="${T2_PROFILE:-${MINIKUBE_PROFILE:-${PROFILE}}}"
+T2_CONTEXT="${T2_CONTEXT:-${CONTROL_API_REAL_PG_CONTEXT:-${PROFILE}}}"
+T2_LOCK_ROOT="${T2_LOCK_ROOT:-${HOME}/.cache/evenfire/minikube-t2-locks}"
+T2_LOCK_TOKEN="${T2_LOCK_TOKEN:-}"
+# Auth-key convergence mutates runtime ConfigMaps and restarts consumers. It is
+# a child of the owning T2/full-setup transition, never an independent writer.
+# Validate the live opaque lease here as well as at the public Make boundary so
+# invoking this helper directly cannot bypass profile ownership.
+source "${ROOT}/scripts/minikube/t2-common.sh"
+[[ -n "${T2_LOCK_TOKEN}" ]] || die "canonical T2 profile lock is required for auth-key convergence"
+T2_LOCK_DIR="${T2_LOCK_ROOT}/${T2_PROFILE}.lock"
+[[ -d "${T2_LOCK_DIR}" && ! -L "${T2_LOCK_DIR}" ]] ||
+ die "canonical T2 profile lock is missing for ${T2_PROFILE}"
+owner_token="$(t2_lock_owner_value TOKEN || true)"
+owner_pid="$(t2_lock_owner_value PID || true)"
+[[ "${owner_token}" == "${T2_LOCK_TOKEN}" ]] ||
+ die "canonical T2 profile lock token does not match"
+[[ "$(t2_lock_owner_value REPOSITORY || true)" == "${T2_PROJECT_DIR}" ]] ||
+ die "canonical T2 profile lock belongs to another worktree"
+[[ "$(t2_lock_owner_value PROFILE || true)" == "${T2_PROFILE}" &&
+ "$(t2_lock_owner_value CONTEXT || true)" == "${T2_CONTEXT}" ]] ||
+ die "canonical T2 profile lock belongs to another profile or context"
+t2_lock_process_matches "${owner_pid}" ||
+ die "canonical T2 profile lock owner is no longer a live process"
+
KCTL=(kubectl --context="${PROFILE}")
SOURCE_SECRET="rpc-proxy-secrets"
SOURCE_NAMESPACE="rpc-proxy"
@@ -70,12 +103,11 @@ MCP_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"
GFS_CONFIGMAP="gfs-config"
GFS_NAMESPACE="gfs"
GFS_TARGET_KEY="jwt-public-key"
-
-log() { printf '[sync-auth-key] %s\n' "$*"; }
-die() {
- log "ERROR: $*" >&2
- exit 1
-}
+# During the T2 recovery transition, credential reconciliation is the producer
+# of the GFS DSN. An explicitly staged sync may update gfs-config and defer
+# active-pod proof until that producer has run; ordinary/full-setup callers
+# remain fail-closed by default.
+GFS_AUTH_SYNC_ALLOW_STAGED="${GFS_AUTH_SYNC_ALLOW_STAGED:-false}"
rollout_restart_with_retry() {
local namespace="$1"
@@ -118,18 +150,42 @@ print(json.dumps({"data": {os.environ["TARGET_KEY"]: source_value}}))
PY
}
-make_applied_hash_patch() {
- APPLIED_HASH="$1" APPLIED_HASH_ANNOTATION="${APPLIED_HASH_ANNOTATION}" python3 - <<'PY'
+make_applied_hash_json_patch() {
+ local namespace="$1" configmap="$2" target_key="$3" target_json
+ target_json="$("${KCTL[@]}" get configmap "${configmap}" -n "${namespace}" -o json)" ||
+ die "cannot snapshot auth target ${namespace}/${configmap} before attestation"
+ TARGET_JSON="$target_json" TARGET_KEY="$target_key" SOURCE_KEY_B64="$source_key_b64" \
+ APPLIED_HASH="$source_hash" APPLIED_HASH_ANNOTATION="$APPLIED_HASH_ANNOTATION" \
+ python3 - <<'PY'
+import base64
import json
import os
-print(json.dumps({
- "metadata": {
- "annotations": {
- os.environ["APPLIED_HASH_ANNOTATION"]: os.environ["APPLIED_HASH"]
- }
- }
-}))
+obj = json.loads(os.environ["TARGET_JSON"])
+metadata = obj.get("metadata") or {}
+data = obj.get("data") or {}
+annotations = metadata.get("annotations") or {}
+rv = str(metadata.get("resourceVersion") or "")
+if not rv:
+ raise SystemExit("auth target has no resourceVersion")
+try:
+ expected = base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=True).decode("utf-8")
+except (ValueError, UnicodeDecodeError) as exc:
+ raise SystemExit(f"invalid source auth key: {exc}")
+key = os.environ["TARGET_KEY"]
+if data.get(key) != expected:
+ raise SystemExit("auth target changed before attestation")
+annotation = os.environ["APPLIED_HASH_ANNOTATION"]
+ops = [
+ {"op": "test", "path": "/metadata/resourceVersion", "value": rv},
+ {"op": "test", "path": "/data/" + key.replace("~", "~0").replace("/", "~1"), "value": expected},
+]
+path = "/metadata/annotations/" + annotation.replace("~", "~0").replace("/", "~1")
+if annotations:
+ ops.append({"op": "replace" if annotation in annotations else "add", "path": path, "value": os.environ["APPLIED_HASH"]})
+else:
+ ops.append({"op": "add", "path": "/metadata/annotations", "value": {annotation: os.environ["APPLIED_HASH"]}})
+print(json.dumps(ops))
PY
}
@@ -140,9 +196,17 @@ read_applied_hash() {
}
mark_key_applied() {
- local namespace="$1" configmap="$2"
- "${KCTL[@]}" patch configmap "${configmap}" -n "${namespace}" --type=merge \
- -p "$(make_applied_hash_patch "${source_hash}")" >/dev/null
+ local namespace="$1" configmap="$2" target_key="${3:-}"
+ if [ -z "${target_key}" ]; then
+ case "${configmap}" in
+ "${CONFIGMAP}") target_key="${TARGET_KEY}" ;;
+ "${GFS_CONFIGMAP}") target_key="${GFS_TARGET_KEY}" ;;
+ *) die "cannot infer auth target key for ${namespace}/${configmap}" ;;
+ esac
+ fi
+ assert_source_unchanged
+ "${KCTL[@]}" patch configmap "${configmap}" -n "${namespace}" --type=json \
+ -p "$(make_applied_hash_json_patch "${namespace}" "${configmap}" "${target_key}")" >/dev/null
}
configmap_key_hash() {
@@ -151,6 +215,20 @@ configmap_key_hash() {
-o "jsonpath={.data.${key}}" | shasum -a 256 | awk '{print $1}'
}
+deployment_desired_replicas() {
+ local namespace="$1" deployment="$2" desired
+ desired="$("${KCTL[@]}" get "deployment/${deployment}" -n "${namespace}" \
+ -o 'jsonpath={.spec.replicas}' 2>&1)" || {
+ log "cannot read desired replicas for ${namespace}/deployment/${deployment}: ${desired}" >&2
+ return 1
+ }
+ [[ "${desired}" =~ ^[0-9]+$ ]] || {
+ log "deployment ${namespace}/deployment/${deployment} returned a non-numeric desired replica count" >&2
+ return 1
+ }
+ printf '%s' "${desired}"
+}
+
write_source_key() {
SOURCE_KEY_B64="${source_key_b64}" python3 - <<'PY'
import base64
@@ -161,6 +239,49 @@ sys.stdout.buffer.write(base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=
PY
}
+assert_source_unchanged() {
+ local current_snapshot current_rv ignored_b64 current_hash
+ current_snapshot="$(read_source_snapshot)" || die "cannot re-read auth source snapshot"
+ IFS=$'\t' read -r current_rv ignored_b64 current_hash <<<"${current_snapshot}"
+ [[ -n "${current_rv}" && -n "${current_hash}" ]] ||
+ die "auth source snapshot is incomplete during convergence"
+ if [[ "${current_rv}" != "${source_resource_version}" || "${current_hash}" != "${source_hash}" ]]; then
+ die "auth source rotated during convergence; refusing to attest a stale key"
+ fi
+}
+
+read_source_snapshot() {
+ local source_json
+ source_json="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o json)" ||
+ return 1
+ SOURCE_JSON="${source_json}" SOURCE_KEY="${SOURCE_KEY}" python3 - <<'PY'
+import base64
+import hashlib
+import json
+import os
+import sys
+
+try:
+ obj = json.loads(os.environ["SOURCE_JSON"])
+ metadata = obj.get("metadata") or {}
+ data = obj.get("data") or {}
+ resource_version = str(metadata.get("resourceVersion") or "")
+ source_key_b64 = data.get(os.environ["SOURCE_KEY"]) or ""
+ source_value = base64.b64decode(source_key_b64, validate=True)
+ source_value.decode("utf-8")
+except (KeyError, TypeError, ValueError, UnicodeDecodeError, json.JSONDecodeError):
+ raise SystemExit(1)
+
+if not resource_version or not source_value:
+ raise SystemExit(1)
+print("%s\t%s\t%s" % (
+ resource_version,
+ source_key_b64,
+ hashlib.sha256(source_value).hexdigest(),
+))
+PY
+}
+
consumer_pods_use_key() {
local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"
local rows pod ready deleting pod_count=0
@@ -216,35 +337,16 @@ if ! source_probe="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAM
fi
die "cannot inspect auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET}: ${source_probe}"
fi
-source_key_b64="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}")"
-if [[ -z "${source_key_b64}" ]]; then
+source_snapshot="$(read_source_snapshot 2>/dev/null || true)"
+if [[ -z "${source_snapshot}" ]]; then
if [[ "${REQUIRE_GFS}" == "true" ]]; then
die "required GFS auth source key ${SOURCE_KEY} is empty"
fi
die "auth source key ${SOURCE_KEY} is empty; refusing to mutate consumers"
fi
-if ! source_hash="$(SOURCE_KEY_B64="${source_key_b64}" python3 - <<'PY'
-import base64
-import hashlib
-import os
-import sys
-
-try:
- source_value = base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=True)
- source_value.decode("utf-8")
-except (ValueError, UnicodeDecodeError) as exc:
- print(f"invalid source auth key encoding: {exc}", file=sys.stderr)
- raise SystemExit(1)
-
-if not source_value:
- print("source auth key decodes empty", file=sys.stderr)
- raise SystemExit(1)
-
-print(hashlib.sha256(source_value).hexdigest())
-PY
-)"; then
- die "source auth key ${SOURCE_KEY} is not valid non-empty UTF-8 base64"
-fi
+IFS=$'\t' read -r source_resource_version source_key_b64 source_hash <<<"${source_snapshot}"
+[[ -n "${source_resource_version}" && -n "${source_key_b64}" && -n "${source_hash}" ]] ||
+ die "source auth key ${SOURCE_KEY} is not a valid non-empty UTF-8 base64 value with a resourceVersion"
if [[ "${REQUIRE_GFS}" == "true" ]] && \
! gfs_target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; then
die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable: ${gfs_target_probe}"
@@ -286,7 +388,7 @@ sync_mcp_host() {
log "${CONFIGMAP}.${TARGET_KEY} matches source but consumer attestation is pending; resuming convergence"
fi
fi
- local deployment_resources deployment resource_name
+ local deployment_resources deployment resource_name desired_replicas
local deployments=()
if ! deployment_resources="$("${KCTL[@]}" get deployment -l "${MCP_DEPLOY_SELECTOR}" \
-n "${CONFIG_NAMESPACE}" -o name 2>&1)"; then
@@ -306,6 +408,15 @@ sync_mcp_host() {
log "No mcp-host deployments exist yet; auth key is synced for future pods"
else
for deployment in "${deployments[@]}"; do
+ desired_replicas="$(deployment_desired_replicas "${CONFIG_NAMESPACE}" "${deployment}")" ||
+ die "cannot determine desired replicas for mcp-host deployment/${deployment}"
+ if [[ "${desired_replicas}" == "0" ]]; then
+ # HCC intentionally suspends stateless Hosts by scaling their managed
+ # Deployment to zero. There is no consumer to restart or prove; the
+ # patched ConfigMap is the authoritative input for the next pod.
+ log "Skipping suspended ${CONFIG_NAMESPACE}/${deployment}; desired replicas=0"
+ continue
+ fi
if [[ "${force_rollout}" != true ]] && \
consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \
"${TARGET_KEY}" "mcp-host deployment/${deployment}"; then
@@ -357,7 +468,7 @@ sync_gfs() {
fi
fi
- local deployments=() rollout_deployments=() deployment deployment_probe role selector rollout_output dsn_encoded dsn
+ local deployments=() rollout_deployments=() deployment deployment_probe role selector rollout_output dsn_encoded dsn desired_replicas
for deployment in gfsc-writer gfsc-reader; do
deployment_probe=""
if deployment_probe="$("${KCTL[@]}" get "deployment/${deployment}" -n "${GFS_NAMESPACE}" 2>&1)"; then
@@ -378,6 +489,12 @@ sync_gfs() {
for deployment in "${deployments[@]}"; do
role="${deployment#gfsc-}"
selector="app=gfs-controller,clerum.io/gfsc-role=${role}"
+ desired_replicas="$(deployment_desired_replicas "${GFS_NAMESPACE}" "${deployment}")" ||
+ die "cannot determine desired replicas for GFS deployment/${deployment}"
+ if [[ "${desired_replicas}" == "0" ]]; then
+ log "Skipping suspended ${GFS_NAMESPACE}/${deployment}; desired replicas=0"
+ continue
+ fi
if [[ "${force_rollout}" != true ]] && \
consumer_pods_use_key "${GFS_NAMESPACE}" "${selector}" gfsc \
GFS_JWT_PUBLIC_KEY "gfsc deployment/${deployment}"; then
@@ -396,9 +513,17 @@ sync_gfs() {
fi
if ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then
+ if [[ "${GFS_AUTH_SYNC_ALLOW_STAGED}" == "true" ]]; then
+ log "Deferring active gfsc auth proof until GFS credentials are reconciled"
+ return 0
+ fi
die "cannot inspect ${GFS_NAMESPACE}/gfs-controller-db before gfsc restart: ${dsn_encoded}"
fi
if [[ -z "${dsn_encoded}" ]]; then
+ if [[ "${GFS_AUTH_SYNC_ALLOW_STAGED}" == "true" ]]; then
+ log "Deferring active gfsc auth proof until GFS credentials are reconciled"
+ return 0
+ fi
die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is empty; refusing to attest gfsc auth convergence"
fi
if ! dsn="$(printf '%s' "${dsn_encoded}" | base64 -d 2>&1)"; then
@@ -431,4 +556,5 @@ if [[ "${SYNC_GFS}" == "true" ]]; then
sync_gfs
fi
+assert_source_unchanged
log "Auth key synced"
diff --git a/scripts/minikube/t0.sh b/scripts/minikube/t0.sh
index 1521503e8..d8f777b26 100755
--- a/scripts/minikube/t0.sh
+++ b/scripts/minikube/t0.sh
@@ -12,15 +12,40 @@ package_dirs=()
seen_packages="|"
while IFS= read -r path; do
[ -n "$path" ] || continue
+ absolute_path="$PROJECT_DIR/$path"
case "$path" in
- *.sh|*.bash) shell_files+=("$PROJECT_DIR/$path") ;;
+ *.sh|*.bash)
+ # Deleted scripts are part of the diff but cannot be parsed. Every
+ # surviving shell script gets a syntax check even when shellcheck is not
+ # installed; shellcheck is an additional lint lane, not the syntax gate.
+ if [ -f "$absolute_path" ]; then
+ bash -n "$absolute_path"
+ shell_files+=("$absolute_path")
+ fi
+ ;;
esac
- top="${path%%/*}"
- if [ "$top" != "$path" ] && [ -f "$PROJECT_DIR/$top/package.json" ] && [[ "$seen_packages" != *"|$top|"* ]]; then
- seen_packages="${seen_packages}${top}|"
- package_dirs+=("$top")
+
+ # Resolve the nearest package.json by walking upward from the changed file.
+ # This covers nested packages such as packages/workflow-sdk instead of
+ # incorrectly mapping them to packages/package.json.
+ candidate="$absolute_path"
+ if [ ! -d "$candidate" ]; then
+ candidate="$(dirname -- "$candidate")"
fi
-done < <(git -C "$PROJECT_DIR" diff --name-only "$ORIGIN_DEV...$HEAD")
+ while [ "$candidate" != "$PROJECT_DIR" ] && [[ "$candidate" == "$PROJECT_DIR"/* ]]; do
+ if [ -f "$candidate/package.json" ]; then
+ package="${candidate#"$PROJECT_DIR/"}"
+ if [[ "$seen_packages" != *"|$package|"* ]]; then
+ seen_packages="${seen_packages}${package}|"
+ package_dirs+=("$package")
+ fi
+ break
+ fi
+ parent="$(dirname -- "$candidate")"
+ [ "$parent" != "$candidate" ] || break
+ candidate="$parent"
+ done
+done < <(git -C "$PROJECT_DIR" diff --name-only --diff-filter=ACMRTUXB "$ORIGIN_DEV...$HEAD")
if [ "${#shell_files[@]}" -gt 0 ]; then
if command -v shellcheck >/dev/null 2>&1; then
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 99a215739..e48bfc47b 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -47,6 +47,7 @@ T2_PLAN_MODE="$T2_PLAN_MODE"
if [ -z "$T2_PLAN_MODE" ]; then T2_PLAN_MODE=false; fi
T2_BOOTSTRAP_REQUIRED=false
T2_PROFILE_HEALTHY=false
+T2_CONTEXT_IDENTITY_VERIFIED=false
T2_UNREADY_DEPLOYMENTS=""
T2_MARKER_MATCHES_HEAD=false
T2_MARKER_JSON=""
@@ -79,6 +80,8 @@ T2_REQUIRED_NAMESPACES="$T2_REQUIRED_NAMESPACES"
if [ -z "$T2_REQUIRED_NAMESPACES" ]; then T2_REQUIRED_NAMESPACES="control-plane gfs mcp-host mcp-server profiles rpc-proxy channels sandbox-recipes sandbox-ui webhook-ingress registry"; fi
T2_REQUIRED_SERVICES="$T2_REQUIRED_SERVICES"
if [ -z "$T2_REQUIRED_SERVICES" ]; then T2_REQUIRED_SERVICES="control-plane/control-postgres control-plane/control-api control-plane/control-ui control-plane/host-context-controller control-plane/workflow-recipes profiles/external-rest-api profiles/profile-ui rpc-proxy/rpc-proxy mcp-server/mcp-proxy"; fi
+T2_REQUIRED_DEPLOYMENTS="$T2_REQUIRED_DEPLOYMENTS"
+if [ -z "$T2_REQUIRED_DEPLOYMENTS" ]; then T2_REQUIRED_DEPLOYMENTS="control-plane/control-api control-plane/host-context-controller profiles/external-rest-api rpc-proxy/rpc-proxy mcp-host/chatllm"; fi
T2_REQUIRED_SECRETS="$T2_REQUIRED_SECRETS"
if [ -z "$T2_REQUIRED_SECRETS" ]; then T2_REQUIRED_SECRETS="control-plane/control-postgres control-plane/control-api-secrets control-plane/control-ui-secrets control-plane/inter-service-tokens profiles/external-rest-api-secrets rpc-proxy/rpc-proxy-secrets mcp-host/chatllm-api-keys gfs/gfs-controller-db gfs/gfs-controller-reader-db"; fi
T2_REQUIRED_CONFIGMAPS="$T2_REQUIRED_CONFIGMAPS"
@@ -252,13 +255,65 @@ t2_context_check() {
host="${host%%:*}"
fi
host="$(printf '%s' "$host" | tr '[:upper:]' '[:lower:]')"
- case "$host" in
- 127.0.0.1|localhost|::1|192.168.*|*.minikube) ;;
- *)
- T2_NEXT_COMMAND='select the generated branch-owned Minikube context, not a remote cluster context'
- t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is not local: $host"
- ;;
- esac
+ if [[ "$host" == 127.0.0.1 || "$host" == localhost || "$host" == ::1 || "$host" == *.minikube ]]; then
+ return 0
+ fi
+ if ! python3 - "$host" <<'PY'
+import ipaddress
+import sys
+
+try:
+ address = ipaddress.ip_address(sys.argv[1])
+except ValueError:
+ raise SystemExit(1)
+raise SystemExit(0 if (address.is_private or address.is_loopback or address.is_link_local) else 1)
+PY
+ then
+ T2_NEXT_COMMAND='select the generated branch-owned Minikube context, not a remote cluster context'
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is not local: $host"
+ return 1
+ fi
+}
+
+t2_profile_context_identity_check() {
+ # A localhost API endpoint is necessary but not sufficient: kubeconfig can
+ # point at another local Minikube profile, and a broad RFC1918 allowlist can
+ # point at an unrelated LAN cluster. Bind the context to the exact profile
+ # identity and node address reported by the selected Minikube instance.
+ if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
+ return 0
+ fi
+ local expected_ip nodes_json
+ expected_ip="$(t2_mk ip 2>/dev/null || true)"
+ if [ -z "$expected_ip" ]; then
+ T2_NEXT_COMMAND='start the verified branch-owned Minikube profile, then retry the gate'
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Minikube did not report an IP for profile $T2_PROFILE"
+ return 1
+ fi
+ nodes_json="$(t2_kc get nodes -o json 2>/dev/null || true)"
+ if [ -z "$nodes_json" ] || ! python3 - "$nodes_json" "$T2_PROFILE" "$expected_ip" <<'PY'
+import json
+import sys
+
+payload = json.loads(sys.argv[1])
+profile = sys.argv[2]
+expected_ip = sys.argv[3]
+for node in payload.get("items", []):
+ metadata = node.get("metadata") or {}
+ labels = metadata.get("labels") or {}
+ if labels.get("minikube.k8s.io/name") != profile:
+ continue
+ for address in (node.get("status") or {}).get("addresses", []):
+ if address.get("type") == "InternalIP" and address.get("address") == expected_ip:
+ raise SystemExit(0)
+raise SystemExit(1)
+PY
+ then
+ T2_NEXT_COMMAND='select the kube-context generated for this exact branch-owned Minikube profile'
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context does not identify Minikube profile $T2_PROFILE at $expected_ip"
+ return 1
+ fi
+ T2_CONTEXT_IDENTITY_VERIFIED=true
}
t2_profile_status() {
@@ -490,22 +545,51 @@ t2_deployment_check() {
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is unavailable'
fi
- if ! unready="$(python3 - "$deployment_json" 2>/dev/null <<'PY'
+ if ! unready="$(python3 - "$deployment_json" "$T2_REQUIRED_DEPLOYMENTS" 2>/dev/null <<'PY'
import json
import sys
payload = json.loads(sys.argv[1])
+required_refs = {}
+for ref in sys.argv[2].split():
+ namespace, separator, name = ref.partition("/")
+ if not separator or not namespace or not name:
+ raise ValueError("invalid required deployment reference")
+ required_refs[(namespace, name)] = None
bad = []
for item in payload.get("items", []):
metadata = item.get("metadata") or {}
+ identity = (metadata.get("namespace"), metadata.get("name"))
+ if identity in required_refs:
+ required_refs[identity] = item
+
+for (namespace, name), item in required_refs.items():
+ if item is None:
+ bad.append(f"{namespace}/{name} missing")
+ continue
spec = item.get("spec") or {}
status = item.get("status") or {}
- desired = int(spec.get("replicas") or 0)
+ def integer(value, field):
+ if value in (None, ""):
+ return 0
+ if isinstance(value, bool):
+ raise ValueError(field)
+ return int(value)
+
+ desired = integer(spec.get("replicas"), "spec.replicas")
+ # A present deployment explicitly scaled to zero is an intentional local
+ # suspension; it is not an unready pod.
if desired == 0:
continue
- ready = int(status.get("readyReplicas") or 0)
- available = int(status.get("availableReplicas") or 0)
- if ready < desired or available < desired:
- bad.append("%s/%s %s/%s" % (metadata.get("namespace", "?"), metadata.get("name", "?"), ready, desired))
+ ready = integer(status.get("readyReplicas"), "status.readyReplicas")
+ available = integer(status.get("availableReplicas"), "status.availableReplicas")
+ updated = integer(status.get("updatedReplicas"), "status.updatedReplicas")
+ unavailable = integer(status.get("unavailableReplicas"), "status.unavailableReplicas")
+ generation = integer(metadata.get("generation"), "metadata.generation")
+ observed = integer(status.get("observedGeneration"), "status.observedGeneration")
+ if generation <= 0 or observed != generation or updated < desired or ready < desired or available < desired or unavailable != 0:
+ bad.append("%s/%s ready=%s/%s updated=%s observed=%s/%s unavailable=%s" % (
+ namespace, name, ready, desired,
+ updated, observed, generation, unavailable))
print("; ".join(bad))
PY
)"; then
@@ -538,6 +622,17 @@ PY
fi
}
+t2_pid_file_matches_process() {
+ local pid_file="$1" pid="$2" recorded_pid recorded_start actual_start
+ recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
+ recorded_start="$(sed -n 's/^PROCESS_START=//p' "$pid_file" 2>/dev/null | head -1 || true)"
+ [ "$recorded_pid" = "$pid" ] || return 1
+ [ -n "$recorded_start" ] || return 1
+ [ "$recorded_start" = unavailable ] && return 0
+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [ -n "$actual_start" ] && [ "$actual_start" = "$recorded_start" ]
+}
+
t2_process_check() {
local uid pid ppid rest command_line comm allowed pid_file recorded_pid
local safe_profile
@@ -547,14 +642,9 @@ t2_process_check() {
# Default IFS so UID/PID/PPID split. `IFS=` left pid empty and skipped every line.
# awk is a loose pre-filter: kubectl as argv0/path token AND a later
# standalone port-forward token. Flags may sit between those tokens.
- # Known limitation: a port-forward whose argv carries no profile/context
- # token (e.g. pf-control-stack's context-less `kubectl -n NS port-forward`)
- # cannot be attributed to a cluster from ps output alone and is skipped —
- # flagging it would also flag unrelated clusters' forwards.
while read -r uid pid ppid rest; do
[ -n "$pid" ] || continue
command_line="$uid $pid $ppid $rest"
- [[ "$command_line" == *"$T2_PROFILE"* || "$command_line" == *"$T2_CONTEXT"* ]] || continue
comm="$(ps -p "$pid" -o comm= 2>/dev/null || true)"
case "$comm" in
*kubectl*) ;;
@@ -566,19 +656,32 @@ t2_process_check() {
/tmp/pf-"$safe_profile"-*.pid; do
[ -f "$pid_file" ] || continue
recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
- [ "$recorded_pid" = "$pid" ] && allowed=true
+ if [ "$recorded_pid" = "$pid" ] && t2_pid_file_matches_process "$pid_file" "$pid"; then
+ allowed=true
+ fi
done
# pf-all-stack.sh is the canonical gate forwarder and records its child
# PIDs in /tmp/pf-<profile>-*.pid. Accept those PIDs as profile-owned too;
# otherwise T1 rejects the forwards that pre-gate-sync just started.
- for pid_file in "/tmp/pf-${profile_safe}-"*.pid; do
+ for pid_file in "/tmp/pf-${safe_profile}-"*.pid; do
[ -f "$pid_file" ] || continue
- pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
- [[ -n "$pid" && "$command_line" == *" $pid "* ]] && allowed=true
+ recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
+ if [[ -n "$recorded_pid" && "$command_line" == *" $recorded_pid "* ]] &&
+ t2_pid_file_matches_process "$pid_file" "$recorded_pid"; then
+ allowed=true
+ fi
done
+ # Unrelated developer forwards launched without --context are not
+ # attributable to this profile and must not make its gate red. A forward
+ # that names this profile/context, or one recorded in this profile's PID
+ # registry, is attributable and must be owned by this run.
+ if [[ "$command_line" != *"$T2_PROFILE"* && "$command_line" != *"$T2_CONTEXT"* && "$allowed" != true ]]; then
+ continue
+ fi
if [ "$allowed" != true ]; then
T2_NEXT_COMMAND='stop the unrelated profile port-forward or select the owner worktree; do not share it'
t2_fail PORT_FORWARD_CONFLICT 'a port-forward for this profile is owned by another process'
+ return 1
fi
done < <(ps -ef 2>/dev/null | awk '/([^[:space:]]*\/)?kubectl([[:space:]]|$)/ && /[[:space:]]port-forward([[:space:]]|$)/ {print}' || true)
}
@@ -749,18 +852,20 @@ EOF
t2_lock_validate_inherited() {
local expected_pid expected_token owner_key owner_value
- t2_lock_profile_id_check
+ t2_lock_profile_id_check || return 1
T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
T2_LOCK_KEY="$(t2_lock_key)"
if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR/owner.env" ]; then
T2_NEXT_COMMAND='run the parent T2 command with its profile lock token, then retry this child operation'
t2_fail PROFILE_LOCK_REQUIRED "inherited profile lock is missing for $T2_PROFILE"
+ return 1
fi
expected_token="$T2_LOCK_TOKEN"
owner_value="$(t2_lock_owner_value TOKEN || true)"
if [ -z "$expected_token" ] || [ "$owner_value" != "$expected_token" ]; then
T2_NEXT_COMMAND='pass the opaque T2_LOCK_TOKEN from the owning T2 process; do not bypass the profile lock'
t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock token is missing or does not match'
+ return 1
fi
for owner_key in REPOSITORY BRANCH HEAD PROFILE CONTEXT WORKTREE_ID LOCK_KEY; do
case "$owner_key" in
@@ -775,12 +880,14 @@ t2_lock_validate_inherited() {
if [ "$(t2_lock_owner_value "$owner_key" || true)" != "$expected_pid" ]; then
T2_NEXT_COMMAND='re-run the operation from the worktree/profile that owns the T2 lock'
t2_fail PROFILE_OWNERSHIP_MISMATCH "profile lock owner does not match $owner_key"
+ return 1
fi
done
expected_pid="$(t2_lock_owner_value PID || true)"
if ! t2_lock_process_matches "$expected_pid"; then
T2_NEXT_COMMAND='restart the parent T2 operation so it can acquire a live profile lock'
t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'
+ return 1
fi
T2_LOCK_HELD=false
T2_LOCK_RELEASED=false
@@ -812,6 +919,12 @@ t2_lock_release() {
if [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; then
t2_evidence_write failure FAIL "$T2_ERROR_CODE"
fi
+ if [ "$cleanup_status" -ne 0 ] && [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ]; then
+ # A PASS written before EXIT is not a valid attestation if the profile lock
+ # could not be released. Invalidate it explicitly and keep the detail
+ # secret-free so a later runtime-only lane cannot reuse unsafe evidence.
+ t2_evidence_write lock-cleanup INVALIDATED 'profile lock cleanup failed; prior attestation is invalid' || true
+ fi
if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
status="$cleanup_status"
fi
@@ -898,6 +1011,12 @@ if status in {"FAIL", "INVALIDATED"}:
elif phase == "complete" and status == "PASS":
attestation = "PASS"
prior["attestationStatus"] = attestation
+lane_attestation = prior.get("laneAttestationStatus", "IN_PROGRESS")
+if phase == "lanes" and status == "PASS":
+ lane_attestation = "PASS"
+elif phase == "lock-cleanup" and status == "INVALIDATED":
+ lane_attestation = "INVALIDATED"
+prior["laneAttestationStatus"] = lane_attestation
prior.setdefault("phases", []).append({
"name": os.environ.get("PHASE", ""),
"status": os.environ.get("STATUS", ""),
@@ -945,7 +1064,7 @@ for candidate in sorted(root.glob("*/evidence.json"), reverse=True):
continue
if data.get("certificationVersion") != 1:
continue
- if data.get("attestationStatus") != "PASS":
+ if data.get("attestationStatus") != "PASS" and data.get("laneAttestationStatus") != "PASS":
continue
if not data.get("runId"):
continue
diff --git a/scripts/minikube/t2-preflight.sh b/scripts/minikube/t2-preflight.sh
index 4a90e54bf..26b73146d 100755
--- a/scripts/minikube/t2-preflight.sh
+++ b/scripts/minikube/t2-preflight.sh
@@ -21,7 +21,12 @@ T2_NEXT_COMMAND='re-run the canonical command with the verified profile'
cleanup_preflight() {
local status=$?
- t2_lock_release "$status"
+ trap - EXIT
+ trap '' INT TERM
+ if ! t2_lock_release "$status"; then
+ [ "$status" -ne 0 ] || status=1
+ fi
+ exit "$status"
}
trap cleanup_preflight EXIT
@@ -29,13 +34,16 @@ main() {
t2_require_commands
t2_repo_metadata
t2_profile_scope
- t2_context_check
+ t2_profile_status
+ if [ "$T2_BOOTSTRAP_REQUIRED" != true ]; then
+ t2_context_check
+ fi
t2_mutation_lock
t2_evidence_init
# Profile status is read before any resource check. A missing/stopped profile
# is a plan for bootstrap, never a reason to invoke pre-gate-sync.
- t2_profile_status
+ t2_profile_context_identity_check
t2_marker_check
t2_image_check
t2_resource_checks
@@ -48,6 +56,7 @@ main() {
if [ "$T2_PLAN_STATE" = full-bootstrap ] && [ "$T2_PLAN_MODE" != true ]; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE IMAGE_SOURCE=local make minikube-setup"
t2_fail BOOTSTRAP_REQUIRED "$T2_PLAN_REASON"
+ return 1
fi
t2_evidence_write preflight PASS "$T2_PLAN_STATE: $T2_PLAN_REASON"
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index 30712ea71..f7547cb9c 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -87,6 +87,7 @@ run_preflight_plan() {
bash "$SCRIPT_DIR/t2-preflight.sh"; then
T2_NEXT_COMMAND='re-run make minikube-t2-preflight and repair its first reported precondition'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'preflight failed before a state transition was selected'
+ return 1
fi
T2_PLAN_STATE="$(python3 - "$T2_PLAN_TMP" <<'PY'
import json
@@ -103,6 +104,7 @@ PY
if [ -z "$T2_PLAN_STATE" ]; then
T2_NEXT_COMMAND='re-run make minikube-t2-preflight and inspect its evidence'
t2_fail BOOTSTRAP_REQUIRED 'preflight did not produce a state transition'
+ return 1
fi
printf '[minikube-t2] transition=%s reason=%s\n' "$T2_PLAN_STATE" "$T2_PLAN_REASON"
}
@@ -115,12 +117,14 @@ run_pvc_reset_if_authorized() {
if [ -z "$T2_EXPECTED_PVC_UID" ]; then
T2_NEXT_COMMAND='set T2_EXPECTED_PVC_UID to the exact recorded UID and keep T2_RESET_PVC=true only for local development'
t2_fail POSTGRES_NOT_READY 'destructive PVC reset requires an exact expected UID'
+ return 1
fi
local actual_uid
actual_uid="$(t2_kc -n "$T2_CONTROL_NAMESPACE" get pvc control-postgres-data -o 'jsonpath={.metadata.uid}' 2>/dev/null || true)"
if [ "$actual_uid" != "$T2_EXPECTED_PVC_UID" ]; then
T2_NEXT_COMMAND='do not reset this PVC; re-check the profile and recorded UID'
t2_fail POSTGRES_NOT_READY 'PVC UID does not match the explicit reset expectation'
+ return 1
fi
printf '[minikube-t2] authorized development-only PVC reset for the exact expected UID\n'
}
@@ -162,6 +166,7 @@ run_bootstrap_or_reconcile() {
*)
T2_NEXT_COMMAND='re-run make minikube-t2-preflight to select bootstrap, targeted sync, full reconcile, or already-synced'
t2_fail BOOTSTRAP_REQUIRED "unknown transition: $T2_PLAN_STATE"
+ return 1
;;
esac
t2_evidence_write transition PASS "$T2_PLAN_STATE"
@@ -210,6 +215,7 @@ run_t1() {
t2_evidence_write T1 FAIL 'Real PostgreSQL lane exited zero but did not emit a complete green T1 result'
T2_NEXT_COMMAND='repair the T1 result contract; it must emit PASS, positive total/passed counts, zero pending tests, and an evidence path'
t2_fail REAL_PG_SUITE_FAILED 'T1 result was incomplete despite a zero process exit'
+ return 1
fi
T2_T1_STATUS=PASS
T2_T1_COUNTS="$(awk -F= '/^T1_TESTS=|^T1_PASSED_TESTS=|^T1_PENDING_TESTS=/{printf "%s%s", (n++ ? "," : ""), $0}' "$T2_T1_OUTPUT")"
@@ -234,6 +240,7 @@ run_final_preflight() {
cat "$T2_FINAL_PREFLIGHT_OUTPUT" >&2 || true
T2_NEXT_COMMAND='repair the first reported final preflight condition, then re-run T2 on the same HEAD'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'final exact-head readiness preflight failed'
+ return 1
fi
T2_PLAN_STATE="$(python3 - "$T2_PLAN_TMP" <<'PY'
import json
@@ -244,6 +251,7 @@ PY
if [ "$T2_PLAN_STATE" != already-synced ]; then
T2_NEXT_COMMAND='re-run make minikube-t2 so pre-gate-sync updates the marker to this HEAD'
t2_fail HEAD_MARKER_MISMATCH "final T2 preflight selected $T2_PLAN_STATE instead of already-synced"
+ return 1
fi
t2_marker_check
T2_T2_STATUS=PASS
@@ -283,6 +291,7 @@ run_healthcheck_if_requested() {
t2_evidence_write Health FAIL 'profile-owned user-facing health command failed'
T2_NEXT_COMMAND='repair the first failing user-facing health check, then re-run T2 on the same HEAD'
t2_fail PROFILE_UNHEALTHY 'user-facing health check failed'
+ return 1
fi
}
@@ -293,6 +302,7 @@ run_playwright_if_requested() {
if [ "$T2_REQUIRE_PLAYWRIGHT" = true ]; then
T2_NEXT_COMMAND='set T2_PLAYWRIGHT_COMMAND to the applicable profile-owned Playwright journey and re-run'
t2_fail ZERO_TESTS_EXECUTED 'Playwright was required but no journey command was supplied'
+ return 1
fi
return 0
fi
@@ -310,7 +320,10 @@ main() {
t2_require_commands
t2_repo_metadata
t2_profile_scope
- t2_context_check
+ t2_profile_status
+ if [ "$T2_BOOTSTRAP_REQUIRED" != true ]; then
+ t2_context_check
+ fi
t2_mutation_lock
t2_evidence_init
@@ -318,11 +331,13 @@ main() {
if [ "$T2_RUN_T0" = true ] && ! t2_lane_completed "$T2_T0_STATUS"; then
T2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T0=true so the static lane is executed'
t2_fail ZERO_TESTS_EXECUTED 'T0 was not executed'
+ return 1
fi
run_preflight_plan
if { [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; } && [ "$T2_PLAN_STATE" != already-synced ]; then
T2_NEXT_COMMAND='run make minikube-t2 so bootstrap/reconcile and T0/T1 execute on this HEAD'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2-only mode requires an already-synced exact-head profile'
+ return 1
fi
if [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; then
t2_marker_check
@@ -334,10 +349,18 @@ main() {
if [ "$T2_RUN_T1" = true ] && ! t2_lane_completed "$T2_T1_STATUS"; then
T2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T1=true so the Real PostgreSQL lane is executed'
t2_fail ZERO_TESTS_EXECUTED 'T1 was not executed'
+ return 1
fi
if [ "$T2_RUN_T1" != true ] && [ "$T2_T1_CERTIFIED" != true ]; then
T2_NEXT_COMMAND='run make minikube-t2 with T2_RUN_T1=true to create an exact-head T1 attestation'
t2_fail CERTIFICATION_REQUIRED 'T1 was skipped without a validated exact-head attestation'
+ return 1
+ fi
+ if [ "$T2_T0_STATUS" = PASS ] && [ "$T2_T1_STATUS" = PASS ]; then
+ # Publish the reusable static/database lane attestation before runtime T2.
+ # If the later runtime lane fails, a subsequent runtime-only invocation may
+ # reuse these exact-head lanes without pretending that T2 already passed.
+ t2_evidence_write lanes PASS 'T0 and T1 completed before runtime T2'
fi
run_final_preflight
run_np08_hcc_authorization
diff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.sh
index c0f9f6b8a..5683b7e6d 100755
--- a/scripts/minikube/wait-gfs-reader-ready.sh
+++ b/scripts/minikube/wait-gfs-reader-ready.sh
@@ -21,16 +21,22 @@ kc() { kubectl --context="$CONTEXT" "$@"; }
log() { printf '[wait-gfs-reader-ready] %s\n' "$*" >&2; }
if ! TIMEOUT_SECONDS="$(python3 - "$TIMEOUT_SECONDS" <<'PY'
+import math
import re
import sys
raw = sys.argv[1]
-if not re.fullmatch(r"[1-9][0-9]{0,4}", raw):
- raise SystemExit("timeout must be a positive integer")
-value = int(raw)
-if value > 86400:
+if re.fullmatch(r"[1-9][0-9]*", raw):
+ total = float(raw)
+else:
+ tokens = re.findall(r"([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h)", raw)
+ if not tokens or "".join(number + unit for number, unit in tokens) != raw:
+ raise SystemExit("timeout must be a positive Go-style duration")
+ factors = {"ns": 1e-9, "us": 1e-6, "µs": 1e-6, "ms": 1e-3, "s": 1.0, "m": 60.0, "h": 3600.0}
+ total = sum(float(number) * factors[unit] for number, unit in tokens)
+if total <= 0 or total > 86400:
raise SystemExit("timeout exceeds the one-day limit")
-print(value)
+print(max(1, math.ceil(total)))
PY
)"; then
log 'refusing an invalid readiness timeout'
@@ -67,9 +73,10 @@ while :; do
log "unable to inspect reader pods for ${GFS_NS}/${DEPLOY}: ${pod_rows}"
exit 1
fi
+ live_ready="$(awk -F'|' 'NF && $2 == "" && $1 == "True" { n++ } END { print n+0 }' <<<"$pod_rows")"
unready="$(awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }' <<<"$pod_rows")"
- if [ "${unready:-0}" -eq 0 ]; then
- log "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with no live unready reader pod"
+ if [ "${live_ready:-0}" -ge "$desired" ] && [ "${unready:-0}" -eq 0 ]; then
+ log "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with ${live_ready} live Ready reader pod(s)"
exit 0
fi
fi
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index 8a12acfb1..fefbe5785 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -48,8 +48,8 @@ if CONTEXT=unverified-context bash deploy/scripts/reconcile-gfs-deploy-credentia
rm -f "$reconcile_context_err"
fail 'credential reconciliation accepted an unverified Kubernetes context'
fi
-grep -Fq 'refusing unverified Kubernetes context' "$reconcile_context_err" \
- || fail 'credential reconciliation did not explain its unverified-context rejection'
+grep -Fq 'worktree is dirty' "$reconcile_context_err" \
+ || fail 'credential reconciliation did not fail closed before an unowned worktree could mutate GFS'
rm -f "$reconcile_context_err"
reconcile_remote_context_err="$(mktemp)"
if CONTEXT=gke_unapproved GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=gke_approved \
@@ -57,13 +57,14 @@ if CONTEXT=gke_unapproved GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=
rm -f "$reconcile_remote_context_err"
fail 'credential reconciliation accepted a GKE context outside the explicit allowlist'
fi
-grep -Fq 'explicit authorization and exact ALLOWED_CONTEXTS membership' "$reconcile_remote_context_err" \
- || fail 'credential reconciliation did not fail closed for an unauthorized GKE context'
+grep -Fq 'worktree is dirty' "$reconcile_remote_context_err" \
+ || fail 'credential reconciliation did not fail closed before a protected context could mutate GFS'
rm -f "$reconcile_remote_context_err"
-grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED' deploy/scripts/reconcile-gfs-deploy-credentials.sh \
- || fail 'remote GFS reconciliation lacks an explicit authorization gate'
-grep -Fq 'ALLOWED_CONTEXTS' deploy/scripts/reconcile-gfs-deploy-credentials.sh \
- || fail 'remote GFS reconciliation lacks an exact context allowlist'
+grep -Fq 't2_mutation_lock' deploy/scripts/reconcile-gfs-deploy-credentials.sh \
+ || fail 'GFS reconciliation does not require the canonical profile mutation lock'
+if grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED=true' deploy/scripts/reconcile-gfs-deploy-credentials.sh; then
+ fail 'GFS reconciliation still trusts caller-provided remote authorization'
+fi
make_block() {
awk -v target="$1" '$0 ~ "^" target ":" {active=1} active && /^\.PHONY:/ {exit} active {print}' Makefile
}
diff --git a/scripts/tests/test-gfs-real-pg-minikube-gate.sh b/scripts/tests/test-gfs-real-pg-minikube-gate.sh
index 99c89b2a0..2105a6e44 100755
--- a/scripts/tests/test-gfs-real-pg-minikube-gate.sh
+++ b/scripts/tests/test-gfs-real-pg-minikube-gate.sh
@@ -7,9 +7,16 @@ SCRIPT="${ROOT_DIR}/scripts/e2e/gfs-real-pg-minikube-gate.sh"
bash -n "${SCRIPT}"
grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "${SCRIPT}"
grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "${SCRIPT}"
-grep -Fq 'gitHead' "${SCRIPT}"
-grep -Fq 'worktreeId' "${SCRIPT}"
-grep -Fq 'port-forward' "${SCRIPT}"
+grep -Fq 'verify_branch_gate' "${SCRIPT}"
+grep -Fq 't2_profile_scope' "${SCRIPT}"
+grep -Fq 't2_profile_context_identity_check' "${SCRIPT}"
+grep -Fq 't2_marker_check' "${SCRIPT}"
+grep -Fq 't2_mutation_lock' "${SCRIPT}"
+grep -Fq 'rollout status deployment/control-postgres' "${SCRIPT}"
+grep -Fq 'get secret control-postgres' "${SCRIPT}"
+grep -Fq 'PROFILE_PG_FORWARD_PID' "${SCRIPT}"
+grep -Fq 'docker run -d --rm' "${SCRIPT}"
+grep -Fq 'ISOLATED_PG_IMAGE' "${SCRIPT}"
grep -Fq 'pending_files' "${SCRIPT}"
grep -Fq -- '--reporter=json' "${SCRIPT}"
grep -Fq "gfs*.realPostgres.integration.test.ts" "${SCRIPT}"
@@ -19,8 +26,11 @@ grep -Fq 'restore_gfs_runtime_credentials' "${SCRIPT}"
grep -Fq 'required gfs-controller-db Secret is missing or unreadable' "${SCRIPT}"
grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "${SCRIPT}"
grep -Fq 'failed to restore branch-profile GFS credentials' "${SCRIPT}"
-grep -Fq 'gke_*)' "${SCRIPT}"
-grep -Fq '*prod*)' "${SCRIPT}"
+grep -Fq 'docker rm -f' "${SCRIPT}"
+if grep -Fq 'docker rm -f "${ISOLATED_CONTAINER}" >/dev/null 2>&1 || true' "${SCRIPT}"; then
+ echo 'FAIL: standalone GFS cleanup suppresses Docker removal failure' >&2
+ exit 1
+fi
grep -Fq 'test-gfs-real-postgres-minikube' "${ROOT_DIR}/Makefile"
grep -Fq "setupFiles: ['test/realPostgres.requirement.ts']" "${ROOT_DIR}/control-api/vitest.config.ts"
grep -Fq 'setupFiles: ["test/realPostgres.requirement.ts"]' "${ROOT_DIR}/gfs-controller/vitest.config.ts"
diff --git a/scripts/tests/test-minikube-gfs-provision-order.sh b/scripts/tests/test-minikube-gfs-provision-order.sh
index 422d0adcd..d1356d556 100755
--- a/scripts/tests/test-minikube-gfs-provision-order.sh
+++ b/scripts/tests/test-minikube-gfs-provision-order.sh
@@ -88,7 +88,7 @@ sync_mcp_host_auth_key() {
}
provision_gfs_serving
-expected=$'sync\nsettle\nreconcile\nconverge'
+expected=$'sync\nsettle\nreconcile\nsync\nconverge'
actual="$(cat "$order_file")"
[ "$actual" = "$expected" ] || {
printf 'expected:\n%s\nactual:\n%s\n' "$expected" "$actual" >&2
diff --git a/scripts/tests/test-minikube-gfs-rollout-shim.sh b/scripts/tests/test-minikube-gfs-rollout-shim.sh
index 6664581df..878ec920c 100755
--- a/scripts/tests/test-minikube-gfs-rollout-shim.sh
+++ b/scripts/tests/test-minikube-gfs-rollout-shim.sh
@@ -62,6 +62,13 @@ grep -q 'gfs-rollout-shim' <<<"$out" || fail 'shim did not announce the readines
grep -q 'rollout status deployment/gfsc-reader' "$FAKE_KUBECTL_LOG" \
&& fail 'shim leaked the generation-based rollout status to the real kubectl'
+for duration in 2m30s 500ms; do
+ PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\n' \
+ GFS_READER_WAIT_POLL_SECONDS=0 \
+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout="$duration" >/dev/null 2>&1 \
+ || fail "shim rejected valid Go duration ${duration}"
+done
+
# 4. A live unready reader pod blocks the wait until timeout (fail-loud).
# The explicit 1s caller timeout is honored; the shim must not silently floor
# it to an unrelated minimum.
@@ -94,4 +101,13 @@ PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \
kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout=240s >/dev/null 2>&1 \
|| fail 'shim readiness wait blocked on a terminating pod'
+# A terminating Ready pod must not satisfy the desired live-reader count. It
+# is already leaving the service and cannot be used as readiness evidence.
+if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \
+ FAKE_POD_ROWS='True|2026-01-01T00:00:00Z\n' \
+ GFS_READER_WAIT_POLL_SECONDS=0 \
+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout=1s >/dev/null 2>&1; then
+ fail 'shim readiness wait went green with only a terminating Ready reader pod'
+fi
+
printf 'PASS: gfs-rollout-shim readiness interception contract\n'
diff --git a/scripts/tests/test-minikube-pre-gate-sync-state.sh b/scripts/tests/test-minikube-pre-gate-sync-state.sh
index 82efc12be..3af20ddeb 100755
--- a/scripts/tests/test-minikube-pre-gate-sync-state.sh
+++ b/scripts/tests/test-minikube-pre-gate-sync-state.sh
@@ -186,7 +186,7 @@ fi
final_cluster_fingerprint_line="$(grep -n 'cluster_fingerprint=.*pre_gate_marker_cluster_fingerprint' "$SCRIPT" | tail -n 1 | cut -d: -f1)"
final_infra_fingerprint_line="$(grep -n 'infra_fingerprint=.*pre_gate_marker_infra_fingerprint' "$SCRIPT" | tail -n 1 | cut -d: -f1)"
-persist_marker_line="$(grep -nF 'persist_cluster_marker "${cluster_fingerprint}" "${infra_fingerprint}"' "$SCRIPT" | tail -n 1 | cut -d: -f1)"
+persist_marker_line="$(grep -nF 'commit_cluster_sync_state "${cluster_fingerprint}" "${infra_fingerprint}"' "$SCRIPT" | tail -n 1 | cut -d: -f1)"
if [ -n "$final_cluster_fingerprint_line" ] &&
[ -n "$final_infra_fingerprint_line" ] &&
[ -n "$persist_marker_line" ] &&
diff --git a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
index ab4243ce7..c03134af3 100755
--- a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
+++ b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
@@ -61,9 +61,12 @@ case " $* " in
exit 0 ;;
*' get rs -l '*)
printf '%b' "${FAKE_RS_ROWS:-}"; exit 0 ;;
+ *' get rs gfsc-reader-stale '*'.spec.replicas'*)
+ [ -s "${FAKE_SCALE_FILE:?}" ] && printf '0' || printf '1'; exit 0 ;;
*' get rs gfsc-reader-proof-rs '*revision*)
printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;
*' scale rs '*)
+ printf '%s' "${4:-gfsc-reader-stale}" >"${FAKE_SCALE_FILE:?}"
exit 0 ;;
*' get pods -l '*gfs-template-hash*)
printf '%b' "${FAKE_PROOF_POD_ROWS:-gfsc-reader-proof|2026-01-02T00:00:01Z|True||gfsc-reader-proof-rs|}"; exit 0 ;;
@@ -104,7 +107,9 @@ exit 1
STUB
chmod +x "$fake_dir/kubectl"
local status=0
+ : >"$fake_dir/scaled-rs"
PATH="$fake_dir:$PATH" CONTEXT="$profile" \
+ T2_LOCK_TOKEN=settle-test-token \
T2_PROJECT_DIR="$repo" T2_PROFILE="$profile" T2_CONTEXT="$profile" \
MINIKUBE_PROFILE="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" \
T2_PROFILE_ROOT="$fake_dir/profiles" T2_PROFILE_ENV="$fake_dir/profiles/$profile/profile.env" \
@@ -113,6 +118,7 @@ STUB
T2_GATE_ID=settle-gfs-reader-test T2_RUN_ID=settle-test-run \
FAKE_KUBECTL_LOG="$fake_dir/kubectl.log" \
FAKE_PATCH_LOG="$fake_dir/patch.json" FAKE_DSN=dsn-value \
+ FAKE_SCALE_FILE="$fake_dir/scaled-rs" \
FAKE_DEPLOY_EXISTS="${1:-1}" \
FAKE_DESIRED="${2:-1}" \
FAKE_READY="${3:-1}" \
diff --git a/scripts/tests/test-minikube-sync-auth-key-durable.sh b/scripts/tests/test-minikube-sync-auth-key-durable.sh
index 8a6c13cb8..3c8ebdb6f 100755
--- a/scripts/tests/test-minikube-sync-auth-key-durable.sh
+++ b/scripts/tests/test-minikube-sync-auth-key-durable.sh
@@ -12,6 +12,24 @@ SOURCE_HASH="$(printf 'public-key' | shasum -a 256 | awk '{print $1}')"
mkdir -p "${STATE_DIR}"
printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"
+# The production helper requires a live parent T2 lease. Keep the behavior
+# tests isolated with a lease owned by this test shell rather than adding a
+# production-only bypass.
+T2_LOCK_ROOT="${TMP_DIR}/locks"
+T2_LOCK_TOKEN='auth-sync-test-token'
+T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"
+T2_PROCESS_START=unavailable
+mkdir -p "${T2_LOCK_DIR}"
+cat >"${T2_LOCK_DIR}/owner.env" <<EOF
+REPOSITORY=${ROOT}
+PROFILE=fake
+CONTEXT=fake
+TOKEN=${T2_LOCK_TOKEN}
+PID=$$
+PROCESS_START=${T2_PROCESS_START}
+EOF
+export T2_PROJECT_DIR="${ROOT}" T2_PROFILE=fake T2_CONTEXT=fake T2_LOCK_ROOT T2_LOCK_TOKEN
+
cat >"${TMP_DIR}/kubectl" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
@@ -30,13 +48,44 @@ increment() {
}
if [[ "${1:-}" == get && "${2:-}" == secret && "${3:-}" == rpc-proxy-secrets ]]; then
- [[ "$*" == *jsonpath* ]] && printf '%s' "${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"
+ if [[ "$*" == *'.metadata.resourceVersion'* ]]; then
+ printf '42'
+ elif [[ "$*" == *'-o json' ]]; then
+ source_b64="${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"
+ source_rv=42
+ if [[ -n "${TEST_ROTATE_ON_SOURCE_SNAPSHOT:-}" ]]; then
+ snapshot_file="${TEST_STATE_DIR}/source-json-snapshot.count"
+ snapshot_count=0
+ [[ -f "${snapshot_file}" ]] && snapshot_count="$(cat "${snapshot_file}")"
+ snapshot_count=$((snapshot_count + 1))
+ printf '%s' "${snapshot_count}" >"${snapshot_file}"
+ if (( snapshot_count >= TEST_ROTATE_ON_SOURCE_SNAPSHOT )); then
+ source_b64='cm90YXRlZA=='
+ source_rv=43
+ fi
+ fi
+ printf '{"metadata":{"resourceVersion":"%s"},"data":{"RPC_PROXY_JWT_PUBLIC_KEY":"%s"}}' "${source_rv}" "${source_b64}"
+ elif [[ "$*" == *jsonpath* ]]; then
+ printf '%s' "${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"
+ fi
exit 0
fi
if [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == mcp-host-config ]]; then
if [[ "$*" == *auth-key-applied-sha256* ]]; then
[[ -f "${TEST_STATE_DIR}/mcp.applied" ]] && cat "${TEST_STATE_DIR}/mcp.applied"
+ elif [[ "$*" == *'-o json' ]]; then
+ python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+state = Path(os.environ['TEST_STATE_DIR'])
+annotations = {}
+if (state / 'mcp.applied').exists():
+ annotations['clerum.io/auth-key-applied-sha256'] = (state / 'mcp.applied').read_text()
+print(json.dumps({'metadata': {'resourceVersion': '42', 'annotations': annotations},
+ 'data': {'CLERUM_AUTH_JWT_PUBLIC_KEY': (state / 'mcp.key').read_text()}}))
+PY
elif [[ "$*" == *jsonpath* ]]; then
cat "${TEST_STATE_DIR}/mcp.key"
fi
@@ -50,6 +99,18 @@ if [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == gfs-config ]]; the
}
if [[ "$*" == *auth-key-applied-sha256* ]]; then
[[ -f "${TEST_STATE_DIR}/gfs.applied" ]] && cat "${TEST_STATE_DIR}/gfs.applied"
+ elif [[ "$*" == *'-o json' ]]; then
+ python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+state = Path(os.environ['TEST_STATE_DIR'])
+annotations = {}
+if (state / 'gfs.applied').exists():
+ annotations['clerum.io/auth-key-applied-sha256'] = (state / 'gfs.applied').read_text()
+print(json.dumps({'metadata': {'resourceVersion': '42', 'annotations': annotations},
+ 'data': {'jwt-public-key': (state / 'gfs.key').read_text()}}))
+PY
elif [[ "$*" == *jsonpath* ]]; then
cat "${TEST_STATE_DIR}/gfs.key"
fi
@@ -84,10 +145,15 @@ fi
if [[ "${1:-}" == get && "${2:-}" == deployment/* ]]; then
case "${2}" in
- deployment/chatllm|deployment/custom-host) exit 0 ;;
+ deployment/chatllm|deployment/custom-host)
+ [[ "$*" == *'.spec.replicas'* ]] && printf '%s' "${TEST_MCP_REPLICAS:-1}"
+ exit 0 ;;
deployment/mcp-host) echo 'Error from server (NotFound): deployments.apps "mcp-host" not found' >&2; exit 1 ;;
deployment/gfsc-writer|deployment/gfsc-reader)
- if [[ "${TEST_GFS_DEPLOYMENTS:-0}" == 1 ]]; then exit 0; fi
+ if [[ "${TEST_GFS_DEPLOYMENTS:-0}" == 1 ]]; then
+ [[ "$*" == *'.spec.replicas'* ]] && printf '1'
+ exit 0
+ fi
printf 'Error from server (NotFound): deployments.apps "%s" not found\n' "${2#deployment/}" >&2
exit 1 ;;
*) exit 1 ;;
@@ -145,7 +211,15 @@ fi
echo "unexpected kubectl invocation: $*" >&2
exit 99
SH
-chmod +x "${TMP_DIR}/kubectl"
+cat >"${TMP_DIR}/ps" <<'SH'
+#!/usr/bin/env bash
+case " $* " in
+ *' -o state='*) printf 'R\n' ;;
+ *' -o lstart='*) printf 'unavailable\n' ;;
+ *) exit 1 ;;
+esac
+SH
+chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/ps"
run_sync() {
local source_hash="${TEST_SOURCE_HASH_OVERRIDE:-${SOURCE_HASH}}"
@@ -206,7 +280,19 @@ if run_sync env TEST_RUNTIME_KEY=old-key TEST_MCP_DEPLOYMENTS=custom \
exit 1
fi
[[ "$(cat "${STATE_DIR}/custom-host-restart.count")" == 1 ]]
-grep -q 'has not consumed the target auth key' "${OUT_FILE}"
+
+# A stateless HCC Host can be intentionally suspended at replicas=0. The
+# ConfigMap still converges, but there is no pod to restart or prove; a future
+# pod must consume the current ConfigMap value.
+printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"
+rm -f "${STATE_DIR}/mcp.applied"
+run_sync env TEST_MCP_REPLICAS=0 TEST_RUNTIME_KEY=old-key \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1
+[[ "$(cat "${STATE_DIR}/mcp.key")" == public-key ]]
+[[ "$(cat "${STATE_DIR}/mcp.applied")" == "${SOURCE_HASH}" ]]
+[[ "$(cat "${STATE_DIR}/chatllm-restart.count")" == 1 ]]
+grep -q 'Skipping suspended mcp-host/chatllm' "${OUT_FILE}"
run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=custom \
bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
@@ -277,4 +363,24 @@ if grep -q 'rollout restart deployment/' "${LOG_FILE}"; then
exit 1
fi
+# A source rotation after consumer proof must invalidate the run before the
+# final success message, even when the previous convergence marker was already
+# written. This is the race the resourceVersion+key snapshot closes.
+printf 'old-key' >"${STATE_DIR}/mcp.key"
+rm -f "${STATE_DIR}/mcp.applied" "${STATE_DIR}/source-json-snapshot.count"
+: >"${LOG_FILE}"
+if TEST_ROTATE_ON_SOURCE_SNAPSHOT=3 run_sync env \
+ TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=chatllm \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1; then
+ cat "${OUT_FILE}" >&2
+ echo 'FAIL: auth-key sync certified a source that rotated after consumer proof' >&2
+ exit 1
+fi
+grep -q 'auth source rotated during convergence' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo 'FAIL: auth-key sync did not report the final source rotation' >&2
+ exit 1
+}
+
echo "PASS: auth-key convergence is durable, resumable, consumer-proven, and idempotent"
diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.sh
index 08d0f320f..5c02f7175 100755
--- a/scripts/tests/test-minikube-sync-auth-key.sh
+++ b/scripts/tests/test-minikube-sync-auth-key.sh
@@ -10,11 +10,38 @@ LOG_FILE="${TMP_DIR}/kubectl.log"
OUT_FILE="${TMP_DIR}/sync-auth-key.out"
mkdir -p "${STATE_DIR}"
+# The production helper requires a live parent T2 lease. Keep the behavior
+# tests isolated with a lease owned by this test shell rather than adding a
+# production-only bypass.
+T2_LOCK_ROOT="${TMP_DIR}/locks"
+T2_LOCK_TOKEN='auth-sync-test-token'
+T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"
+T2_PROCESS_START=unavailable
+mkdir -p "${T2_LOCK_DIR}"
+cat >"${T2_LOCK_DIR}/owner.env" <<EOF
+REPOSITORY=${ROOT}
+PROFILE=fake
+CONTEXT=fake
+TOKEN=${T2_LOCK_TOKEN}
+PID=$$
+PROCESS_START=${T2_PROCESS_START}
+EOF
+export T2_PROJECT_DIR="${ROOT}" T2_PROFILE=fake T2_CONTEXT=fake T2_LOCK_ROOT T2_LOCK_TOKEN
+
cat >"${TMP_DIR}/sleep" <<'SH'
#!/usr/bin/env bash
exit 0
SH
+cat >"${TMP_DIR}/ps" <<'SH'
+#!/usr/bin/env bash
+case " $* " in
+ *' -o state='*) printf 'R\n' ;;
+ *' -o lstart='*) printf 'unavailable\n' ;;
+ *) exit 1 ;;
+esac
+SH
+
cat >"${TMP_DIR}/kubectl" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
@@ -49,7 +76,13 @@ restart_count() {
if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "rpc-proxy-secrets" ]]; then
[[ "${TEST_SOURCE_SECRET_PRESENT:-1}" == "1" ]] || exit 1
- if has_output_flag "$@"; then
+ if [[ "$*" == *'.metadata.resourceVersion'* ]]; then
+ printf '42'
+ elif [[ "$*" == *'-o json' ]]; then
+ source_b64='cHVibGljLWtleQ=='
+ [[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] && source_b64=''
+ printf '{"metadata":{"resourceVersion":"42"},"data":{"RPC_PROXY_JWT_PUBLIC_KEY":"%s"}}' "$source_b64"
+ elif has_output_flag "$@"; then
[[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] || printf 'cHVibGljLWtleQ=='
fi
exit 0
@@ -57,7 +90,9 @@ fi
if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-config" ]]; then
if has_output_flag "$@"; then
- if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ if [[ "$*" == *'-o json' ]]; then
+ printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"CLERUM_AUTH_JWT_PUBLIC_KEY":"old-key"}}'
+ elif [[ "$*" == *auth-key-applied-sha256* ]]; then
printf '%s' "${TEST_MCP_APPLIED_HASH:-old-hash}"
else
printf 'old-key'
@@ -72,7 +107,9 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]
exit 1
fi
if has_output_flag "$@"; then
- if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ if [[ "$*" == *'-o json' ]]; then
+ printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"jwt-public-key":"old-gfs-key"}}'
+ elif [[ "$*" == *auth-key-applied-sha256* ]]; then
printf '%s' "${TEST_GFS_APPLIED_HASH:-old-hash}"
else
printf 'old-gfs-key'
@@ -112,6 +149,7 @@ if [[ "${1:-}" == "get" && "${2:-}" == deployment/* ]]; then
exit 1
fi
fi
+ [[ "$*" == *'.spec.replicas'* ]] && printf '1'
exit 0
fi
@@ -176,7 +214,7 @@ echo "unexpected kubectl invocation: $*" >&2
exit 99
SH
-chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/sleep"
+chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/sleep" "${TMP_DIR}/ps"
if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" PATH="${TMP_DIR}:$PATH" \
bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake >"${OUT_FILE}" 2>&1; then
@@ -311,6 +349,34 @@ if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFI
fi
grep -q 'patch configmap gfs-config' "${LOG_FILE}"
+: >"${LOG_FILE}"
+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: strict GFS auth sync accepted an empty DSN before reconciliation" >&2
+ exit 1
+fi
+grep -q 'gfs-controller-db.connection-string is empty' "${OUT_FILE}"
+
+: >"${LOG_FILE}"
+if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \
+ GFS_AUTH_SYNC_ALLOW_STAGED=true \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ cat "${OUT_FILE}" >&2
+ cat "${LOG_FILE}" >&2
+ echo "FAIL: staged GFS auth sync did not defer proof until reconciliation" >&2
+ exit 1
+fi
+grep -q 'Deferring active gfsc auth proof until GFS credentials are reconciled' "${OUT_FILE}"
+if grep -q 'rollout restart deployment/gfsc-' "${LOG_FILE}"; then
+ echo "FAIL: staged GFS auth sync restarted a consumer before DSN provisioning" >&2
+ cat "${LOG_FILE}" >&2
+ exit 1
+fi
+
: >"${LOG_FILE}"
if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DSN_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 TEST_GFS_STATUS_FAILURE=42 \
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index c09cbeabf..a96e25360 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -35,7 +35,24 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" "$T0" \
"$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"; do
bash -n "$file"
done
-python3 -m py_compile "$GFS_FILTER"
+python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'
+import os
+import py_compile
+import sys
+import tempfile
+
+source = sys.argv[1]
+tmp_root = sys.argv[2]
+fd, compiled = tempfile.mkstemp(prefix="evenfire-filter-gfs-", suffix=".pyc", dir=tmp_root)
+os.close(fd)
+try:
+ py_compile.compile(source, cfile=compiled, doraise=True)
+finally:
+ try:
+ os.unlink(compiled)
+ except FileNotFoundError:
+ pass
+PY
grep -Fq 't0.sh' "$T2"
grep -Fq 'T0_SHELLCHECK=PASS' "$T0"
grep -Fq 'npm run build' "$T0"
@@ -64,15 +81,17 @@ fi
grep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON"
grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON"
grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"
+grep -Fq 'T2_REQUIRED_DEPLOYMENTS' "$COMMON"
grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"
grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"
grep -Fq 'postgres:16-alpine' "$T1"
-grep -Fq 'control-api/test/*realPostgres*.test.ts' "$T1"
+grep -Fq '*realPostgres*.test.ts' "$T1"
+grep -Fq 'is_isolated_control_api_file' "$T1"
grep -Fq 'start_isolated_postgres' "$T1"
grep -Fq 'require_isolated_control_api_files' "$T1"
grep -Fq 'run_suite control-api isolated "$ISOLATED_DSN"' "$T1"
if grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"; then
- echo 'FAIL: control-api Real PostgreSQL suites must not use shared control-postgres' >&2
+ echo 'FAIL: control-api Real PostgreSQL suites still target the shared profile database' >&2
exit 1
fi
if grep -Fq 'run_suite control-api isolated "$ADMIN_DSN"' "$T1"; then
@@ -217,21 +236,23 @@ grep -Fq 'full-bootstrap' "$PREFLIGHT" "$T2"
grep -Fq 'run_pre_gate' "$T2"
grep -Fq 'run_targeted_sync' "$T2"
grep -Fq 'full-reconcile' "$T2"
-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
+grep -Fq 'T2_SKIP_LOCK=true' "$T2"
+grep -Fq 'T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
grep -Fq 'REUSE_DB=true' "$T2"
grep -Fq 'T2_T0_STATUS=NOT_RUN' "$T2"
grep -Fq 'T2_T1_STATUS=NOT_RUN' "$T2"
grep -Fq 'T2_NP08_HCC_AUTHORIZATION_STATUS=NOT_RUN' "$T2"
-grep -Fq "T2_T0_STATUS\" != PASS" "$T2"
-grep -Fq "T2_T1_STATUS\" != PASS" "$T2"
+grep -Fq 't2_lane_completed "$T2_T0_STATUS"' "$T2"
+grep -Fq 't2_lane_completed "$T2_T1_STATUS"' "$T2"
grep -Fq 'T2_HEALTHCHECK_COMMAND' "$T2"
grep -Fq 'run_np08_hcc_authorization' "$T2"
grep -Fq "CLERUM_PROFILE_PORTS_ENV=\"\$T2_PORTS_ENV\"" "$T2"
grep -Fq 'NP08_HCC_AUTHORIZATION PASS' "$T2"
grep -Fq "NP08_HCC_AUTHORIZATION=\$T2_NP08_HCC_AUTHORIZATION_STATUS" "$T2"
grep -Fq 'already-synced' "$T2" "$COMMON"
-grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
-grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"
+grep -Fq 'T2_LOCK_TOKEN="$T2_LOCK_TOKEN"' "$T2"
+grep -Fq 'T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
+grep -Fq 'T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"
grep -Fq 'T2_PLAN_MODE=false' "$PREFLIGHT"
grep -Fq -- '--skip-port-forwards' "$T2"
grep -Fq 't2_lane_completed' "$T2"
@@ -318,6 +339,13 @@ bash -c '
if T2_SKIP_LOCK=true T2_LOCK_TOKEN=wrong bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; then
exit 1
fi
+ saved_token="$T2_LOCK_TOKEN"
+ T2_LOCK_TOKEN=wrong
+ if t2_lock_validate_inherited; then
+ echo "inherited lock validation returned success after a guard fired" >&2
+ exit 1
+ fi
+ T2_LOCK_TOKEN="$saved_token"
t2_lock_release 0
' bash "$COMMON"
if [ -e "$tmp/fenced-locks/clerum-fence.lock" ]; then
diff --git a/scripts/tests/test-minikube-t2-public-boundary.sh b/scripts/tests/test-minikube-t2-public-boundary.sh
index d3e2d631a..78ceef2a1 100755
--- a/scripts/tests/test-minikube-t2-public-boundary.sh
+++ b/scripts/tests/test-minikube-t2-public-boundary.sh
@@ -29,6 +29,12 @@ git -C "$ROOT" ls-files --others --exclude-standard -z |
while IFS= read -r -d '' path; do
[ -f "$ROOT/$path" ] || continue
printf '+++ b/%s\n' "$path"
+ # A repository-local binary cannot be made line-safe by BSD sed. Keep the
+ # path header (so sensitive filenames are still rejected), but skip binary
+ # contents instead of allowing the scanner itself to fail by locale.
+ if ! LC_ALL=C grep -Iq . "$ROOT/$path"; then
+ continue
+ fi
sed 's/^/+/' "$ROOT/$path"
done >>"$tmp"
@@ -91,7 +97,12 @@ for line in diff.splitlines():
match = re.search(expression, value)
if match and reason == "private key" and "evidence-scanner" in current:
continue
- if match and reason == "credential assignment" and match.group(1) and safe_fixture.search(match.group(1)):
+ if (
+ match
+ and reason == "credential assignment"
+ and match.group(1)
+ and (safe_fixture.search(match.group(1)) or "$" in match.group(1))
+ ):
continue
if match:
bad.append((current or "<unknown>", reason))
diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.sh
index 568eb76bd..a75dc5be5 100755
--- a/scripts/tests/test-minikube-t2-scenarios.sh
+++ b/scripts/tests/test-minikube-t2-scenarios.sh
@@ -65,6 +65,7 @@ repo_env=(
T2_PROFILE_ROOT="$tmp/profiles"
T2_PROFILE_ENV="$profile_root/profile.env"
T2_PORTS_ENV="$profile_root/ports.env"
+ T2_REQUIRED_DEPLOYMENTS=gfs/gfsc-reader
T2_BRANCH=feat/scenario
T2_HEAD="$feature_sha"
T2_LOCK_ROOT="$tmp/locks"
@@ -119,6 +120,20 @@ env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
+expect_code DEVELOPMENT_SCOPE_REQUIRED public-endpoint public-endpoint \
+ env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=8.8.8.8 \
+ bash -c 'source "$1"; t2_context_check' bash "$COMMON"
+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=192.168.1.99 \
+ bash -c 'source "$1"; t2_context_check' bash "$COMMON"
+
+identity_json='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"PROFILE"}},"status":{"addresses":[{"type":"InternalIP","address":"172.17.0.3"}]}}]}'
+identity_ok="$(env "${repo_env[@]}" IDENTITY_JSON="$identity_json" \
+ bash -c 'source "$1"; T2_BOOTSTRAP_REQUIRED=false; t2_mk(){ printf 172.17.0.3; }; t2_kc(){ printf "%s" "$IDENTITY_JSON" | sed "s/PROFILE/$T2_PROFILE/g"; }; t2_profile_context_identity_check; printf "%s" "$T2_CONTEXT_IDENTITY_VERIFIED"' bash "$COMMON")"
+[ "$identity_ok" = true ] || fail "exact Minikube profile/node identity did not verify"
+identity_wrong='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}},"status":{"addresses":[{"type":"InternalIP","address":"172.17.0.3"}]}}]}'
+expect_code DEVELOPMENT_SCOPE_REQUIRED wrong-profile-identity wrong-profile-identity \
+ env "${repo_env[@]}" IDENTITY_JSON="$identity_wrong" \
+ bash -c 'source "$1"; T2_BOOTSTRAP_REQUIRED=false; t2_mk(){ printf 172.17.0.3; }; t2_kc(){ printf "%s" "$IDENTITY_JSON"; }; t2_profile_context_identity_check' bash "$COMMON"
missing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"
[ "$missing_profile_state" = full-bootstrap ] || fail "missing profile selected $missing_profile_state instead of full-bootstrap"
@@ -255,6 +270,14 @@ case "$planner_unready" in
*) fail 'planner full-reconcile reason does not name the unready deployment' ;;
esac
+ready_json='{"items":[{"metadata":{"namespace":"gfs","name":"gfsc-reader","generation":7},"spec":{"replicas":1},"status":{"observedGeneration":7,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}]}'
+env "${repo_env[@]}" READY_JSON="$ready_json" \
+ bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$READY_JSON"; }; t2_deployment_check' bash "$COMMON"
+stale_generation_json='{"items":[{"metadata":{"namespace":"gfs","name":"gfsc-reader","generation":8},"spec":{"replicas":1},"status":{"observedGeneration":7,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}]}'
+expect_code PROFILE_UNHEALTHY stale-generation stale-generation \
+ env "${repo_env[@]}" INVALID_INVENTORY="$stale_generation_json" \
+ bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"
+
# Negative pin: the standalone preflight and the final exact-head check
# (T2_PLAN_MODE=false) stay fail-loud on an unready deployment.
expect_code PROFILE_UNHEALTHY unready-final-preflight unready-final-preflight \
@@ -404,6 +427,11 @@ if ! run_process_check "$tmp/ps-ef-wrap.out" "$tmp/pf-wrap.err"; then
fail 't2_process_check failed on a wrapper that only mentions kubectl (comm=bash)'
fi
+printf '%s\n' "$pf_ctl" >"$tmp/ps-ef-contextless.out"
+if ! run_process_check "$tmp/ps-ef-contextless.out" "$tmp/pf-contextless.err"; then
+ fail 't2_process_check rejected an unrelated context-less port-forward'
+fi
+
grep -Fq 'REUSE_DB=true' "$ROOT/scripts/minikube/t2.sh"
grep -Fq 'CONTROL_DB_RESET_PVC_UID' "$ROOT/scripts/minikube/t2.sh"
@@ -414,7 +442,7 @@ cat >"$forward_tmp/bin/ps" <<'EOF'
printf '%s\n' 'root 4242 1 0 00:00 ? 00:00:00 kubectl --context=profile-a -n control-plane port-forward svc/control-api 30100:8090'
EOF
chmod +x "$forward_tmp/bin/ps"
-printf '4242\n' >"/tmp/pf-profile-a-control-api.pid"
+printf '4242\nPROCESS_START=unavailable\n' >"/tmp/pf-profile-a-control-api.pid"
forward_check="$(env PATH="$forward_tmp/bin:$PATH" T2_PROFILE=profile-a T2_CONTEXT=profile-a \
T2_PROFILE_ROOT="$forward_tmp/profile-cache" bash -c 'source "$1"; t2_process_check; printf PASS' bash "$COMMON")"
rm -f "/tmp/pf-profile-a-control-api.pid"