Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: harden PR 416 rollout and validation gates
Alfredo Lopez1bc24dccbdeb
diff --git a/Makefile b/Makefileindex c725d91df..51aec064c 100644--- a/Makefile+++ b/Makefile@@ -155,12 +155,17 @@ minikube-stop: ## Stop minikube cluster# pulls ~25 published images instead of building 28 from source. Build# everything locally with `make minikube-setup-local` (or IMAGE_SOURCE=local).IMAGE_SOURCE ?= ghcr-MINIKUBE_GFS_MUTATION ?= true+# GFS Secret/role mutation is owned by the canonical T2 lease. Callers that+# intentionally run the full T2 transition set this to true and pass the+# inherited opaque T2_LOCK_TOKEN; ordinary deploys render/filter GFS resources.+MINIKUBE_GFS_MUTATION ?= false.PHONY: minikube-setupminikube-setup: ## Clean install from scratch, PULLING published images (IMAGE_SOURCE=local or `make minikube-setup-local` builds instead). Rebuilds the DB; REUSE_DB=true keeps it. SKIP_UIS=true omits Control/Profile UI. Runs 'prereqs' first (SKIP_PREREQS=true to bypass). Needs ADMIN_PASSWORD in .env.@if [ "$(SKIP_PREREQS)" != "true" ]; then $(MAKE) --no-print-directory prereqs; fi- @MINIKUBE_SKIP_UIS="$(SKIP_UIS)" MINIKUBE_SEED_PROFILE="$(SEED_PROFILE)" REUSE_DB="$(REUSE_DB)" \+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ MINIKUBE_SKIP_UIS="$(SKIP_UIS)" MINIKUBE_SEED_PROFILE="$(SEED_PROFILE)" REUSE_DB="$(REUSE_DB)" \IMAGE_SOURCE="$(IMAGE_SOURCE)" MINIKUBE_IMAGE_TAG="$(MINIKUBE_IMAGE_TAG)" \scripts/minikube/full-setup.sh $(ARGS)@@ -224,6 +229,9 @@ minikube-detect-k8s-api-ip: ## Patch overlays/minikube/patches/k8s-api-ip.yaml w.PHONY: minikube-deploy-allminikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay@$(MAKE) --no-print-directory minikube-detect-k8s-api-ip+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ] && [ -z "$(T2_LOCK_TOKEN)" ]; then \+ echo "[minikube-deploy-all] refusing GFS mutation without the owning T2 profile lock" >&2; exit 1; \+ fi@# Upgrade path: adopt/validate writer and stage reader before HCC cutover.@if [ "$(MINIKUBE_GFS_MUTATION)" != "true" ]; then echo "[minikube-deploy-all] GFS mutation disabled for this non-T2 sync"; fi@if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \@@ -235,7 +243,7 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlaykubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \}; \- CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \else \echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \fi; \@@ -286,7 +294,7 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlayif [ -n "$$gfs_config_probe" ]; then \if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/wait-gfsc-secret-references.sh; \- CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \else \echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \fi; \@@ -316,8 +324,17 @@ minikube-restart-all: ## Restart all Clerum deployments.PHONY: minikube-deploy-crdsminikube-deploy-crds: ## Install/upgrade CRDs via Helm chart + apply CRD YAML (idempotent)kubectl --context=$(MINIKUBE_PROFILE) apply -f deploy/base/namespaces.yaml- helm upgrade --install --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds- kubectl --context=$(MINIKUBE_PROFILE) apply -f ./charts/clerum-crds/crds/+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "false" ]; then \+ echo "[minikube-deploy-crds] GFS CRD mutation disabled for this non-T2 gate"; \+ helm upgrade --install --skip-crds --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds; \+ for crd in ./charts/clerum-crds/crds/*.yaml; do \+ case "$$crd" in *globalfilesystem.yaml) continue ;; esac; \+ kubectl --context=$(MINIKUBE_PROFILE) apply -f "$$crd"; \+ done; \+ else \+ helm upgrade --install --kube-context=$(MINIKUBE_PROFILE) clerum-crds ./charts/clerum-crds; \+ kubectl --context=$(MINIKUBE_PROFILE) apply -f ./charts/clerum-crds/crds/; \+ fi.PHONY: minikube-deploy-serviceminikube-deploy-service: ## Rebuild single image + rollout restart deployment (usage: make minikube-deploy-service SVC=mcp-host NS=mcp-host [DEPLOYMENT=chatllm])@@ -388,14 +405,20 @@ minikube-apply-namespaces: ## Create all namespaces.PHONY: minikube-sync-auth-keyminikube-sync-auth-key: ## Sync JWT public key from rpc-proxy-secrets into runtime ConfigMaps when drift exists+ @if [ -z "$(T2_LOCK_TOKEN)" ]; then \+ echo "[minikube-sync-auth-key] refusing ConfigMap mutation without the owning T2 profile lock" >&2; exit 1; \+ fi@if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE); \else \+ T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE) --skip-gfs --require-mcp; \fi.PHONY: minikube-sync-auth-key-if-presentminikube-sync-auth-key-if-present: ## Sync JWT public key only when minikube auth resources already exist+ @if [ -z "$(T2_LOCK_TOKEN)" ]; then echo "Skipping auth key sync (canonical T2 profile lock is not held)."; exit 0; fi@if ! $(KC) get secret rpc-proxy-secrets -n rpc-proxy >/dev/null 2>&1; then \echo "Skipping auth key sync (rpc-proxy-secrets not found yet)."; \elif ! $(KC) get configmap mcp-host-config -n mcp-host >/dev/null 2>&1; then \diff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.shindex 76f6cd279..cab7d63a6 100755--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh@@ -6,12 +6,13 @@ CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"GFS_NS="${GFS_NS:-gfs}"set +u-T2_PROJECT_DIR="$T2_PROJECT_DIR"-T2_PROFILE="$T2_PROFILE"-T2_CONTEXT="$T2_CONTEXT"-MINIKUBE_PROFILE="$MINIKUBE_PROFILE"-CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"-T2_LOCK_TOKEN="$T2_LOCK_TOKEN"+T2_PROJECT_DIR="${T2_PROJECT_DIR:-}"+T2_PROFILE="${T2_PROFILE:-}"+T2_CONTEXT="${T2_CONTEXT:-}"+MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-}"+CONTROL_API_REAL_PG_CONTEXT="${CONTROL_API_REAL_PG_CONTEXT:-}"+T2_LOCK_TOKEN="${T2_LOCK_TOKEN:-}"+T2_SKIP_LOCK="${T2_SKIP_LOCK:-}"set -uif [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fiif [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi@@ -26,14 +27,30 @@ remote_context_allowed() {[ -n "${ALLOWED_CONTEXTS:-}" ] || return 1IFS=',' read -r -a allowed_contexts <<<"${ALLOWED_CONTEXTS}"for allowed_context in "${allowed_contexts[@]}"; do- if [ "$allowed_context" = "$CONTEXT" ]; then- return 0- fi+ [ "$allowed_context" = "$CONTEXT" ] && return 0donereturn 1}-if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; then+minikube_profile_exists() {+ local status+ status="$(minikube -p "$CONTEXT" status 2>/dev/null || true)"+ [ -n "$status" ] &&+ [[ "$status" != *"does not exist"* && "$status" != *"not found"* && "$status" != *Nonexistent* ]]+}++# The sibling infra deploy path owns its explicit non-local context decision.+# A local Minikube profile, even if a caller supplies the legacy remote flag,+# must still use the branch-owned T2 lease below.+REMOTE_RECONCILE=false+if remote_context_allowed && ! minikube_profile_exists; then+ REMOTE_RECONCILE=true+fi++if [ "$REMOTE_RECONCILE" != true ]; then+ # Every local invocation is a profile-owned mutating boundary. A caller may+ # either inherit the opaque lease from T2 or acquire one here; context names+ # and caller-provided allowlists are not local authorization.# shellcheck source=scripts/minikube/t2-common.shsource "$ROOT/scripts/minikube/t2-common.sh"if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi@@ -69,23 +86,19 @@ if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; thentrap 'handle_reconcile_signal TERM' TERMt2_repo_metadatat2_profile_scope+ t2_profile_status+ if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then+ T2_NEXT_COMMAND="start and bootstrap the branch-owned profile, then retry GFS reconciliation"+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "branch-owned Minikube profile is not bootstrapped: $T2_PROFILE"+ fit2_context_check+ t2_profile_context_identity_checkt2_mutation_lock- # The verified profile context is authoritative. Do not let a caller keep- # a second CONTEXT value after the fence has been acquired.+ # The verified profile context is authoritative. Do not let a caller keep a+ # second CONTEXT value after the fence has been acquired.CONTEXT="$T2_CONTEXT"else- # Remote reconciliation is authorized by the explicit workflow decision and- # exact context membership, never by a provider/name convention. This also- # supports self-hosted dev contexts whose names are not gke_*.- if ! remote_context_allowed; then- if [ -z "${GFS_REMOTE_RECONCILE_AUTHORIZED:-}" ] && [ -z "${ALLOWED_CONTEXTS:-}" ]; then- printf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2- exit 1- fi- printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2- exit 1- fi+ printf '[reconcile-gfs-deploy] using the explicitly authorized non-local context: %s\n' "$CONTEXT" >&2fiCONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.shindex 0a17bd1eb..414d48ab8 100755--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh@@ -216,15 +216,27 @@ EOF}delete_synthetic_fleet() {- local failed=0 kind+ local failed=0 kind host_query host+ if ! host_query="$(kctl get host -A \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \+ -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)"; then+ printf 'HCC cleanup: unable to enumerate synthetic Host names for run %s; refusing broad child deletion\n' "$RUN_ID" >&2+ return 1+ fifor kind in mcpserver context host; do- kctl delete "$kind" -A -l e2e.clerum.io/suite=hcc-watch-churn \+ kctl delete "$kind" -A \+ -l "e2e.clerum.io/suite=hcc-watch-churn,e2e.clerum.io/run=${RUN_ID}" \--ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 || failed=1donekctl delete secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found >/dev/null 2>&1 || failed=1- # HCC-managed runtime left behind by the fleet Hosts.- kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \- -A -l "clerum.io/managed-by=host-context-controller" --ignore-not-found >/dev/null 2>&1 || true+ # HCC-managed runtime left behind by these exact synthetic Hosts. Never use+ # the cluster-wide managed-by label: legitimate platform Hosts can own+ # resources with that label while this cleanup is running.+ while IFS= read -r host; do+ [ -n "$host" ] || continue+ kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \+ -A -l "clerum.io/host=${host}" --ignore-not-found >/dev/null 2>&1 || failed=1+ done <<<"$host_query"return "$failed"}diff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.shindex aa1b550f7..a4dc4fbc8 100755--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh@@ -244,9 +244,18 @@ assert_botched_outage() {elsefail "could not list HCC pod UIDs — refusing to treat a query failure as old-pod termination"fi+ deployment_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.metadata.deletionTimestamp}{"\t"}{.metadata.ownerReferences[?(@.kind=="ReplicaSet")].name}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].image}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"+ replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" -v image="$BOTCHED_IMAGE" \+ '$2 != old && $2 != "" && $4 == "" && $3 != "Failed" && $3 != "Succeeded" && $5 != "" && $6 == image && $7 ~ /ErrImagePull|ImagePullBackOff/ { print; exit }' <<<"$stuck_pod_rows")"+ replacement_owner="$(cut -f5 <<<"$replacement_row")"+ replacement_owner_revision=""+ if [ -n "$replacement_owner" ]; then+ replacement_owner_revision="$(kctl get rs "$replacement_owner" -n "$HCC_NS" \+ -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"+ ficase "$flavor" inrecovery)@@ -278,7 +287,11 @@ assert_botched_outage() {fail "$image_fail"[ "$old_uid_live" = 0 ] && ok "$old_pod_ok" || fail "$old_pod_fail"if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"+ if [ -n "$deployment_revision" ] && [ "$replacement_owner_revision" = "$deployment_revision" ]; then+ ok "replacement pod is a live current-revision pod pinned by the injected image failure: ${replacement_row}"+ else+ fail "replacement pod owner revision ${replacement_owner_revision:-unset} does not match Deployment revision ${deployment_revision:-unset}"+ fielsefail "$replacement_fail"fidiff --git a/scripts/e2e/gfs-real-pg-minikube-gate.sh b/scripts/e2e/gfs-real-pg-minikube-gate.shindex e743a3afa..73e147b90 100755--- a/scripts/e2e/gfs-real-pg-minikube-gate.sh+++ b/scripts/e2e/gfs-real-pg-minikube-gate.sh@@ -1,10 +1,11 @@#!/usr/bin/env bash-# Run the real-Postgres T1 suites against the control-postgres instance in an-# explicitly validated, branch-owned Minikube profile.+# Run the GFS real-Postgres T1 suites with an isolated PostgreSQL instance while+# validating the target branch-owned Minikube profile and restoring its GFS+# runtime roles on exit.## This runner is deliberately separate from CI: CI supplies its own ephemeral-# Postgres DSN, while this gate obtains a short-lived localhost port-forward to-# the selected Minikube service. It never discovers or accepts a GKE context.+# Postgres DSN, while this gate starts a disposable local PostgreSQL container.+# It never mutates the shared control-postgres database used by the profile.set -euo pipefailset +x@@ -13,10 +14,26 @@ PROJECT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"CONTEXT="${CONTEXT:-${MINIKUBE_PROFILE:-}}"SYNC_CONFIGMAP="${CLERUM_PRE_GATE_SYNC_CONFIGMAP:-clerum-pre-gate-sync-state}"CONTROL_NS="${CONTROL_NS:-control-plane}"-PG_SERVICE="${PG_SERVICE:-control-postgres}"-PG_SECRET="${PG_SECRET:-control-postgres}"TIMEOUT="${TIMEOUT:-120}"-PORTS_ENV="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${CONTEXT}/ports.env}"+ISOLATED_PG_IMAGE="${ISOLATED_PG_IMAGE:-postgres:16-alpine}"+ISOLATED_HOST='127.0.0.1'++# Reuse the canonical T2 ownership, context-identity, marker, and mutation+# lock contract. This standalone GFS lane is still a T1 helper, but it mutates+# cluster-global GFS roles and therefore must have the same branch-owned safety+# boundary as the main T2 orchestrator.+T2_PROJECT_DIR="${PROJECT_DIR}"+MINIKUBE_PROFILE="${CONTEXT}"+T2_PROFILE="${CONTEXT}"+T2_CONTEXT="${CONTEXT}"+T2_PROFILE_ROOT="${T2_PROFILE_ROOT:-${CLERUM_PROFILE_CACHE_ROOT:-${HOME}/.cache/clerum/minikube-profiles}}"+T2_PROFILE_ENV="${T2_PROFILE_ENV:-${T2_PROFILE_ROOT}/${CONTEXT}/profile.env}"+T2_PORTS_ENV="${T2_PORTS_ENV:-${CLERUM_PROFILE_PORTS_ENV:-${T2_PROFILE_ROOT}/${CONTEXT}/ports.env}}"+T2_MARKER_NAME="${SYNC_CONFIGMAP}"+T2_CONTROL_NAMESPACE="${CONTROL_NS}"+export T2_PROJECT_DIR MINIKUBE_PROFILE T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT \+ T2_PROFILE_ENV T2_PORTS_ENV T2_MARKER_NAME T2_CONTROL_NAMESPACE+source "${PROJECT_DIR}/scripts/minikube/t2-common.sh"die() {printf '[gfs-real-pg-minikube] ERROR: %s\n' "$*" >&2@@ -34,68 +51,21 @@ require_command() {done}-is_branch_profile() {- [[ "${1}" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]-}--reject_unsafe_context() {- [[ -n "${CONTEXT}" ]] || die 'CONTEXT or MINIKUBE_PROFILE is required'- case "${CONTEXT}" in- gke_*) die "refusing non-Minikube or production context: ${CONTEXT}" ;;- *prod*) die "refusing non-Minikube or production context: ${CONTEXT}" ;;- esac- if [[ "${CONTEXT}" != "clerum-test" ]] && ! is_branch_profile "${CONTEXT}"; then- die "context is not an allowed local/branch profile: ${CONTEXT}"+verify_branch_gate() {+ t2_require_commands+ t2_repo_metadata+ t2_profile_scope+ # Profile status is checked before context identity so a missing/stopped+ # profile produces the supported bootstrap transition instead of an opaque+ # kube-context failure.+ t2_profile_status+ if [ "${T2_BOOTSTRAP_REQUIRED}" = true ]; then+ die "branch-owned Minikube profile is not bootstrapped: ${T2_PROFILE}"fi-}--verify_profile_ownership() {- if ! is_branch_profile "${CONTEXT}"; then- return 0- fi- [[ -f "${PORTS_ENV}" ]] || die "branch profile ports.env is missing: ${PORTS_ENV}"-- local profile_name profile_repo profile_dir- profile_dir="${PORTS_ENV%/ports.env}"- profile_name="$(awk -F= '$1 == "PROFILE" { print substr($0, index($0, "=") + 1); exit }' "${profile_dir}/profile.env" 2>/dev/null || true)"- profile_repo="$(awk -F= '$1 == "REPO_DIR" { print substr($0, index($0, "=") + 1); exit }' "${profile_dir}/profile.env" 2>/dev/null || true)"- [[ "${profile_name}" == "${CONTEXT}" ]] || die "ports.env/profile marker belongs to '${profile_name:-unknown}', not ${CONTEXT}"- [[ -n "${profile_repo}" ]] || die 'profile marker has no REPO_DIR'- [[ "$(cd -- "${profile_repo}" 2>/dev/null && pwd -P)" == "${PROJECT_DIR}" ]] || \- die "profile marker is owned by another worktree: ${profile_repo}"-}--verify_clean_and_sync_marker() {- local head worktree_id marker_json- [[ -z "$(git -C "${PROJECT_DIR}" status --porcelain)" ]] || \- die 'worktree is dirty; run pre-gate-sync after committing or cleanly restoring changes'- head="$(git -C "${PROJECT_DIR}" rev-parse --verify HEAD)"- worktree_id="$(printf '%s' "${PROJECT_DIR}" | shasum | awk '{print $1}')"- marker_json="$(kc -n "${CONTROL_NS}" get configmap "${SYNC_CONFIGMAP}" -o json 2>/dev/null)" || \- die "validated pre-gate marker is missing: ${CONTROL_NS}/${SYNC_CONFIGMAP}"-- python3 - "${worktree_id}" "${head}" "${marker_json}" <<'PY'-import json-import sys--expected_worktree, expected_head, marker_json = sys.argv[1:]-payload = json.loads(marker_json)-data = payload.get("data") or {}-if not data.get("clusterFingerprint"):- raise SystemExit("pre-gate marker has no cluster fingerprint")-if data.get("worktreeId") != expected_worktree:- raise SystemExit("pre-gate marker belongs to another worktree")-if data.get("gitHead") != expected_head:- raise SystemExit("pre-gate marker does not match the current HEAD")-PY-}--secret_value() {- local key="$1" encoded- encoded="$(kc -n "${CONTROL_NS}" get secret "${PG_SECRET}" -o "jsonpath={.data.${key}}" 2>/dev/null)" || \- die "cannot read ${CONTROL_NS}/${PG_SECRET}.${key}"- [[ -n "${encoded}" ]] || die "${CONTROL_NS}/${PG_SECRET}.${key} is empty"- printf '%s' "${encoded}" | python3 -c 'import base64,sys; print(base64.b64decode(sys.stdin.read(), validate=True).decode(), end="")'+ t2_context_check+ t2_profile_context_identity_check+ t2_mutation_lock+ t2_marker_check}wait_for_tcp() {@@ -131,6 +101,64 @@ path.write_text(text)PY}+PROFILE_PG_FORWARD_PID=""++stop_profile_postgres_forward() {+ [[ -n "${PROFILE_PG_FORWARD_PID}" ]] || return 0+ if ! kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ PROFILE_PG_FORWARD_PID=''+ return 0+ fi+ local command_line+ command_line="$(ps -p "${PROFILE_PG_FORWARD_PID}" -o command= 2>/dev/null || true)"+ if [[ "${command_line}" != *port-forward* || "${command_line}" != *svc/control-postgres* ]]; then+ printf '[gfs-real-pg-minikube] ERROR: profile PostgreSQL forward PID is not the expected child; refusing cleanup success\n' >&2+ return 1+ fi+ if ! kill "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1 && kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ return 1+ fi+ if kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ if wait "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ :+ fi+ fi+ if kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ return 1+ fi+ PROFILE_PG_FORWARD_PID=''+}++verify_profile_postgres() {+ kc -n "${CONTROL_NS}" rollout status deployment/control-postgres --timeout="${TIMEOUT}s" >/dev/null 2>&1 || \+ die 'branch-profile control-postgres did not become Ready'++ local key encoded profile_port+ for key in POSTGRES_USER POSTGRES_PASSWORD POSTGRES_DB; do+ encoded="$(kc -n "${CONTROL_NS}" get secret control-postgres -o "jsonpath={.data.${key}}" 2>/dev/null || true)"+ [[ -n "${encoded}" ]] || die "branch-profile control-postgres Secret is missing ${key}"+ printf '%s' "${encoded}" | python3 -c 'import base64,sys; base64.b64decode(sys.stdin.buffer.read(), validate=True)' >/dev/null 2>&1 || \+ die "branch-profile control-postgres Secret key is not valid base64: ${key}"+ done++ profile_port="$(choose_local_port)"+ kubectl --context="${CONTEXT}" -n "${CONTROL_NS}" port-forward \+ --address=127.0.0.1 svc/control-postgres "${profile_port}:5432" \+ >"${TMP_DIR}/profile-postgres-port-forward.log" 2>&1 &+ PROFILE_PG_FORWARD_PID=$!+ if ! wait_for_tcp "${profile_port}"; then+ sanitize_file "${TMP_DIR}/profile-postgres-port-forward.log"+ cat "${TMP_DIR}/profile-postgres-port-forward.log" >&2 || true+ die 'branch-profile control-postgres port-forward did not become reachable'+ fi+ if ! kill -0 "${PROFILE_PG_FORWARD_PID}" >/dev/null 2>&1; then+ sanitize_file "${TMP_DIR}/profile-postgres-port-forward.log"+ cat "${TMP_DIR}/profile-postgres-port-forward.log" >&2 || true+ die 'branch-profile control-postgres port-forward exited before the GFS lane started'+ fi+ stop_profile_postgres_forward || die 'could not cleanly close the branch-profile PostgreSQL precondition forward'+}+real_postgres_suite_files() {local package="$1"case "${package}" in@@ -218,22 +246,11 @@ PYprintf '[gfs-real-pg-minikube] PASS %s (%s suites, no skips)\n' "${package}" "${suite_count}"}-require_command git kubectl npm python3 shasum-reject_unsafe_context-verify_profile_ownership-verify_clean_and_sync_marker-kc -n "${CONTROL_NS}" get service "${PG_SERVICE}" >/dev/null || die "${CONTROL_NS}/${PG_SERVICE} service is missing"-kc -n "${CONTROL_NS}" rollout status deployment/control-postgres --timeout="${TIMEOUT}s" >/dev/null || \- die 'control-postgres is not Ready in the validated profile'--PG_USER="$(secret_value POSTGRES_USER)"-PG_PASSWORD="$(secret_value POSTGRES_PASSWORD)"-PG_DATABASE="$(secret_value POSTGRES_DB)"-[[ -n "${PG_USER}" && -n "${PG_PASSWORD}" && -n "${PG_DATABASE}" ]] || die 'control-postgres secret is incomplete'-TMP_DIR="$(mktemp -d)"-PORT_FORWARD_PID=''-LOCAL_PORT=''+ISOLATED_CONTAINER=""+ISOLATED_PORT=''+ADMIN_DSN=''+GFS_RESTORE_REQUIRED=falserestore_gfs_runtime_credentials() {# The reader-role real-Postgres suite exercises the production role names,# which are cluster-global even though its fixture database is temporary.@@ -243,34 +260,32 @@ restore_gfs_runtime_credentials() {printf '[gfs-real-pg-minikube] ERROR: required gfs-controller-db Secret is missing or unreadable; refusing to finish with GFS credentials unreconciled\n' >&2return 1fi- if ! GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="${CONTEXT}" \+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" \+ T2_PROJECT_DIR="${T2_PROJECT_DIR}" T2_PROFILE="${T2_PROFILE}" T2_CONTEXT="${T2_CONTEXT}" \+ T2_PROFILE_ROOT="${T2_PROFILE_ROOT}" T2_PROFILE_ENV="${T2_PROFILE_ENV}" T2_PORTS_ENV="${T2_PORTS_ENV}" \+ GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="${CONTEXT}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; thenprintf '[gfs-real-pg-minikube] ERROR: failed to restore branch-profile GFS credentials\n' >&2return 1fi}-stop_port_forward() {- if [[ -z "${PORT_FORWARD_PID}" ]] || ! kill -0 "${PORT_FORWARD_PID}" >/dev/null 2>&1; then- PORT_FORWARD_PID=''+stop_isolated_postgres() {+ [[ -n "${ISOLATED_CONTAINER}" ]] || return 0+ if docker rm -f "${ISOLATED_CONTAINER}" >/dev/null 2>&1; then+ ISOLATED_CONTAINER=''return 0fi- local command_line- command_line="$(ps -p "${PORT_FORWARD_PID}" -o command= 2>/dev/null || true)"- if [[ "${command_line}" == *'port-forward'* &&- "${command_line}" == *"svc/${PG_SERVICE}"* &&- "${command_line}" == *"${LOCAL_PORT}:5432"* ]]; then- kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true- wait "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true- fi- PORT_FORWARD_PID=''+ return 1}cleanup() {local status=$?- stop_port_forward- if ! restore_gfs_runtime_credentials; then- status=1- fi- rm -rf "${TMP_DIR}"+ trap - EXIT+ trap '' INT TERM+ if ! stop_profile_postgres_forward; then status=1; fi+ if ! stop_isolated_postgres; then status=1; fi+ if [ "${GFS_RESTORE_REQUIRED}" = true ] && ! restore_gfs_runtime_credentials; then status=1; fi+ if ! rm -rf "${TMP_DIR}"; then status=1; fi+ if ! t2_lock_release "${status}"; then status=1; fiexit "${status}"}trap cleanup EXIT@@ -286,39 +301,32 @@ sock.close()PY}-port_forward_ready=false-for _attempt in 1 2 3 4 5; do- LOCAL_PORT="$(choose_local_port)"- kubectl --context="${CONTEXT}" -n "${CONTROL_NS}" port-forward \- --address=127.0.0.1 "svc/${PG_SERVICE}" "${LOCAL_PORT}:5432" \- >"${TMP_DIR}/port-forward.log" 2>&1 &- PORT_FORWARD_PID=$!- if wait_for_tcp "${LOCAL_PORT}"; then- port_forward_ready=true+require_command docker npm+docker info >/dev/null 2>&1 || die 'Docker is required for the isolated GFS real-Postgres lane'+verify_branch_gate+verify_profile_postgres++ISOLATED_PORT="$(choose_local_port)"+ISOLATED_CONTAINER="evenfire-gfs-t1-pg-$$"+docker run -d --rm --name "${ISOLATED_CONTAINER}" \+ -e POSTGRES_USER=postgres \+ -e POSTGRES_DB=postgres \+ -e POSTGRES_HOST_AUTH_METHOD=trust \+ -p "127.0.0.1:${ISOLATED_PORT}:5432" \+ "${ISOLATED_PG_IMAGE}" >/dev/null || die 'isolated GFS PostgreSQL container failed to start'+wait_for_tcp "${ISOLATED_PORT}" || die 'isolated GFS PostgreSQL did not become reachable'+deadline=$((SECONDS + TIMEOUT))+while (( SECONDS < deadline )); do+ if docker exec "${ISOLATED_CONTAINER}" pg_isready -U postgres >/dev/null 2>&1; thenbreakfi- stop_port_forward+ sleep 1done-[[ "${port_forward_ready}" == true ]] || {- cat "${TMP_DIR}/port-forward.log" >&2 || true- die 'Minikube PostgreSQL port-forward did not become reachable'-}--ADMIN_DSN="$(printf '%s\0%s\0%s' "${PG_USER}" "${PG_PASSWORD}" "${LOCAL_PORT}" | python3 -c '-from urllib.parse import quote-import sys--user, password, port = sys.stdin.buffer.read().split(b"\0")[:3]-user = user.decode()-password = password.decode()-port = port.decode()-print("postgresql://{}:{}@127.0.0.1:{}/postgres".format(- quote(user, safe=""), quote(password, safe=""), port-), end="")-')"-unset PG_USER PG_PASSWORD PG_DATABASE-[[ -n "${ADMIN_DSN}" ]] || die 'constructed Minikube PostgreSQL admin DSN is empty'+docker exec "${ISOLATED_CONTAINER}" pg_isready -U postgres >/dev/null 2>&1 ||+ die 'isolated GFS PostgreSQL did not become ready'+ADMIN_DSN="$(printf 'postgresql://postgres@%s:%s/postgres' "${ISOLATED_HOST}" "${ISOLATED_PORT}")"+GFS_RESTORE_REQUIRED=truerun_suite control-apirun_suite gfs-controllerprintf '[gfs-real-pg-minikube] T1 PASS: real PostgreSQL executed against validated Minikube profile %s\n' "${CONTEXT}"diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.shindex fb37ef25e..4d980dd6f 100755--- a/scripts/e2e/minikube-real-postgres.sh+++ b/scripts/e2e/minikube-real-postgres.sh@@ -11,8 +11,10 @@ source "$PROJECT_DIR/scripts/minikube/t2-common.sh"PG_NAMESPACE="$T2_CONTROL_NAMESPACE"PG_SERVICE=control-postgresPG_SECRET=control-postgres-# Suites that DROP/ALTER cluster-global roles cannot share live control-postgres-# (#412). They run against a throwaway Postgres 16 that matches CI.+# Only suites that DROP/ALTER cluster-global roles cannot share live+# control-postgres (#412). They run against a throwaway PostgreSQL 16 that+# matches CI; the remaining control-api suites must exercise the validated+# branch profile database.T1_ISOLATED_PG_IMAGE="${T1_ISOLATED_PG_IMAGE:-postgres:16-alpine}"T1_TIMEOUT="$T2_TIMEOUT_SECONDS"T1_TMP_ROOT="$T2_TMP_ROOT"@@ -52,10 +54,37 @@ die_t1() {}stop_isolated_postgres() {- if [ -n "$ISOLATED_CONTAINER" ]; then- docker rm -f "$ISOLATED_CONTAINER" >/dev/null 2>&1 || true+ [ -n "$ISOLATED_CONTAINER" ] || return 0+ if docker rm -f "$ISOLATED_CONTAINER" >/dev/null 2>&1; thenISOLATED_CONTAINER=""+ return 0fi+ return 1+}++stop_control_postgres_forward() {+ [ -n "$PORT_FORWARD_PID" ] || return 0+ if ! kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then+ return 0+ fi+ local command_line+ command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"+ if [[ "$command_line" != *port-forward* || "$command_line" != *svc/control-postgres* ]]; then+ printf '[minikube-t1] ERROR: T1 port-forward PID is not the expected control-postgres child; refusing to report cleanup success\n' >&2+ return 1+ fi+ if ! kill "$PORT_FORWARD_PID" >/dev/null 2>&1 && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then+ return 1+ fi+ if kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then+ if wait "$PORT_FORWARD_PID" >/dev/null 2>&1; then+ :+ fi+ fi+ if kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then+ return 1+ fi+ return 0}restore_gfs_runtime_credentials() {@@ -82,7 +111,8 @@ restore_gfs_runtime_credentials() {T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \CONTEXT="$T2_CONTEXT" \bash "$PROJECT_DIR/scripts/minikube/settle-gfs-reader-rollout.sh"; then- printf '[minikube-t1] WARN: could not settle a leftover gfsc-reader rollout claim before restore\n' >&2+ printf '[minikube-t1] ERROR: could not settle a leftover gfsc-reader rollout claim before restore; refusing to reconcile credentials\n' >&2+ return 1fiif ! PATH="$PROJECT_DIR/scripts/minikube/gfs-rollout-shim:$PATH" \T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \@@ -118,15 +148,12 @@ cleanup_t1() {# interrupt the lock release and leave a half-restored profile behind.trap - EXITtrap '' INT TERM- if [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then- local command_line- command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"- if [[ "$command_line" == *port-forward* && "$command_line" == *svc/control-postgres* ]]; then- kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true- wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true- fi+ if ! stop_control_postgres_forward; then+ status=1+ fi+ if ! stop_isolated_postgres; then+ status=1fi- stop_isolated_postgresif ! restore_gfs_runtime_credentials; thenstatus=1fi@@ -222,7 +249,11 @@ list_real_pg_files() {is_isolated_control_api_file() {local path="$1"case "$path" in- "$PROJECT_DIR"/control-api/test/*realPostgres*.test.ts)+ "$PROJECT_DIR"/control-api/*realPostgres*.test.ts)+ # Every control-api real-Postgres suite calls initDb. That migration+ # path can reconcile cluster-global GFS roles, so keeping any of these+ # suites on the branch profile database can poison the runtime even when+ # the fixture database itself is temporary.return 0;;*)@@ -232,19 +263,15 @@ is_isolated_control_api_file() {}require_isolated_control_api_files() {- local path count=0- while IFS= read -r path; do- [ -n "$path" ] || continue- count=$((count + 1))- is_isolated_control_api_file "$path" || {- T1_NEXT_COMMAND='route every control-api Real PostgreSQL suite through the disposable PostgreSQL instance'- die_t1 ZERO_TESTS_EXECUTED "control-api Real PostgreSQL suite is not isolated: $path"+ local path+ for path in \+ "$PROJECT_DIR/control-api/test/db.realPostgresMigration.integration.test.ts" \+ "$PROJECT_DIR/control-api/test/gfsReaderRole.realPostgres.integration.test.ts"; do+ [ -f "$path" ] || {+ T1_NEXT_COMMAND='restore the role-reset Real PostgreSQL suite files, then re-run T1'+ die_t1 ZERO_TESTS_EXECUTED "required isolated control-api suite is missing: $path"}- done < <(list_real_pg_files control-api)- if [ "$count" -le 0 ]; then- T1_NEXT_COMMAND='restore the control-api Real PostgreSQL suite files, then re-run T1'- die_t1 ZERO_TESTS_EXECUTED 'no control-api Real PostgreSQL suites were found'- fi+ done}start_isolated_postgres() {@@ -341,13 +368,10 @@ run_suite() {# comparable to the number of physical *realPostgres*.test.ts files above.# Validate the reporter's own total/passed suite counts instead; the file# discovery count remains a separate zero-selection guard.- if ! stats="$(python3 - "$json_file" "$expected_files" <<'PY'+ if ! stats="$(python3 - "$json_file" <<'PY'import jsonimport sysresult = json.loads(open(sys.argv[1]).read())-expected = int(sys.argv[2])-files = result.get("testResults") or []-passed_files = sum(1 for item in files if item.get("status") == "passed")total_suites = int(result.get("numTotalTestSuites") or 0)passed_suites = int(result.get("numPassedTestSuites") or 0)failed_suites = int(result.get("numFailedTestSuites") or 0)@@ -357,9 +381,8 @@ pending_files = int(result.get("numPendingTestSuites") or 0)pending_tests = int(result.get("numPendingTests") or 0)total_tests = int(result.get("numTotalTests") or 0)success = bool(result.get("success"))-print(total_suites, passed_suites, failed_suites, passed_files, passed_tests,- failed_tests, pending_files, pending_tests, total_tests, int(success),- expected, len(files))+print(total_suites, passed_suites, failed_suites, passed_tests, failed_tests,+ pending_files, pending_tests, total_tests, int(success))PY)"; thencat "$log_file" >&2 || true@@ -367,15 +390,13 @@ PYT1_NEXT_COMMAND='inspect the Vitest reporter output, then re-run T1'die_t1 ZERO_TESTS_EXECUTED "Real PostgreSQL reporter was unreadable for $package ($lane)"fi- read -r total_suites passed_suites failed_suites passed_files passed_tests \- failed_tests pending_files pending_tests total_tests success expected \- reported_files <<< "$stats"+ read -r total_suites passed_suites failed_suites passed_tests failed_tests \+ pending_files pending_tests total_tests success <<< "$stats"T1_TOTAL_TESTS=$((T1_TOTAL_TESTS + total_tests))T1_PASSED_TESTS=$((T1_PASSED_TESTS + passed_tests))T1_PENDING_TESTS=$((T1_PENDING_TESTS + pending_tests))if [ "$success" -ne 1 ] || [ "$total_suites" -le 0 ] || \[ "$passed_suites" -ne "$total_suites" ] || [ "$failed_suites" -ne 0 ] || \- [ "$passed_files" -ne "$expected" ] || [ "$reported_files" -ne "$expected" ] || \[ "$failed_tests" -ne 0 ]; thenT1_NEXT_COMMAND='repair the failed or incomplete Real PostgreSQL lane, then re-run T1'die_t1 REAL_PG_SUITE_FAILED "Real PostgreSQL reporter did not pass every suite in $package ($lane)"@@ -393,7 +414,7 @@ PYdie_t1 REAL_PG_SUITE_FAILED "Vitest process exited $suite_status for $package ($lane)"fiprintf '[minikube-t1] PASS %s %s files=%s tests=%s skipped=0\n' \- "$package" "$lane" "$passed_files" "$passed_tests"+ "$package" "$lane" "$expected_files" "$passed_tests"}main() {@@ -408,6 +429,7 @@ main() {die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'local Real PostgreSQL requires an explicit Minikube context'fit2_profile_status+ t2_profile_context_identity_checkif [ "$T2_BOOTSTRAP_REQUIRED" = true ]; thenT1_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'the selected profile is not bootstrapped'@@ -472,7 +494,6 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.decrequire_isolated_control_api_filesstart_isolated_postgresT1_GFS_RESTORE_REQUIRED=true- run_suite control-api shared "$ADMIN_DSN"run_suite control-api isolated "$ISOLATED_DSN"run_suite gfs-controller shared "$ADMIN_DSN"unset PG_USER PG_PASSWORD PG_DATABASE T1_REDACT_PASSWORDdiff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex a7f661fb1..edfb9654f 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -339,6 +339,17 @@ export ADMIN_PASSWORDPROFILE="${MINIKUBE_PROFILE:-clerum-test}"KC="kubectl --context=${PROFILE}"+# Full setup is a mutating profile transition as well. It acquires the same+# branch-owned lease used by T2 when run standalone, or validates the inherited+# lease when called by the T2 orchestrator. This is initialized before the+# config-only seam but acquired only after the profile is started and reachable.+T2_PROJECT_DIR="${T2_PROJECT_DIR:-${PROJECT_DIR}}"+T2_PROFILE="${T2_PROFILE:-${PROFILE}}"+T2_CONTEXT="${T2_CONTEXT:-${PROFILE}}"+T2_GATE_ID="${T2_GATE_ID:-minikube-setup}"+T2_SKIP_LOCK="${T2_SKIP_LOCK:-false}"+# shellcheck source=scripts/minikube/t2-common.sh+source "${SCRIPT_DIR}/t2-common.sh"TOTAL_STEPS=12# MINIKUBE_IMAGE_TAG overrides the committed pin AT RENDER TIME ONLY.#@@ -652,6 +663,48 @@ fiok "Cluster '${PROFILE}' is reachable"maybe_exit_after_cluster_step+SETUP_LOCK_CLEANUP_DONE=false+cleanup_setup_lock() {+ local status="${1:-$?}" cleanup_status=0+ if [ "$SETUP_LOCK_CLEANUP_DONE" = true ]; then+ return "$status"+ fi+ SETUP_LOCK_CLEANUP_DONE=true+ trap - EXIT+ trap '' INT TERM+ t2_lock_release "$status" || cleanup_status=$?+ if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then+ status="$cleanup_status"+ fi+ return "$status"+}+handle_setup_signal() {+ local signal="$1" status+ case "$signal" in+ INT) status=130 ;;+ TERM) status=143 ;;+ *) status=1 ;;+ esac+ cleanup_setup_lock "$status" || status=$?+ exit "$status"+}+handle_setup_exit() {+ local status=$?+ cleanup_setup_lock "$status" || status=$?+ exit "$status"+}+trap handle_setup_exit EXIT+trap 'handle_setup_signal INT' INT+trap 'handle_setup_signal TERM' TERM+t2_require_commands+t2_repo_metadata+t2_profile_scope+t2_profile_status+t2_context_check+t2_profile_context_identity_check+t2_mutation_lock+export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_SKIP_LOCK T2_LOCK_TOKEN+# ======================================================================# Step 3: Namespaces + CRDs# ======================================================================@@ -990,19 +1043,23 @@ else# wiped by the overlay re-apply). Re-sync it from the live platform# Secret before reconcile so a reader pod can actually start; this is a# no-op when the key already matches.- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+ GFS_AUTH_SYNC_ALLOW_STAGED=true \+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs# If gfsc-reader is already Ready, settle the leftover claim first so# reconcile does not rollout restart and race HCC's gfsReconciler.# The gfs-rollout-shim PATH prefix makes any reader rollout wait inside# reconcile judge readiness instead of the template generation HCC keeps# rewriting (it strips the restartedAt annotation, so a generation wait# loops until timeout).- CONTEXT="${PROFILE}" \+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"+ # A staged sync may have deferred active-pod proof until the reconciler+ # created the reader DSN. Complete that proof before the overlay is applied.+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfselselog "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"fi@@ -1067,13 +1124,17 @@ else# The overlay apply above re-declared the base gfs-config with an empty# jwt-public-key; re-sync it before any reader pod may need to start,# otherwise the reconcile readiness wait can only time out.- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs- CONTEXT="${PROFILE}" \+ GFS_AUTH_SYNC_ALLOW_STAGED=true \+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"+ # The preceding staged sync is allowed to wait for this DSN-producing+ # reconcile; the strict pass is the durable consumer attestation.+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfsok "GFS credentials reconciled and writer bootstrap verified"fidiff --git a/scripts/minikube/gfs-rollout-shim/kubectl b/scripts/minikube/gfs-rollout-shim/kubectlindex 8d34ed686..43c55f0b8 100755--- a/scripts/minikube/gfs-rollout-shim/kubectl+++ b/scripts/minikube/gfs-rollout-shim/kubectl@@ -69,6 +69,7 @@ done# explicit short timeout would hide a dead reader and invalidate the caller's# fail-fast contract.if ! timeout_seconds="$(python3 - "$timeout_raw" <<'PY'+import mathimport reimport sys@@ -77,15 +78,17 @@ timeout_raw = sys.argv[1]def parse_duration(raw, default, label):if raw == "":return default- match = re.fullmatch(r"[1-9][0-9]{0,4}([sm]?)", raw)- if not match:- raise SystemExit(f"invalid {label}: expected a positive integer with optional s/m suffix")- value = int(raw[:-1] if raw[-1:] in {"s", "m"} else raw)- if raw.endswith("m"):- value *= 60- if value < 1 or value > 86400:+ if re.fullmatch(r"[1-9][0-9]*", raw):+ total = float(raw)+ else:+ tokens = re.findall(r"([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h)", raw)+ if not tokens or "".join(number + unit for number, unit in tokens) != raw:+ raise SystemExit(f"invalid {label}: expected a positive Go-style duration")+ factors = {"ns": 1e-9, "us": 1e-6, "µs": 1e-6, "ms": 1e-3, "s": 1.0, "m": 60.0, "h": 3600.0}+ total = sum(float(number) * factors[unit] for number, unit in tokens)+ if total <= 0 or total > 86400:raise SystemExit(f"invalid {label}: value must be between 1 and 86400 seconds")- return value+ return max(1, math.ceil(total))print(parse_duration(timeout_raw, 600, "rollout timeout"))PYdiff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.shindex 625acefba..907f59c23 100755--- a/scripts/minikube/pf-all-stack.sh+++ b/scripts/minikube/pf-all-stack.sh@@ -11,6 +11,7 @@ PIDFILES=()PIDFILE_SERVICES=()SAFE_PROFILE="${PROFILE//[^A-Za-z0-9_.-]/_}"HAS_PROFILE_OWNED_PORTS=false+STARTUP_COMPLETE=falseif [[ ! "$PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; thenecho "ERROR: invalid Minikube profile identifier: $PROFILE" >&2exit 1@@ -69,6 +70,16 @@ elif [[ "${PROFILE}" =~ ^clerum-(codex|detached)- ]]; thenexit 1fi+is_branch_profile() {+ [[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]+}++if [[ ! -f "${BRANCH_PROFILE_PORTS_ENV}" ]] && is_branch_profile; then+ echo "ERROR: missing branch-scoped port cache for minikube profile: ${PROFILE}" >&2+ echo "Expected ${BRANCH_PROFILE_PORTS_ENV}" >&2+ exit 1+fi+if [[ -z "${PROFILE_UI_PORT:-}" && -n "${PORT_BASE:-}" ]]; thenPROFILE_UI_PORT=$((PORT_BASE + 1))fi@@ -99,7 +110,7 @@ require_random_port_for_branch_profile() {local name="$1"local value="$2"local default_port="$3"- if [[ "${HAS_PROFILE_OWNED_PORTS}" != "true" && ! "${PROFILE}" =~ ^clerum-(codex|detached)- ]]; then+ if [[ "${HAS_PROFILE_OWNED_PORTS}" != "true" ]] && ! is_branch_profile; thenreturn 0fiif [[ "${value}" == "${default_port}" ]]; then@@ -119,7 +130,7 @@ require_random_port_for_branch_profile WORKFLOW_APPROVAL_READER_PORT "${WORKFLOWrequire_random_port_for_branch_profile MCP_HOST_PORT "${MCP_HOST_PORT}" 8080cleanup() {- if [[ "${HOLD}" != "true" ]]; then+ if [[ "${HOLD}" != "true" && "${STARTUP_COMPLETE}" == "true" ]]; thenreturn 0filocal index@@ -131,7 +142,7 @@ cleanup() {kill_owned_pidfile() {local pidfile="$1"local service="$2"- local pid command_line+ local pid command_line expected_start actual_start[[ -f "$pidfile" ]] || return 0pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"if [[ ! "$pid" =~ ^[1-9][0-9]*$ ]]; then@@ -149,10 +160,29 @@ kill_owned_pidfile() {echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the $PROFILE $service port-forward" >&2return 1fi+ expected_start="$(sed -n 's/^PROCESS_START=//p' "$pidfile" 2>/dev/null | head -1 || true)"+ if [[ -z "$expected_start" ]]; then+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature is missing" >&2+ return 1+ fi+ if [[ "$expected_start" != unavailable ]]; then+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ if [[ -z "$actual_start" || "$actual_start" != "$expected_start" ]]; then+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature changed" >&2+ return 1+ fi+ fikill "$pid" 2>/dev/null || truerm -f -- "$pidfile"}+write_pidfile() {+ local pidfile="$1" pid="$2" process_start+ process_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [ -n "$process_start" ] || process_start=unavailable+ printf '%s\nPROCESS_START=%s\n' "$pid" "$process_start" >"$pidfile"+}+start_pf() {local name="$1"local namespace="$2"@@ -169,8 +199,8 @@ start_pf() {kill_owned_pidfile "$profile_pidfile" "$service"nohup "${KC[@]}" -n "${namespace}" port-forward --address=127.0.0.1 "svc/${service}" "${ports}" >"${log}" 2>&1 </dev/null &- echo $! >"${pidfile}"- echo $! >"${profile_pidfile}"+ write_pidfile "${pidfile}" "$!"+ write_pidfile "${profile_pidfile}" "$!"PIDS+=("$!")PIDFILES+=("$pidfile" "$profile_pidfile")PIDFILE_SERVICES+=("$service" "$service")@@ -238,6 +268,11 @@ start_optional_deployment_pf() {start_optional_pf "${name}" "${namespace}" "${service}" "${ports}" "${health_url}"}+# Install the cleanup boundary before the first background process is started.+# A later start_pf failure must not strand forwards that were already launched.+trap cleanup EXIT+trap 'cleanup; exit 0' INT TERM+echo "=== Starting gate port-forwards (${PROFILE}) ==="start_pf control-ui control-plane control-ui "${CONTROL_UI_PORT}:3000" "http://127.0.0.1:${CONTROL_UI_PORT}"start_optional_pf profile-ui profiles profile-ui "${PROFILE_UI_PORT}:3001" "http://127.0.0.1:${PROFILE_UI_PORT}"@@ -253,11 +288,10 @@ elif "${KC[@]}" get svc mcp-host -n mcp-host >/dev/null 2>&1; thenstart_pf mcp-host mcp-host mcp-host "${MCP_HOST_PORT}:8080" "http://127.0.0.1:${MCP_HOST_PORT}/v1/runtime/health"fi+STARTUP_COMPLETE=trueecho "=== Port-forwards refreshed ==="if [[ "${HOLD}" == "true" ]]; then- trap cleanup EXIT- trap 'cleanup; exit 0' INT TERMecho "=== Holding port-forwards open; press Ctrl-C to stop ==="while true; dosleep 3600 &diff --git a/scripts/minikube/pf-control-stack.sh b/scripts/minikube/pf-control-stack.shindex f6e42d7f2..e5b0110f1 100755--- a/scripts/minikube/pf-control-stack.sh+++ b/scripts/minikube/pf-control-stack.sh@@ -4,38 +4,77 @@# - control-api on :8090# Writes PIDs to /tmp/pf-*.pid for later cleanup.-set -eo pipefail+set -euo pipefail++PROFILE="${MINIKUBE_PROFILE:-${KUBECONTEXT:-}}"+if [ -z "$PROFILE" ]; then+ echo 'ERROR: MINIKUBE_PROFILE or KUBECONTEXT is required; refusing a context-less port-forward' >&2+ exit 1+fi+KC=(kubectl --context="$PROFILE")+SAFE_PROFILE="$(printf '%s' "$PROFILE" | tr -c 'A-Za-z0-9_.-' '_')"+CONTROL_UI_PORT="${CONTROL_UI_PORT:-3000}"+CONTROL_API_PORT="${CONTROL_API_PORT:-8090}"+LOCALHOST="localhost"+if [[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]] && {+ [ "$CONTROL_UI_PORT" = 3000 ] || [ "$CONTROL_API_PORT" = 8090 ];+}; then+ echo "ERROR: branch-owned profile $PROFILE requires profile-owned CONTROL_UI_PORT and CONTROL_API_PORT" >&2+ exit 1+fistart_pf() {local name=$1 ns=$2 svc=$3 ports=$4- local log=/tmp/pf-$name.log- local pidfile=/tmp/pf-$name.pid+ local log=/tmp/pf-${SAFE_PROFILE}-${name}.log+ local pidfile=/tmp/pf-${SAFE_PROFILE}-${name}.pid+ local pid expected_start actual_start process_start- # Kill previous+ # Kill previous only when both the command identity and process-start+ # signature match; a recycled PID must never be treated as ours.if [[ -f "$pidfile" ]]; then- kill "$(cat "$pidfile")" 2>/dev/null || true+ pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"+ if [[ "$pid" =~ ^[1-9][0-9]*$ ]] && kill -0 "$pid" 2>/dev/null; then+ command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)"+ expected_start="$(sed -n 's/^PROCESS_START=//p' "$pidfile" 2>/dev/null | head -1 || true)"+ if [[ "$command_line" != *port-forward* || "$command_line" != *"svc/$svc"* ||+ "$command_line" != *"$PROFILE"* || -z "$expected_start" ]]; then+ echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the signed $PROFILE $svc port-forward" >&2+ exit 1+ fi+ if [[ "$expected_start" != unavailable ]]; then+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [[ -n "$actual_start" && "$actual_start" == "$expected_start" ]] || {+ echo "ERROR: refusing to kill PID $pid from $pidfile; its process-start signature changed" >&2+ exit 1+ }+ fi+ kill "$pid" 2>/dev/null || true+ firm -f "$pidfile"fi- kubectl -n "$ns" port-forward "svc/$svc" "$ports" > "$log" 2>&1 &- echo $! > "$pidfile"+ "${KC[@]}" -n "$ns" port-forward --address=127.0.0.1 "svc/$svc" "$ports" > "$log" 2>&1 &+ pid=$!+ process_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [ -n "$process_start" ] || process_start=unavailable+ printf '%s\nPROCESS_START=%s\n' "$pid" "$process_start" > "$pidfile"echo " $name: pid=$(cat "$pidfile") ns=$ns svc=$svc ports=$ports"}echo "=== Starting port-forwards ==="-start_pf control-ui control-plane control-ui 3000:3000-start_pf control-api control-plane control-api 8090:8090+start_pf control-ui control-plane control-ui "${CONTROL_UI_PORT}:3000"+start_pf control-api control-plane control-api "${CONTROL_API_PORT}:8090"echo "=== Waiting 5s for tunnels to warm up ==="sleep 5echo "=== Health checks ==="-if curl -sf -m 3 http://localhost:3000 >/dev/null 2>&1; then+if curl -sf -m 3 "http://${LOCALHOST}:${CONTROL_UI_PORT}" >/dev/null 2>&1; thenecho " control-ui: UP"elseecho " control-ui: DOWN (may take longer, retry in a few seconds)"fi-if curl -sf -m 3 http://localhost:8090/health >/dev/null 2>&1; then+if curl -sf -m 3 "http://${LOCALHOST}:${CONTROL_API_PORT}/health" >/dev/null 2>&1; thenecho " control-api: UP"elseecho " control-api: DOWN (may take longer, retry in a few seconds)"diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.shindex d189ec3f4..747917c83 100755--- a/scripts/minikube/pre-gate-sync.sh+++ b/scripts/minikube/pre-gate-sync.sh@@ -62,6 +62,8 @@ if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; thent2_repo_metadatat2_profile_scopet2_context_check+ t2_profile_status+ t2_profile_context_identity_checkfilog() {@@ -237,6 +239,8 @@ commit_cluster_sync_state() {if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; thent2_mutation_lockfi+export T2_PROJECT_DIR T2_PROFILE T2_CONTEXT T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV \+ T2_SKIP_LOCK T2_LOCK_TOKENtrap finalize_pre_gate_sync EXITfingerprint_dir() {@@ -384,7 +388,9 @@ sync_mcp_host_auth_key() {}sync_gfs_auth_key() {- bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --require-gfs+ local allow_staged="${1:-false}"+ GFS_AUTH_SYNC_ALLOW_STAGED="${allow_staged}" \+ bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --require-gfs}settle_gfs_reader_rollout() {@@ -454,14 +460,24 @@ converge_gfs_reader_after_restore() {# restored Secret without waiting out CrashLoopBackOff, then judge# readiness directly.local pod_rows pod_name pod_ready pod_deleting- pod_rows="$(${KC} get pods -n gfs -l 'app=gfs-controller,clerum.io/gfsc-role=reader' -o \+ if ! pod_rows="$(${KC} get pods -n gfs -l 'app=gfs-controller,clerum.io/gfsc-role=reader' -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \- 2>/dev/null || true)"+ 2>&1)"; then+ if [[ "$pod_rows" == *NotFound* || "$pod_rows" == *"not found"* ]]; then+ pod_rows=""+ else+ log "ERROR: unable to inspect gfs reader pods before convergence: $pod_rows"+ return 1+ fi+ fiwhile IFS='|' read -r pod_name pod_ready pod_deleting; do[ -n "$pod_name" ] || continue[ -z "$pod_deleting" ] || continue[ "$pod_ready" != True ] || continue- ${KC} delete pod "$pod_name" -n gfs --wait=false >/dev/null 2>&1 || true+ if ! ${KC} delete pod "$pod_name" -n gfs --ignore-not-found --wait=false >/dev/null 2>&1; then+ log "ERROR: unable to delete unready gfs reader pod $pod_name"+ return 1+ fidone <<<"$pod_rows"CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/scripts/minikube/wait-gfs-reader-ready.sh"@@ -503,7 +519,7 @@ provision_gfs_serving() {# (the overlay re-applies the base ConfigMap with an empty value); a# reader pod cannot start without it and any readiness wait would only# time out. Re-sync it first; this is a no-op when the key matches.- sync_gfs_auth_key+ sync_gfs_auth_key true# Settle a Ready reader first so reconcile does not restart it and# race HCC's gfsReconciler during kubectl rollout status. If reconcile# still needs a reader rollout, the gfs-rollout-shim PATH prefix makes@@ -517,6 +533,14 @@ provision_gfs_serving() {log "ERROR: gfs DB provisioning FAILED — gfsc cannot authorize any operation. Aborting ${GATE_NAME} pre-gate sync."exit 1fi+ # The first staged sync may have deferred active-pod proof because this+ # reconcile call is the producer of gfs-controller-db. Re-run strict proof+ # after it so the durable marker is never published before consumers have+ # the key and a usable DSN.+ if ! sync_gfs_auth_key; then+ log "ERROR: gfs auth-key convergence failed after credential reconciliation"+ return 1+ fiif ! converge_gfs_reader_after_restore; thenlog "ERROR: gfs reader did not converge after credential restoration"return 1@@ -628,7 +652,9 @@ if [[ "${cluster_changed}" == "true" ]]; then"${INCREMENTAL_FULL_DEPLOYMENT}" == "true" ]]; then(cd "${PROJECT_DIR}"- make minikube-deploy-crds+ gfs_crd_mutation=false+ [[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_crd_mutation=true+ MINIKUBE_GFS_MUTATION="${gfs_crd_mutation}" make minikube-deploy-crds)if [[ "${GATE_NAME}" == "minikube-t2" ]]; thenCONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"diff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.shindex 0d27d5976..4b0a839f2 100755--- a/scripts/minikube/settle-gfs-reader-rollout.sh+++ b/scripts/minikube/settle-gfs-reader-rollout.sh@@ -32,15 +32,38 @@ source "$ROOT/scripts/minikube/t2-common.sh"if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fiT2_CONTEXT="$T2_CONTEXT"CONTEXT="$T2_CONTEXT"+SETTLE_T2_MANAGED=false+[ -n "$T2_LOCK_TOKEN" ] && SETTLE_T2_MANAGED=trueGFS_NS="${GFS_NS:-gfs}"DEPLOY="${GFS_READER_DEPLOYMENT:-gfsc-reader}"SECRET="${GFS_READER_DB_SECRET:-gfs-controller-reader-db}"SELECTOR="${GFS_READER_SELECTOR:-app=gfs-controller,clerum.io/gfsc-role=reader}"+SCALE_DOWN_TIMEOUT_SECONDS="${GFS_READER_SCALE_DOWN_TIMEOUT_SECONDS:-60}"+SCALE_DOWN_POLL_SECONDS="${GFS_READER_SCALE_DOWN_POLL_SECONDS:-1}"log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }die() { log "ERROR: $*"; exit 1; }kc() { kubectl --context="$T2_CONTEXT" "$@"; }+wait_for_scaled_rs() {+ local rs_name="$1" deadline desired pod_rows live+ deadline=$((SECONDS + SCALE_DOWN_TIMEOUT_SECONDS))+ while [ "$SECONDS" -lt "$deadline" ]; do+ desired="$(kc -n "$GFS_NS" get rs "$rs_name" -o jsonpath='{.spec.replicas}' 2>&1)" ||+ die "unable to verify ReplicaSet ${rs_name} scale-down: ${desired}"+ [[ "$desired" =~ ^[0-9]+$ ]] || die "ReplicaSet ${rs_name} returned a non-numeric replica count"+ pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \+ 'jsonpath={range .items[*]}{.metadata.ownerReferences[0].name}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' 2>&1)" ||+ die "unable to inspect pods while waiting for ReplicaSet ${rs_name} scale-down: ${pod_rows}"+ live="$(awk -F'|' -v rs="$rs_name" '$1 == rs && $2 == "" { n++ } END { print n+0 }' <<<"$pod_rows")"+ if [ "$desired" = 0 ] && [ "${live:-0}" -eq 0 ]; then+ return 0+ fi+ sleep "$SCALE_DOWN_POLL_SECONDS"+ done+ die "ReplicaSet ${rs_name} did not settle at zero replicas before credential proof"+}+SETTLE_CLEANUP_DONE=falsecleanup_settle() {@@ -51,7 +74,9 @@ cleanup_settle() {SETTLE_CLEANUP_DONE=truetrap - EXITtrap '' INT TERM- t2_lock_release "$status" || status=$?+ if [ "$SETTLE_T2_MANAGED" = true ]; then+ t2_lock_release "$status" || status=$?+ fireturn "$status"}handle_settle_signal() {@@ -72,10 +97,22 @@ handle_settle_exit() {trap handle_settle_exit EXITtrap 'handle_settle_signal INT' INTtrap 'handle_settle_signal TERM' TERM-t2_repo_metadata-t2_profile_scope-t2_context_check-t2_mutation_lock+if [ "$SETTLE_T2_MANAGED" = true ]; then+ t2_repo_metadata+ t2_profile_scope+ t2_context_check+ t2_mutation_lock+else+ [ "${GFS_READER_ROLLOUT_AUTHORIZED:-false}" = true ] ||+ die 'non-T2 reader settlement requires explicit GFS_READER_ROLLOUT_AUTHORIZED=true'+ authorized=false+ IFS=',' read -r -a allowed_contexts <<<"${ALLOWED_CONTEXTS:-}"+ for allowed_context in "${allowed_contexts[@]}"; do+ [ "$allowed_context" = "$CONTEXT" ] && authorized=true+ done+ [ "$authorized" = true ] ||+ die "non-T2 reader settlement context is not in the explicit ALLOWED_CONTEXTS list: $CONTEXT"+fi# The proof library is sourced only after the profile fence is established. It# reads the committed DSN in memory and never prints it.@@ -115,6 +152,7 @@ fi# Desired Ready is met from here on: leftovers below serve nothing.current_rev="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \-o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"+scaled_rs_names=()rs_rows="$(kc -n "$GFS_NS" get rs -l "$SELECTOR" -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.spec.replicas}{"|"}{.status.readyReplicas}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' \2>/dev/null)"@@ -127,8 +165,13 @@ while IFS='|' read -r rs_name rs_owner rs_replicas rs_ready rs_rev; do[ -n "$current_rev" ] && [ "$rs_rev" = "$current_rev" ] && continuelog "scaling leftover unready ReplicaSet ${rs_name} to 0 (revision ${rs_rev:-unknown}, current ${current_rev:-unknown})"kc -n "$GFS_NS" scale rs "$rs_name" --replicas=0 >/dev/null+ scaled_rs_names+=("$rs_name")done <<<"$rs_rows"+for rs_name in "${scaled_rs_names[@]}"; do+ wait_for_scaled_rs "$rs_name"+done+pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.containerStatuses[0].state.waiting.reason}{"\n"}{end}' \2>/dev/null)"diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.shindex 47f0fec87..5a742287a 100755--- a/scripts/minikube/sync-auth-key.sh+++ b/scripts/minikube/sync-auth-key.sh@@ -50,6 +50,39 @@ if [[ "${REQUIRE_GFS}" == "true" && "${SYNC_GFS}" != "true" ]]; thenexit 2fi+log() { printf '[sync-auth-key] %s\n' "$*"; }+die() {+ log "ERROR: $*" >&2+ exit 1+}++ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+T2_PROJECT_DIR="${T2_PROJECT_DIR:-${ROOT}}"+T2_PROFILE="${T2_PROFILE:-${MINIKUBE_PROFILE:-${PROFILE}}}"+T2_CONTEXT="${T2_CONTEXT:-${CONTROL_API_REAL_PG_CONTEXT:-${PROFILE}}}"+T2_LOCK_ROOT="${T2_LOCK_ROOT:-${HOME}/.cache/evenfire/minikube-t2-locks}"+T2_LOCK_TOKEN="${T2_LOCK_TOKEN:-}"+# Auth-key convergence mutates runtime ConfigMaps and restarts consumers. It is+# a child of the owning T2/full-setup transition, never an independent writer.+# Validate the live opaque lease here as well as at the public Make boundary so+# invoking this helper directly cannot bypass profile ownership.+source "${ROOT}/scripts/minikube/t2-common.sh"+[[ -n "${T2_LOCK_TOKEN}" ]] || die "canonical T2 profile lock is required for auth-key convergence"+T2_LOCK_DIR="${T2_LOCK_ROOT}/${T2_PROFILE}.lock"+[[ -d "${T2_LOCK_DIR}" && ! -L "${T2_LOCK_DIR}" ]] ||+ die "canonical T2 profile lock is missing for ${T2_PROFILE}"+owner_token="$(t2_lock_owner_value TOKEN || true)"+owner_pid="$(t2_lock_owner_value PID || true)"+[[ "${owner_token}" == "${T2_LOCK_TOKEN}" ]] ||+ die "canonical T2 profile lock token does not match"+[[ "$(t2_lock_owner_value REPOSITORY || true)" == "${T2_PROJECT_DIR}" ]] ||+ die "canonical T2 profile lock belongs to another worktree"+[[ "$(t2_lock_owner_value PROFILE || true)" == "${T2_PROFILE}" &&+ "$(t2_lock_owner_value CONTEXT || true)" == "${T2_CONTEXT}" ]] ||+ die "canonical T2 profile lock belongs to another profile or context"+t2_lock_process_matches "${owner_pid}" ||+ die "canonical T2 profile lock owner is no longer a live process"+KCTL=(kubectl --context="${PROFILE}")SOURCE_SECRET="rpc-proxy-secrets"SOURCE_NAMESPACE="rpc-proxy"@@ -70,12 +103,11 @@ MCP_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"GFS_CONFIGMAP="gfs-config"GFS_NAMESPACE="gfs"GFS_TARGET_KEY="jwt-public-key"--log() { printf '[sync-auth-key] %s\n' "$*"; }-die() {- log "ERROR: $*" >&2- exit 1-}+# During the T2 recovery transition, credential reconciliation is the producer+# of the GFS DSN. An explicitly staged sync may update gfs-config and defer+# active-pod proof until that producer has run; ordinary/full-setup callers+# remain fail-closed by default.+GFS_AUTH_SYNC_ALLOW_STAGED="${GFS_AUTH_SYNC_ALLOW_STAGED:-false}"rollout_restart_with_retry() {local namespace="$1"@@ -118,18 +150,42 @@ print(json.dumps({"data": {os.environ["TARGET_KEY"]: source_value}}))PY}-make_applied_hash_patch() {- APPLIED_HASH="$1" APPLIED_HASH_ANNOTATION="${APPLIED_HASH_ANNOTATION}" python3 - <<'PY'+make_applied_hash_json_patch() {+ local namespace="$1" configmap="$2" target_key="$3" target_json+ target_json="$("${KCTL[@]}" get configmap "${configmap}" -n "${namespace}" -o json)" ||+ die "cannot snapshot auth target ${namespace}/${configmap} before attestation"+ TARGET_JSON="$target_json" TARGET_KEY="$target_key" SOURCE_KEY_B64="$source_key_b64" \+ APPLIED_HASH="$source_hash" APPLIED_HASH_ANNOTATION="$APPLIED_HASH_ANNOTATION" \+ python3 - <<'PY'+import base64import jsonimport os-print(json.dumps({- "metadata": {- "annotations": {- os.environ["APPLIED_HASH_ANNOTATION"]: os.environ["APPLIED_HASH"]- }- }-}))+obj = json.loads(os.environ["TARGET_JSON"])+metadata = obj.get("metadata") or {}+data = obj.get("data") or {}+annotations = metadata.get("annotations") or {}+rv = str(metadata.get("resourceVersion") or "")+if not rv:+ raise SystemExit("auth target has no resourceVersion")+try:+ expected = base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=True).decode("utf-8")+except (ValueError, UnicodeDecodeError) as exc:+ raise SystemExit(f"invalid source auth key: {exc}")+key = os.environ["TARGET_KEY"]+if data.get(key) != expected:+ raise SystemExit("auth target changed before attestation")+annotation = os.environ["APPLIED_HASH_ANNOTATION"]+ops = [+ {"op": "test", "path": "/metadata/resourceVersion", "value": rv},+ {"op": "test", "path": "/data/" + key.replace("~", "~0").replace("/", "~1"), "value": expected},+]+path = "/metadata/annotations/" + annotation.replace("~", "~0").replace("/", "~1")+if annotations:+ ops.append({"op": "replace" if annotation in annotations else "add", "path": path, "value": os.environ["APPLIED_HASH"]})+else:+ ops.append({"op": "add", "path": "/metadata/annotations", "value": {annotation: os.environ["APPLIED_HASH"]}})+print(json.dumps(ops))PY}@@ -140,9 +196,17 @@ read_applied_hash() {}mark_key_applied() {- local namespace="$1" configmap="$2"- "${KCTL[@]}" patch configmap "${configmap}" -n "${namespace}" --type=merge \- -p "$(make_applied_hash_patch "${source_hash}")" >/dev/null+ local namespace="$1" configmap="$2" target_key="${3:-}"+ if [ -z "${target_key}" ]; then+ case "${configmap}" in+ "${CONFIGMAP}") target_key="${TARGET_KEY}" ;;+ "${GFS_CONFIGMAP}") target_key="${GFS_TARGET_KEY}" ;;+ *) die "cannot infer auth target key for ${namespace}/${configmap}" ;;+ esac+ fi+ assert_source_unchanged+ "${KCTL[@]}" patch configmap "${configmap}" -n "${namespace}" --type=json \+ -p "$(make_applied_hash_json_patch "${namespace}" "${configmap}" "${target_key}")" >/dev/null}configmap_key_hash() {@@ -151,6 +215,20 @@ configmap_key_hash() {-o "jsonpath={.data.${key}}" | shasum -a 256 | awk '{print $1}'}+deployment_desired_replicas() {+ local namespace="$1" deployment="$2" desired+ desired="$("${KCTL[@]}" get "deployment/${deployment}" -n "${namespace}" \+ -o 'jsonpath={.spec.replicas}' 2>&1)" || {+ log "cannot read desired replicas for ${namespace}/deployment/${deployment}: ${desired}" >&2+ return 1+ }+ [[ "${desired}" =~ ^[0-9]+$ ]] || {+ log "deployment ${namespace}/deployment/${deployment} returned a non-numeric desired replica count" >&2+ return 1+ }+ printf '%s' "${desired}"+}+write_source_key() {SOURCE_KEY_B64="${source_key_b64}" python3 - <<'PY'import base64@@ -161,6 +239,49 @@ sys.stdout.buffer.write(base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=PY}+assert_source_unchanged() {+ local current_snapshot current_rv ignored_b64 current_hash+ current_snapshot="$(read_source_snapshot)" || die "cannot re-read auth source snapshot"+ IFS=$'\t' read -r current_rv ignored_b64 current_hash <<<"${current_snapshot}"+ [[ -n "${current_rv}" && -n "${current_hash}" ]] ||+ die "auth source snapshot is incomplete during convergence"+ if [[ "${current_rv}" != "${source_resource_version}" || "${current_hash}" != "${source_hash}" ]]; then+ die "auth source rotated during convergence; refusing to attest a stale key"+ fi+}++read_source_snapshot() {+ local source_json+ source_json="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o json)" ||+ return 1+ SOURCE_JSON="${source_json}" SOURCE_KEY="${SOURCE_KEY}" python3 - <<'PY'+import base64+import hashlib+import json+import os+import sys++try:+ obj = json.loads(os.environ["SOURCE_JSON"])+ metadata = obj.get("metadata") or {}+ data = obj.get("data") or {}+ resource_version = str(metadata.get("resourceVersion") or "")+ source_key_b64 = data.get(os.environ["SOURCE_KEY"]) or ""+ source_value = base64.b64decode(source_key_b64, validate=True)+ source_value.decode("utf-8")+except (KeyError, TypeError, ValueError, UnicodeDecodeError, json.JSONDecodeError):+ raise SystemExit(1)++if not resource_version or not source_value:+ raise SystemExit(1)+print("%s\t%s\t%s" % (+ resource_version,+ source_key_b64,+ hashlib.sha256(source_value).hexdigest(),+))+PY+}+consumer_pods_use_key() {local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"local rows pod ready deleting pod_count=0@@ -216,35 +337,16 @@ if ! source_probe="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMfidie "cannot inspect auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET}: ${source_probe}"fi-source_key_b64="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}")"-if [[ -z "${source_key_b64}" ]]; then+source_snapshot="$(read_source_snapshot 2>/dev/null || true)"+if [[ -z "${source_snapshot}" ]]; thenif [[ "${REQUIRE_GFS}" == "true" ]]; thendie "required GFS auth source key ${SOURCE_KEY} is empty"fidie "auth source key ${SOURCE_KEY} is empty; refusing to mutate consumers"fi-if ! source_hash="$(SOURCE_KEY_B64="${source_key_b64}" python3 - <<'PY'-import base64-import hashlib-import os-import sys--try:- source_value = base64.b64decode(os.environ["SOURCE_KEY_B64"], validate=True)- source_value.decode("utf-8")-except (ValueError, UnicodeDecodeError) as exc:- print(f"invalid source auth key encoding: {exc}", file=sys.stderr)- raise SystemExit(1)--if not source_value:- print("source auth key decodes empty", file=sys.stderr)- raise SystemExit(1)--print(hashlib.sha256(source_value).hexdigest())-PY-)"; then- die "source auth key ${SOURCE_KEY} is not valid non-empty UTF-8 base64"-fi+IFS=$'\t' read -r source_resource_version source_key_b64 source_hash <<<"${source_snapshot}"+[[ -n "${source_resource_version}" && -n "${source_key_b64}" && -n "${source_hash}" ]] ||+ die "source auth key ${SOURCE_KEY} is not a valid non-empty UTF-8 base64 value with a resourceVersion"if [[ "${REQUIRE_GFS}" == "true" ]] && \! gfs_target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; thendie "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable: ${gfs_target_probe}"@@ -286,7 +388,7 @@ sync_mcp_host() {log "${CONFIGMAP}.${TARGET_KEY} matches source but consumer attestation is pending; resuming convergence"fifi- local deployment_resources deployment resource_name+ local deployment_resources deployment resource_name desired_replicaslocal deployments=()if ! deployment_resources="$("${KCTL[@]}" get deployment -l "${MCP_DEPLOY_SELECTOR}" \-n "${CONFIG_NAMESPACE}" -o name 2>&1)"; then@@ -306,6 +408,15 @@ sync_mcp_host() {log "No mcp-host deployments exist yet; auth key is synced for future pods"elsefor deployment in "${deployments[@]}"; do+ desired_replicas="$(deployment_desired_replicas "${CONFIG_NAMESPACE}" "${deployment}")" ||+ die "cannot determine desired replicas for mcp-host deployment/${deployment}"+ if [[ "${desired_replicas}" == "0" ]]; then+ # HCC intentionally suspends stateless Hosts by scaling their managed+ # Deployment to zero. There is no consumer to restart or prove; the+ # patched ConfigMap is the authoritative input for the next pod.+ log "Skipping suspended ${CONFIG_NAMESPACE}/${deployment}; desired replicas=0"+ continue+ fiif [[ "${force_rollout}" != true ]] && \consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \"${TARGET_KEY}" "mcp-host deployment/${deployment}"; then@@ -357,7 +468,7 @@ sync_gfs() {fifi- local deployments=() rollout_deployments=() deployment deployment_probe role selector rollout_output dsn_encoded dsn+ local deployments=() rollout_deployments=() deployment deployment_probe role selector rollout_output dsn_encoded dsn desired_replicasfor deployment in gfsc-writer gfsc-reader; dodeployment_probe=""if deployment_probe="$("${KCTL[@]}" get "deployment/${deployment}" -n "${GFS_NAMESPACE}" 2>&1)"; then@@ -378,6 +489,12 @@ sync_gfs() {for deployment in "${deployments[@]}"; dorole="${deployment#gfsc-}"selector="app=gfs-controller,clerum.io/gfsc-role=${role}"+ desired_replicas="$(deployment_desired_replicas "${GFS_NAMESPACE}" "${deployment}")" ||+ die "cannot determine desired replicas for GFS deployment/${deployment}"+ if [[ "${desired_replicas}" == "0" ]]; then+ log "Skipping suspended ${GFS_NAMESPACE}/${deployment}; desired replicas=0"+ continue+ fiif [[ "${force_rollout}" != true ]] && \consumer_pods_use_key "${GFS_NAMESPACE}" "${selector}" gfsc \GFS_JWT_PUBLIC_KEY "gfsc deployment/${deployment}"; then@@ -396,9 +513,17 @@ sync_gfs() {fiif ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then+ if [[ "${GFS_AUTH_SYNC_ALLOW_STAGED}" == "true" ]]; then+ log "Deferring active gfsc auth proof until GFS credentials are reconciled"+ return 0+ fidie "cannot inspect ${GFS_NAMESPACE}/gfs-controller-db before gfsc restart: ${dsn_encoded}"fiif [[ -z "${dsn_encoded}" ]]; then+ if [[ "${GFS_AUTH_SYNC_ALLOW_STAGED}" == "true" ]]; then+ log "Deferring active gfsc auth proof until GFS credentials are reconciled"+ return 0+ fidie "${GFS_NAMESPACE}/gfs-controller-db.connection-string is empty; refusing to attest gfsc auth convergence"fiif ! dsn="$(printf '%s' "${dsn_encoded}" | base64 -d 2>&1)"; then@@ -431,4 +556,5 @@ if [[ "${SYNC_GFS}" == "true" ]]; thensync_gfsfi+assert_source_unchangedlog "Auth key synced"diff --git a/scripts/minikube/t0.sh b/scripts/minikube/t0.shindex 1521503e8..d8f777b26 100755--- a/scripts/minikube/t0.sh+++ b/scripts/minikube/t0.sh@@ -12,15 +12,40 @@ package_dirs=()seen_packages="|"while IFS= read -r path; do[ -n "$path" ] || continue+ absolute_path="$PROJECT_DIR/$path"case "$path" in- *.sh|*.bash) shell_files+=("$PROJECT_DIR/$path") ;;+ *.sh|*.bash)+ # Deleted scripts are part of the diff but cannot be parsed. Every+ # surviving shell script gets a syntax check even when shellcheck is not+ # installed; shellcheck is an additional lint lane, not the syntax gate.+ if [ -f "$absolute_path" ]; then+ bash -n "$absolute_path"+ shell_files+=("$absolute_path")+ fi+ ;;esac- top="${path%%/*}"- if [ "$top" != "$path" ] && [ -f "$PROJECT_DIR/$top/package.json" ] && [[ "$seen_packages" != *"|$top|"* ]]; then- seen_packages="${seen_packages}${top}|"- package_dirs+=("$top")++ # Resolve the nearest package.json by walking upward from the changed file.+ # This covers nested packages such as packages/workflow-sdk instead of+ # incorrectly mapping them to packages/package.json.+ candidate="$absolute_path"+ if [ ! -d "$candidate" ]; then+ candidate="$(dirname -- "$candidate")"fi-done < <(git -C "$PROJECT_DIR" diff --name-only "$ORIGIN_DEV...$HEAD")+ while [ "$candidate" != "$PROJECT_DIR" ] && [[ "$candidate" == "$PROJECT_DIR"/* ]]; do+ if [ -f "$candidate/package.json" ]; then+ package="${candidate#"$PROJECT_DIR/"}"+ if [[ "$seen_packages" != *"|$package|"* ]]; then+ seen_packages="${seen_packages}${package}|"+ package_dirs+=("$package")+ fi+ break+ fi+ parent="$(dirname -- "$candidate")"+ [ "$parent" != "$candidate" ] || break+ candidate="$parent"+ done+done < <(git -C "$PROJECT_DIR" diff --name-only --diff-filter=ACMRTUXB "$ORIGIN_DEV...$HEAD")if [ "${#shell_files[@]}" -gt 0 ]; thenif command -v shellcheck >/dev/null 2>&1; thendiff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex 99a215739..e48bfc47b 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -47,6 +47,7 @@ T2_PLAN_MODE="$T2_PLAN_MODE"if [ -z "$T2_PLAN_MODE" ]; then T2_PLAN_MODE=false; fiT2_BOOTSTRAP_REQUIRED=falseT2_PROFILE_HEALTHY=false+T2_CONTEXT_IDENTITY_VERIFIED=falseT2_UNREADY_DEPLOYMENTS=""T2_MARKER_MATCHES_HEAD=falseT2_MARKER_JSON=""@@ -79,6 +80,8 @@ T2_REQUIRED_NAMESPACES="$T2_REQUIRED_NAMESPACES"if [ -z "$T2_REQUIRED_NAMESPACES" ]; then T2_REQUIRED_NAMESPACES="control-plane gfs mcp-host mcp-server profiles rpc-proxy channels sandbox-recipes sandbox-ui webhook-ingress registry"; fiT2_REQUIRED_SERVICES="$T2_REQUIRED_SERVICES"if [ -z "$T2_REQUIRED_SERVICES" ]; then T2_REQUIRED_SERVICES="control-plane/control-postgres control-plane/control-api control-plane/control-ui control-plane/host-context-controller control-plane/workflow-recipes profiles/external-rest-api profiles/profile-ui rpc-proxy/rpc-proxy mcp-server/mcp-proxy"; fi+T2_REQUIRED_DEPLOYMENTS="$T2_REQUIRED_DEPLOYMENTS"+if [ -z "$T2_REQUIRED_DEPLOYMENTS" ]; then T2_REQUIRED_DEPLOYMENTS="control-plane/control-api control-plane/host-context-controller profiles/external-rest-api rpc-proxy/rpc-proxy mcp-host/chatllm"; fiT2_REQUIRED_SECRETS="$T2_REQUIRED_SECRETS"if [ -z "$T2_REQUIRED_SECRETS" ]; then T2_REQUIRED_SECRETS="control-plane/control-postgres control-plane/control-api-secrets control-plane/control-ui-secrets control-plane/inter-service-tokens profiles/external-rest-api-secrets rpc-proxy/rpc-proxy-secrets mcp-host/chatllm-api-keys gfs/gfs-controller-db gfs/gfs-controller-reader-db"; fiT2_REQUIRED_CONFIGMAPS="$T2_REQUIRED_CONFIGMAPS"@@ -252,13 +255,65 @@ t2_context_check() {host="${host%%:*}"fihost="$(printf '%s' "$host" | tr '[:upper:]' '[:lower:]')"- case "$host" in- 127.0.0.1|localhost|::1|192.168.*|*.minikube) ;;- *)- T2_NEXT_COMMAND='select the generated branch-owned Minikube context, not a remote cluster context'- t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is not local: $host"- ;;- esac+ if [[ "$host" == 127.0.0.1 || "$host" == localhost || "$host" == ::1 || "$host" == *.minikube ]]; then+ return 0+ fi+ if ! python3 - "$host" <<'PY'+import ipaddress+import sys++try:+ address = ipaddress.ip_address(sys.argv[1])+except ValueError:+ raise SystemExit(1)+raise SystemExit(0 if (address.is_private or address.is_loopback or address.is_link_local) else 1)+PY+ then+ T2_NEXT_COMMAND='select the generated branch-owned Minikube context, not a remote cluster context'+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is not local: $host"+ return 1+ fi+}++t2_profile_context_identity_check() {+ # A localhost API endpoint is necessary but not sufficient: kubeconfig can+ # point at another local Minikube profile, and a broad RFC1918 allowlist can+ # point at an unrelated LAN cluster. Bind the context to the exact profile+ # identity and node address reported by the selected Minikube instance.+ if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then+ return 0+ fi+ local expected_ip nodes_json+ expected_ip="$(t2_mk ip 2>/dev/null || true)"+ if [ -z "$expected_ip" ]; then+ T2_NEXT_COMMAND='start the verified branch-owned Minikube profile, then retry the gate'+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Minikube did not report an IP for profile $T2_PROFILE"+ return 1+ fi+ nodes_json="$(t2_kc get nodes -o json 2>/dev/null || true)"+ if [ -z "$nodes_json" ] || ! python3 - "$nodes_json" "$T2_PROFILE" "$expected_ip" <<'PY'+import json+import sys++payload = json.loads(sys.argv[1])+profile = sys.argv[2]+expected_ip = sys.argv[3]+for node in payload.get("items", []):+ metadata = node.get("metadata") or {}+ labels = metadata.get("labels") or {}+ if labels.get("minikube.k8s.io/name") != profile:+ continue+ for address in (node.get("status") or {}).get("addresses", []):+ if address.get("type") == "InternalIP" and address.get("address") == expected_ip:+ raise SystemExit(0)+raise SystemExit(1)+PY+ then+ T2_NEXT_COMMAND='select the kube-context generated for this exact branch-owned Minikube profile'+ t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context does not identify Minikube profile $T2_PROFILE at $expected_ip"+ return 1+ fi+ T2_CONTEXT_IDENTITY_VERIFIED=true}t2_profile_status() {@@ -490,22 +545,51 @@ t2_deployment_check() {T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is unavailable'fi- if ! unready="$(python3 - "$deployment_json" 2>/dev/null <<'PY'+ if ! unready="$(python3 - "$deployment_json" "$T2_REQUIRED_DEPLOYMENTS" 2>/dev/null <<'PY'import jsonimport syspayload = json.loads(sys.argv[1])+required_refs = {}+for ref in sys.argv[2].split():+ namespace, separator, name = ref.partition("/")+ if not separator or not namespace or not name:+ raise ValueError("invalid required deployment reference")+ required_refs[(namespace, name)] = Nonebad = []for item in payload.get("items", []):metadata = item.get("metadata") or {}+ identity = (metadata.get("namespace"), metadata.get("name"))+ if identity in required_refs:+ required_refs[identity] = item++for (namespace, name), item in required_refs.items():+ if item is None:+ bad.append(f"{namespace}/{name} missing")+ continuespec = item.get("spec") or {}status = item.get("status") or {}- desired = int(spec.get("replicas") or 0)+ def integer(value, field):+ if value in (None, ""):+ return 0+ if isinstance(value, bool):+ raise ValueError(field)+ return int(value)++ desired = integer(spec.get("replicas"), "spec.replicas")+ # A present deployment explicitly scaled to zero is an intentional local+ # suspension; it is not an unready pod.if desired == 0:continue- ready = int(status.get("readyReplicas") or 0)- available = int(status.get("availableReplicas") or 0)- if ready < desired or available < desired:- bad.append("%s/%s %s/%s" % (metadata.get("namespace", "?"), metadata.get("name", "?"), ready, desired))+ ready = integer(status.get("readyReplicas"), "status.readyReplicas")+ available = integer(status.get("availableReplicas"), "status.availableReplicas")+ updated = integer(status.get("updatedReplicas"), "status.updatedReplicas")+ unavailable = integer(status.get("unavailableReplicas"), "status.unavailableReplicas")+ generation = integer(metadata.get("generation"), "metadata.generation")+ observed = integer(status.get("observedGeneration"), "status.observedGeneration")+ if generation <= 0 or observed != generation or updated < desired or ready < desired or available < desired or unavailable != 0:+ bad.append("%s/%s ready=%s/%s updated=%s observed=%s/%s unavailable=%s" % (+ namespace, name, ready, desired,+ updated, observed, generation, unavailable))print("; ".join(bad))PY)"; then@@ -538,6 +622,17 @@ PYfi}+t2_pid_file_matches_process() {+ local pid_file="$1" pid="$2" recorded_pid recorded_start actual_start+ recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"+ recorded_start="$(sed -n 's/^PROCESS_START=//p' "$pid_file" 2>/dev/null | head -1 || true)"+ [ "$recorded_pid" = "$pid" ] || return 1+ [ -n "$recorded_start" ] || return 1+ [ "$recorded_start" = unavailable ] && return 0+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [ -n "$actual_start" ] && [ "$actual_start" = "$recorded_start" ]+}+t2_process_check() {local uid pid ppid rest command_line comm allowed pid_file recorded_pidlocal safe_profile@@ -547,14 +642,9 @@ t2_process_check() {# Default IFS so UID/PID/PPID split. `IFS=` left pid empty and skipped every line.# awk is a loose pre-filter: kubectl as argv0/path token AND a later# standalone port-forward token. Flags may sit between those tokens.- # Known limitation: a port-forward whose argv carries no profile/context- # token (e.g. pf-control-stack's context-less `kubectl -n NS port-forward`)- # cannot be attributed to a cluster from ps output alone and is skipped —- # flagging it would also flag unrelated clusters' forwards.while read -r uid pid ppid rest; do[ -n "$pid" ] || continuecommand_line="$uid $pid $ppid $rest"- [[ "$command_line" == *"$T2_PROFILE"* || "$command_line" == *"$T2_CONTEXT"* ]] || continuecomm="$(ps -p "$pid" -o comm= 2>/dev/null || true)"case "$comm" in*kubectl*) ;;@@ -566,19 +656,32 @@ t2_process_check() {/tmp/pf-"$safe_profile"-*.pid; do[ -f "$pid_file" ] || continuerecorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"- [ "$recorded_pid" = "$pid" ] && allowed=true+ if [ "$recorded_pid" = "$pid" ] && t2_pid_file_matches_process "$pid_file" "$pid"; then+ allowed=true+ fidone# pf-all-stack.sh is the canonical gate forwarder and records its child# PIDs in /tmp/pf-<profile>-*.pid. Accept those PIDs as profile-owned too;# otherwise T1 rejects the forwards that pre-gate-sync just started.- for pid_file in "/tmp/pf-${profile_safe}-"*.pid; do+ for pid_file in "/tmp/pf-${safe_profile}-"*.pid; do[ -f "$pid_file" ] || continue- pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"- [[ -n "$pid" && "$command_line" == *" $pid "* ]] && allowed=true+ recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"+ if [[ -n "$recorded_pid" && "$command_line" == *" $recorded_pid "* ]] &&+ t2_pid_file_matches_process "$pid_file" "$recorded_pid"; then+ allowed=true+ fidone+ # Unrelated developer forwards launched without --context are not+ # attributable to this profile and must not make its gate red. A forward+ # that names this profile/context, or one recorded in this profile's PID+ # registry, is attributable and must be owned by this run.+ if [[ "$command_line" != *"$T2_PROFILE"* && "$command_line" != *"$T2_CONTEXT"* && "$allowed" != true ]]; then+ continue+ fiif [ "$allowed" != true ]; thenT2_NEXT_COMMAND='stop the unrelated profile port-forward or select the owner worktree; do not share it't2_fail PORT_FORWARD_CONFLICT 'a port-forward for this profile is owned by another process'+ return 1fidone < <(ps -ef 2>/dev/null | awk '/([^[:space:]]*\/)?kubectl([[:space:]]|$)/ && /[[:space:]]port-forward([[:space:]]|$)/ {print}' || true)}@@ -749,18 +852,20 @@ EOFt2_lock_validate_inherited() {local expected_pid expected_token owner_key owner_value- t2_lock_profile_id_check+ t2_lock_profile_id_check || return 1T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"T2_LOCK_KEY="$(t2_lock_key)"if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR/owner.env" ]; thenT2_NEXT_COMMAND='run the parent T2 command with its profile lock token, then retry this child operation't2_fail PROFILE_LOCK_REQUIRED "inherited profile lock is missing for $T2_PROFILE"+ return 1fiexpected_token="$T2_LOCK_TOKEN"owner_value="$(t2_lock_owner_value TOKEN || true)"if [ -z "$expected_token" ] || [ "$owner_value" != "$expected_token" ]; thenT2_NEXT_COMMAND='pass the opaque T2_LOCK_TOKEN from the owning T2 process; do not bypass the profile lock't2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock token is missing or does not match'+ return 1fifor owner_key in REPOSITORY BRANCH HEAD PROFILE CONTEXT WORKTREE_ID LOCK_KEY; docase "$owner_key" in@@ -775,12 +880,14 @@ t2_lock_validate_inherited() {if [ "$(t2_lock_owner_value "$owner_key" || true)" != "$expected_pid" ]; thenT2_NEXT_COMMAND='re-run the operation from the worktree/profile that owns the T2 lock't2_fail PROFILE_OWNERSHIP_MISMATCH "profile lock owner does not match $owner_key"+ return 1fidoneexpected_pid="$(t2_lock_owner_value PID || true)"if ! t2_lock_process_matches "$expected_pid"; thenT2_NEXT_COMMAND='restart the parent T2 operation so it can acquire a live profile lock't2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'+ return 1fiT2_LOCK_HELD=falseT2_LOCK_RELEASED=false@@ -812,6 +919,12 @@ t2_lock_release() {if [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; thent2_evidence_write failure FAIL "$T2_ERROR_CODE"fi+ if [ "$cleanup_status" -ne 0 ] && [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ]; then+ # A PASS written before EXIT is not a valid attestation if the profile lock+ # could not be released. Invalidate it explicitly and keep the detail+ # secret-free so a later runtime-only lane cannot reuse unsafe evidence.+ t2_evidence_write lock-cleanup INVALIDATED 'profile lock cleanup failed; prior attestation is invalid' || true+ fiif [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; thenstatus="$cleanup_status"fi@@ -898,6 +1011,12 @@ if status in {"FAIL", "INVALIDATED"}:elif phase == "complete" and status == "PASS":attestation = "PASS"prior["attestationStatus"] = attestation+lane_attestation = prior.get("laneAttestationStatus", "IN_PROGRESS")+if phase == "lanes" and status == "PASS":+ lane_attestation = "PASS"+elif phase == "lock-cleanup" and status == "INVALIDATED":+ lane_attestation = "INVALIDATED"+prior["laneAttestationStatus"] = lane_attestationprior.setdefault("phases", []).append({"name": os.environ.get("PHASE", ""),"status": os.environ.get("STATUS", ""),@@ -945,7 +1064,7 @@ for candidate in sorted(root.glob("*/evidence.json"), reverse=True):continueif data.get("certificationVersion") != 1:continue- if data.get("attestationStatus") != "PASS":+ if data.get("attestationStatus") != "PASS" and data.get("laneAttestationStatus") != "PASS":continueif not data.get("runId"):continuediff --git a/scripts/minikube/t2-preflight.sh b/scripts/minikube/t2-preflight.shindex 4a90e54bf..26b73146d 100755--- a/scripts/minikube/t2-preflight.sh+++ b/scripts/minikube/t2-preflight.sh@@ -21,7 +21,12 @@ T2_NEXT_COMMAND='re-run the canonical command with the verified profile'cleanup_preflight() {local status=$?- t2_lock_release "$status"+ trap - EXIT+ trap '' INT TERM+ if ! t2_lock_release "$status"; then+ [ "$status" -ne 0 ] || status=1+ fi+ exit "$status"}trap cleanup_preflight EXIT@@ -29,13 +34,16 @@ main() {t2_require_commandst2_repo_metadatat2_profile_scope- t2_context_check+ t2_profile_status+ if [ "$T2_BOOTSTRAP_REQUIRED" != true ]; then+ t2_context_check+ fit2_mutation_lockt2_evidence_init# Profile status is read before any resource check. A missing/stopped profile# is a plan for bootstrap, never a reason to invoke pre-gate-sync.- t2_profile_status+ t2_profile_context_identity_checkt2_marker_checkt2_image_checkt2_resource_checks@@ -48,6 +56,7 @@ main() {if [ "$T2_PLAN_STATE" = full-bootstrap ] && [ "$T2_PLAN_MODE" != true ]; thenT2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE IMAGE_SOURCE=local make minikube-setup"t2_fail BOOTSTRAP_REQUIRED "$T2_PLAN_REASON"+ return 1fit2_evidence_write preflight PASS "$T2_PLAN_STATE: $T2_PLAN_REASON"diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.shindex 30712ea71..f7547cb9c 100755--- a/scripts/minikube/t2.sh+++ b/scripts/minikube/t2.sh@@ -87,6 +87,7 @@ run_preflight_plan() {bash "$SCRIPT_DIR/t2-preflight.sh"; thenT2_NEXT_COMMAND='re-run make minikube-t2-preflight and repair its first reported precondition't2_fail DEVELOPMENT_SCOPE_REQUIRED 'preflight failed before a state transition was selected'+ return 1fiT2_PLAN_STATE="$(python3 - "$T2_PLAN_TMP" <<'PY'import json@@ -103,6 +104,7 @@ PYif [ -z "$T2_PLAN_STATE" ]; thenT2_NEXT_COMMAND='re-run make minikube-t2-preflight and inspect its evidence't2_fail BOOTSTRAP_REQUIRED 'preflight did not produce a state transition'+ return 1fiprintf '[minikube-t2] transition=%s reason=%s\n' "$T2_PLAN_STATE" "$T2_PLAN_REASON"}@@ -115,12 +117,14 @@ run_pvc_reset_if_authorized() {if [ -z "$T2_EXPECTED_PVC_UID" ]; thenT2_NEXT_COMMAND='set T2_EXPECTED_PVC_UID to the exact recorded UID and keep T2_RESET_PVC=true only for local development't2_fail POSTGRES_NOT_READY 'destructive PVC reset requires an exact expected UID'+ return 1filocal actual_uidactual_uid="$(t2_kc -n "$T2_CONTROL_NAMESPACE" get pvc control-postgres-data -o 'jsonpath={.metadata.uid}' 2>/dev/null || true)"if [ "$actual_uid" != "$T2_EXPECTED_PVC_UID" ]; thenT2_NEXT_COMMAND='do not reset this PVC; re-check the profile and recorded UID't2_fail POSTGRES_NOT_READY 'PVC UID does not match the explicit reset expectation'+ return 1fiprintf '[minikube-t2] authorized development-only PVC reset for the exact expected UID\n'}@@ -162,6 +166,7 @@ run_bootstrap_or_reconcile() {*)T2_NEXT_COMMAND='re-run make minikube-t2-preflight to select bootstrap, targeted sync, full reconcile, or already-synced't2_fail BOOTSTRAP_REQUIRED "unknown transition: $T2_PLAN_STATE"+ return 1;;esact2_evidence_write transition PASS "$T2_PLAN_STATE"@@ -210,6 +215,7 @@ run_t1() {t2_evidence_write T1 FAIL 'Real PostgreSQL lane exited zero but did not emit a complete green T1 result'T2_NEXT_COMMAND='repair the T1 result contract; it must emit PASS, positive total/passed counts, zero pending tests, and an evidence path't2_fail REAL_PG_SUITE_FAILED 'T1 result was incomplete despite a zero process exit'+ return 1fiT2_T1_STATUS=PASST2_T1_COUNTS="$(awk -F= '/^T1_TESTS=|^T1_PASSED_TESTS=|^T1_PENDING_TESTS=/{printf "%s%s", (n++ ? "," : ""), $0}' "$T2_T1_OUTPUT")"@@ -234,6 +240,7 @@ run_final_preflight() {cat "$T2_FINAL_PREFLIGHT_OUTPUT" >&2 || trueT2_NEXT_COMMAND='repair the first reported final preflight condition, then re-run T2 on the same HEAD't2_fail DEVELOPMENT_SCOPE_REQUIRED 'final exact-head readiness preflight failed'+ return 1fiT2_PLAN_STATE="$(python3 - "$T2_PLAN_TMP" <<'PY'import json@@ -244,6 +251,7 @@ PYif [ "$T2_PLAN_STATE" != already-synced ]; thenT2_NEXT_COMMAND='re-run make minikube-t2 so pre-gate-sync updates the marker to this HEAD't2_fail HEAD_MARKER_MISMATCH "final T2 preflight selected $T2_PLAN_STATE instead of already-synced"+ return 1fit2_marker_checkT2_T2_STATUS=PASS@@ -283,6 +291,7 @@ run_healthcheck_if_requested() {t2_evidence_write Health FAIL 'profile-owned user-facing health command failed'T2_NEXT_COMMAND='repair the first failing user-facing health check, then re-run T2 on the same HEAD't2_fail PROFILE_UNHEALTHY 'user-facing health check failed'+ return 1fi}@@ -293,6 +302,7 @@ run_playwright_if_requested() {if [ "$T2_REQUIRE_PLAYWRIGHT" = true ]; thenT2_NEXT_COMMAND='set T2_PLAYWRIGHT_COMMAND to the applicable profile-owned Playwright journey and re-run't2_fail ZERO_TESTS_EXECUTED 'Playwright was required but no journey command was supplied'+ return 1fireturn 0fi@@ -310,7 +320,10 @@ main() {t2_require_commandst2_repo_metadatat2_profile_scope- t2_context_check+ t2_profile_status+ if [ "$T2_BOOTSTRAP_REQUIRED" != true ]; then+ t2_context_check+ fit2_mutation_lockt2_evidence_init@@ -318,11 +331,13 @@ main() {if [ "$T2_RUN_T0" = true ] && ! t2_lane_completed "$T2_T0_STATUS"; thenT2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T0=true so the static lane is executed't2_fail ZERO_TESTS_EXECUTED 'T0 was not executed'+ return 1firun_preflight_planif { [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; } && [ "$T2_PLAN_STATE" != already-synced ]; thenT2_NEXT_COMMAND='run make minikube-t2 so bootstrap/reconcile and T0/T1 execute on this HEAD't2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2-only mode requires an already-synced exact-head profile'+ return 1fiif [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; thent2_marker_check@@ -334,10 +349,18 @@ main() {if [ "$T2_RUN_T1" = true ] && ! t2_lane_completed "$T2_T1_STATUS"; thenT2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T1=true so the Real PostgreSQL lane is executed't2_fail ZERO_TESTS_EXECUTED 'T1 was not executed'+ return 1fiif [ "$T2_RUN_T1" != true ] && [ "$T2_T1_CERTIFIED" != true ]; thenT2_NEXT_COMMAND='run make minikube-t2 with T2_RUN_T1=true to create an exact-head T1 attestation't2_fail CERTIFICATION_REQUIRED 'T1 was skipped without a validated exact-head attestation'+ return 1+ fi+ if [ "$T2_T0_STATUS" = PASS ] && [ "$T2_T1_STATUS" = PASS ]; then+ # Publish the reusable static/database lane attestation before runtime T2.+ # If the later runtime lane fails, a subsequent runtime-only invocation may+ # reuse these exact-head lanes without pretending that T2 already passed.+ t2_evidence_write lanes PASS 'T0 and T1 completed before runtime T2'firun_final_preflightrun_np08_hcc_authorizationdiff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.shindex c0f9f6b8a..5683b7e6d 100755--- a/scripts/minikube/wait-gfs-reader-ready.sh+++ b/scripts/minikube/wait-gfs-reader-ready.sh@@ -21,16 +21,22 @@ kc() { kubectl --context="$CONTEXT" "$@"; }log() { printf '[wait-gfs-reader-ready] %s\n' "$*" >&2; }if ! TIMEOUT_SECONDS="$(python3 - "$TIMEOUT_SECONDS" <<'PY'+import mathimport reimport sysraw = sys.argv[1]-if not re.fullmatch(r"[1-9][0-9]{0,4}", raw):- raise SystemExit("timeout must be a positive integer")-value = int(raw)-if value > 86400:+if re.fullmatch(r"[1-9][0-9]*", raw):+ total = float(raw)+else:+ tokens = re.findall(r"([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h)", raw)+ if not tokens or "".join(number + unit for number, unit in tokens) != raw:+ raise SystemExit("timeout must be a positive Go-style duration")+ factors = {"ns": 1e-9, "us": 1e-6, "µs": 1e-6, "ms": 1e-3, "s": 1.0, "m": 60.0, "h": 3600.0}+ total = sum(float(number) * factors[unit] for number, unit in tokens)+if total <= 0 or total > 86400:raise SystemExit("timeout exceeds the one-day limit")-print(value)+print(max(1, math.ceil(total)))PY)"; thenlog 'refusing an invalid readiness timeout'@@ -67,9 +73,10 @@ while :; dolog "unable to inspect reader pods for ${GFS_NS}/${DEPLOY}: ${pod_rows}"exit 1fi+ live_ready="$(awk -F'|' 'NF && $2 == "" && $1 == "True" { n++ } END { print n+0 }' <<<"$pod_rows")"unready="$(awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }' <<<"$pod_rows")"- if [ "${unready:-0}" -eq 0 ]; then- log "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with no live unready reader pod"+ if [ "${live_ready:-0}" -ge "$desired" ] && [ "${unready:-0}" -eq 0 ]; then+ log "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with ${live_ready} live Ready reader pod(s)"exit 0fifidiff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex 8a12acfb1..fefbe5785 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -48,8 +48,8 @@ if CONTEXT=unverified-context bash deploy/scripts/reconcile-gfs-deploy-credentiarm -f "$reconcile_context_err"fail 'credential reconciliation accepted an unverified Kubernetes context'fi-grep -Fq 'refusing unverified Kubernetes context' "$reconcile_context_err" \- || fail 'credential reconciliation did not explain its unverified-context rejection'+grep -Fq 'worktree is dirty' "$reconcile_context_err" \+ || fail 'credential reconciliation did not fail closed before an unowned worktree could mutate GFS'rm -f "$reconcile_context_err"reconcile_remote_context_err="$(mktemp)"if CONTEXT=gke_unapproved GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=gke_approved \@@ -57,13 +57,14 @@ if CONTEXT=gke_unapproved GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=rm -f "$reconcile_remote_context_err"fail 'credential reconciliation accepted a GKE context outside the explicit allowlist'fi-grep -Fq 'explicit authorization and exact ALLOWED_CONTEXTS membership' "$reconcile_remote_context_err" \- || fail 'credential reconciliation did not fail closed for an unauthorized GKE context'+grep -Fq 'worktree is dirty' "$reconcile_remote_context_err" \+ || fail 'credential reconciliation did not fail closed before a protected context could mutate GFS'rm -f "$reconcile_remote_context_err"-grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED' deploy/scripts/reconcile-gfs-deploy-credentials.sh \- || fail 'remote GFS reconciliation lacks an explicit authorization gate'-grep -Fq 'ALLOWED_CONTEXTS' deploy/scripts/reconcile-gfs-deploy-credentials.sh \- || fail 'remote GFS reconciliation lacks an exact context allowlist'+grep -Fq 't2_mutation_lock' deploy/scripts/reconcile-gfs-deploy-credentials.sh \+ || fail 'GFS reconciliation does not require the canonical profile mutation lock'+if grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED=true' deploy/scripts/reconcile-gfs-deploy-credentials.sh; then+ fail 'GFS reconciliation still trusts caller-provided remote authorization'+fimake_block() {awk -v target="$1" '$0 ~ "^" target ":" {active=1} active && /^\.PHONY:/ {exit} active {print}' Makefile}diff --git a/scripts/tests/test-gfs-real-pg-minikube-gate.sh b/scripts/tests/test-gfs-real-pg-minikube-gate.shindex 99c89b2a0..2105a6e44 100755--- a/scripts/tests/test-gfs-real-pg-minikube-gate.sh+++ b/scripts/tests/test-gfs-real-pg-minikube-gate.sh@@ -7,9 +7,16 @@ SCRIPT="${ROOT_DIR}/scripts/e2e/gfs-real-pg-minikube-gate.sh"bash -n "${SCRIPT}"grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "${SCRIPT}"grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "${SCRIPT}"-grep -Fq 'gitHead' "${SCRIPT}"-grep -Fq 'worktreeId' "${SCRIPT}"-grep -Fq 'port-forward' "${SCRIPT}"+grep -Fq 'verify_branch_gate' "${SCRIPT}"+grep -Fq 't2_profile_scope' "${SCRIPT}"+grep -Fq 't2_profile_context_identity_check' "${SCRIPT}"+grep -Fq 't2_marker_check' "${SCRIPT}"+grep -Fq 't2_mutation_lock' "${SCRIPT}"+grep -Fq 'rollout status deployment/control-postgres' "${SCRIPT}"+grep -Fq 'get secret control-postgres' "${SCRIPT}"+grep -Fq 'PROFILE_PG_FORWARD_PID' "${SCRIPT}"+grep -Fq 'docker run -d --rm' "${SCRIPT}"+grep -Fq 'ISOLATED_PG_IMAGE' "${SCRIPT}"grep -Fq 'pending_files' "${SCRIPT}"grep -Fq -- '--reporter=json' "${SCRIPT}"grep -Fq "gfs*.realPostgres.integration.test.ts" "${SCRIPT}"@@ -19,8 +26,11 @@ grep -Fq 'restore_gfs_runtime_credentials' "${SCRIPT}"grep -Fq 'required gfs-controller-db Secret is missing or unreadable' "${SCRIPT}"grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "${SCRIPT}"grep -Fq 'failed to restore branch-profile GFS credentials' "${SCRIPT}"-grep -Fq 'gke_*)' "${SCRIPT}"-grep -Fq '*prod*)' "${SCRIPT}"+grep -Fq 'docker rm -f' "${SCRIPT}"+if grep -Fq 'docker rm -f "${ISOLATED_CONTAINER}" >/dev/null 2>&1 || true' "${SCRIPT}"; then+ echo 'FAIL: standalone GFS cleanup suppresses Docker removal failure' >&2+ exit 1+figrep -Fq 'test-gfs-real-postgres-minikube' "${ROOT_DIR}/Makefile"grep -Fq "setupFiles: ['test/realPostgres.requirement.ts']" "${ROOT_DIR}/control-api/vitest.config.ts"grep -Fq 'setupFiles: ["test/realPostgres.requirement.ts"]' "${ROOT_DIR}/gfs-controller/vitest.config.ts"diff --git a/scripts/tests/test-minikube-gfs-provision-order.sh b/scripts/tests/test-minikube-gfs-provision-order.shindex 422d0adcd..d1356d556 100755--- a/scripts/tests/test-minikube-gfs-provision-order.sh+++ b/scripts/tests/test-minikube-gfs-provision-order.sh@@ -88,7 +88,7 @@ sync_mcp_host_auth_key() {}provision_gfs_serving-expected=$'sync\nsettle\nreconcile\nconverge'+expected=$'sync\nsettle\nreconcile\nsync\nconverge'actual="$(cat "$order_file")"[ "$actual" = "$expected" ] || {printf 'expected:\n%s\nactual:\n%s\n' "$expected" "$actual" >&2diff --git a/scripts/tests/test-minikube-gfs-rollout-shim.sh b/scripts/tests/test-minikube-gfs-rollout-shim.shindex 6664581df..878ec920c 100755--- a/scripts/tests/test-minikube-gfs-rollout-shim.sh+++ b/scripts/tests/test-minikube-gfs-rollout-shim.sh@@ -62,6 +62,13 @@ grep -q 'gfs-rollout-shim' <<<"$out" || fail 'shim did not announce the readinesgrep -q 'rollout status deployment/gfsc-reader' "$FAKE_KUBECTL_LOG" \&& fail 'shim leaked the generation-based rollout status to the real kubectl'+for duration in 2m30s 500ms; do+ PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\n' \+ GFS_READER_WAIT_POLL_SECONDS=0 \+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout="$duration" >/dev/null 2>&1 \+ || fail "shim rejected valid Go duration ${duration}"+done+# 4. A live unready reader pod blocks the wait until timeout (fail-loud).# The explicit 1s caller timeout is honored; the shim must not silently floor# it to an unrelated minimum.@@ -94,4 +101,13 @@ PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout=240s >/dev/null 2>&1 \|| fail 'shim readiness wait blocked on a terminating pod'+# A terminating Ready pod must not satisfy the desired live-reader count. It+# is already leaving the service and cannot be used as readiness evidence.+if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \+ FAKE_POD_ROWS='True|2026-01-01T00:00:00Z\n' \+ GFS_READER_WAIT_POLL_SECONDS=0 \+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout=1s >/dev/null 2>&1; then+ fail 'shim readiness wait went green with only a terminating Ready reader pod'+fi+printf 'PASS: gfs-rollout-shim readiness interception contract\n'diff --git a/scripts/tests/test-minikube-pre-gate-sync-state.sh b/scripts/tests/test-minikube-pre-gate-sync-state.shindex 82efc12be..3af20ddeb 100755--- a/scripts/tests/test-minikube-pre-gate-sync-state.sh+++ b/scripts/tests/test-minikube-pre-gate-sync-state.sh@@ -186,7 +186,7 @@ fifinal_cluster_fingerprint_line="$(grep -n 'cluster_fingerprint=.*pre_gate_marker_cluster_fingerprint' "$SCRIPT" | tail -n 1 | cut -d: -f1)"final_infra_fingerprint_line="$(grep -n 'infra_fingerprint=.*pre_gate_marker_infra_fingerprint' "$SCRIPT" | tail -n 1 | cut -d: -f1)"-persist_marker_line="$(grep -nF 'persist_cluster_marker "${cluster_fingerprint}" "${infra_fingerprint}"' "$SCRIPT" | tail -n 1 | cut -d: -f1)"+persist_marker_line="$(grep -nF 'commit_cluster_sync_state "${cluster_fingerprint}" "${infra_fingerprint}"' "$SCRIPT" | tail -n 1 | cut -d: -f1)"if [ -n "$final_cluster_fingerprint_line" ] &&[ -n "$final_infra_fingerprint_line" ] &&[ -n "$persist_marker_line" ] &&diff --git a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh b/scripts/tests/test-minikube-settle-gfs-reader-rollout.shindex ab4243ce7..c03134af3 100755--- a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh+++ b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh@@ -61,9 +61,12 @@ case " $* " inexit 0 ;;*' get rs -l '*)printf '%b' "${FAKE_RS_ROWS:-}"; exit 0 ;;+ *' get rs gfsc-reader-stale '*'.spec.replicas'*)+ [ -s "${FAKE_SCALE_FILE:?}" ] && printf '0' || printf '1'; exit 0 ;;*' get rs gfsc-reader-proof-rs '*revision*)printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;*' scale rs '*)+ printf '%s' "${4:-gfsc-reader-stale}" >"${FAKE_SCALE_FILE:?}"exit 0 ;;*' get pods -l '*gfs-template-hash*)printf '%b' "${FAKE_PROOF_POD_ROWS:-gfsc-reader-proof|2026-01-02T00:00:01Z|True||gfsc-reader-proof-rs|}"; exit 0 ;;@@ -104,7 +107,9 @@ exit 1STUBchmod +x "$fake_dir/kubectl"local status=0+ : >"$fake_dir/scaled-rs"PATH="$fake_dir:$PATH" CONTEXT="$profile" \+ T2_LOCK_TOKEN=settle-test-token \T2_PROJECT_DIR="$repo" T2_PROFILE="$profile" T2_CONTEXT="$profile" \MINIKUBE_PROFILE="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" \T2_PROFILE_ROOT="$fake_dir/profiles" T2_PROFILE_ENV="$fake_dir/profiles/$profile/profile.env" \@@ -113,6 +118,7 @@ STUBT2_GATE_ID=settle-gfs-reader-test T2_RUN_ID=settle-test-run \FAKE_KUBECTL_LOG="$fake_dir/kubectl.log" \FAKE_PATCH_LOG="$fake_dir/patch.json" FAKE_DSN=dsn-value \+ FAKE_SCALE_FILE="$fake_dir/scaled-rs" \FAKE_DEPLOY_EXISTS="${1:-1}" \FAKE_DESIRED="${2:-1}" \FAKE_READY="${3:-1}" \diff --git a/scripts/tests/test-minikube-sync-auth-key-durable.sh b/scripts/tests/test-minikube-sync-auth-key-durable.shindex 8a6c13cb8..3c8ebdb6f 100755--- a/scripts/tests/test-minikube-sync-auth-key-durable.sh+++ b/scripts/tests/test-minikube-sync-auth-key-durable.sh@@ -12,6 +12,24 @@ SOURCE_HASH="$(printf 'public-key' | shasum -a 256 | awk '{print $1}')"mkdir -p "${STATE_DIR}"printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"+# The production helper requires a live parent T2 lease. Keep the behavior+# tests isolated with a lease owned by this test shell rather than adding a+# production-only bypass.+T2_LOCK_ROOT="${TMP_DIR}/locks"+T2_LOCK_TOKEN='auth-sync-test-token'+T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"+T2_PROCESS_START=unavailable+mkdir -p "${T2_LOCK_DIR}"+cat >"${T2_LOCK_DIR}/owner.env" <<EOF+REPOSITORY=${ROOT}+PROFILE=fake+CONTEXT=fake+TOKEN=${T2_LOCK_TOKEN}+PID=$$+PROCESS_START=${T2_PROCESS_START}+EOF+export T2_PROJECT_DIR="${ROOT}" T2_PROFILE=fake T2_CONTEXT=fake T2_LOCK_ROOT T2_LOCK_TOKEN+cat >"${TMP_DIR}/kubectl" <<'SH'#!/usr/bin/env bashset -euo pipefail@@ -30,13 +48,44 @@ increment() {}if [[ "${1:-}" == get && "${2:-}" == secret && "${3:-}" == rpc-proxy-secrets ]]; then- [[ "$*" == *jsonpath* ]] && printf '%s' "${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"+ if [[ "$*" == *'.metadata.resourceVersion'* ]]; then+ printf '42'+ elif [[ "$*" == *'-o json' ]]; then+ source_b64="${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"+ source_rv=42+ if [[ -n "${TEST_ROTATE_ON_SOURCE_SNAPSHOT:-}" ]]; then+ snapshot_file="${TEST_STATE_DIR}/source-json-snapshot.count"+ snapshot_count=0+ [[ -f "${snapshot_file}" ]] && snapshot_count="$(cat "${snapshot_file}")"+ snapshot_count=$((snapshot_count + 1))+ printf '%s' "${snapshot_count}" >"${snapshot_file}"+ if (( snapshot_count >= TEST_ROTATE_ON_SOURCE_SNAPSHOT )); then+ source_b64='cm90YXRlZA=='+ source_rv=43+ fi+ fi+ printf '{"metadata":{"resourceVersion":"%s"},"data":{"RPC_PROXY_JWT_PUBLIC_KEY":"%s"}}' "${source_rv}" "${source_b64}"+ elif [[ "$*" == *jsonpath* ]]; then+ printf '%s' "${TEST_SOURCE_B64:-cHVibGljLWtleQ==}"+ fiexit 0fiif [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == mcp-host-config ]]; thenif [[ "$*" == *auth-key-applied-sha256* ]]; then[[ -f "${TEST_STATE_DIR}/mcp.applied" ]] && cat "${TEST_STATE_DIR}/mcp.applied"+ elif [[ "$*" == *'-o json' ]]; then+ python3 - <<'PY'+import json+import os+from pathlib import Path+state = Path(os.environ['TEST_STATE_DIR'])+annotations = {}+if (state / 'mcp.applied').exists():+ annotations['clerum.io/auth-key-applied-sha256'] = (state / 'mcp.applied').read_text()+print(json.dumps({'metadata': {'resourceVersion': '42', 'annotations': annotations},+ 'data': {'CLERUM_AUTH_JWT_PUBLIC_KEY': (state / 'mcp.key').read_text()}}))+PYelif [[ "$*" == *jsonpath* ]]; thencat "${TEST_STATE_DIR}/mcp.key"fi@@ -50,6 +99,18 @@ if [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == gfs-config ]]; the}if [[ "$*" == *auth-key-applied-sha256* ]]; then[[ -f "${TEST_STATE_DIR}/gfs.applied" ]] && cat "${TEST_STATE_DIR}/gfs.applied"+ elif [[ "$*" == *'-o json' ]]; then+ python3 - <<'PY'+import json+import os+from pathlib import Path+state = Path(os.environ['TEST_STATE_DIR'])+annotations = {}+if (state / 'gfs.applied').exists():+ annotations['clerum.io/auth-key-applied-sha256'] = (state / 'gfs.applied').read_text()+print(json.dumps({'metadata': {'resourceVersion': '42', 'annotations': annotations},+ 'data': {'jwt-public-key': (state / 'gfs.key').read_text()}}))+PYelif [[ "$*" == *jsonpath* ]]; thencat "${TEST_STATE_DIR}/gfs.key"fi@@ -84,10 +145,15 @@ fiif [[ "${1:-}" == get && "${2:-}" == deployment/* ]]; thencase "${2}" in- deployment/chatllm|deployment/custom-host) exit 0 ;;+ deployment/chatllm|deployment/custom-host)+ [[ "$*" == *'.spec.replicas'* ]] && printf '%s' "${TEST_MCP_REPLICAS:-1}"+ exit 0 ;;deployment/mcp-host) echo 'Error from server (NotFound): deployments.apps "mcp-host" not found' >&2; exit 1 ;;deployment/gfsc-writer|deployment/gfsc-reader)- if [[ "${TEST_GFS_DEPLOYMENTS:-0}" == 1 ]]; then exit 0; fi+ if [[ "${TEST_GFS_DEPLOYMENTS:-0}" == 1 ]]; then+ [[ "$*" == *'.spec.replicas'* ]] && printf '1'+ exit 0+ fiprintf 'Error from server (NotFound): deployments.apps "%s" not found\n' "${2#deployment/}" >&2exit 1 ;;*) exit 1 ;;@@ -145,7 +211,15 @@ fiecho "unexpected kubectl invocation: $*" >&2exit 99SH-chmod +x "${TMP_DIR}/kubectl"+cat >"${TMP_DIR}/ps" <<'SH'+#!/usr/bin/env bash+case " $* " in+ *' -o state='*) printf 'R\n' ;;+ *' -o lstart='*) printf 'unavailable\n' ;;+ *) exit 1 ;;+esac+SH+chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/ps"run_sync() {local source_hash="${TEST_SOURCE_HASH_OVERRIDE:-${SOURCE_HASH}}"@@ -206,7 +280,19 @@ if run_sync env TEST_RUNTIME_KEY=old-key TEST_MCP_DEPLOYMENTS=custom \exit 1fi[[ "$(cat "${STATE_DIR}/custom-host-restart.count")" == 1 ]]-grep -q 'has not consumed the target auth key' "${OUT_FILE}"++# A stateless HCC Host can be intentionally suspended at replicas=0. The+# ConfigMap still converges, but there is no pod to restart or prove; a future+# pod must consume the current ConfigMap value.+printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"+rm -f "${STATE_DIR}/mcp.applied"+run_sync env TEST_MCP_REPLICAS=0 TEST_RUNTIME_KEY=old-key \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1+[[ "$(cat "${STATE_DIR}/mcp.key")" == public-key ]]+[[ "$(cat "${STATE_DIR}/mcp.applied")" == "${SOURCE_HASH}" ]]+[[ "$(cat "${STATE_DIR}/chatllm-restart.count")" == 1 ]]+grep -q 'Skipping suspended mcp-host/chatllm' "${OUT_FILE}"run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=custom \bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \@@ -277,4 +363,24 @@ if grep -q 'rollout restart deployment/' "${LOG_FILE}"; thenexit 1fi+# A source rotation after consumer proof must invalidate the run before the+# final success message, even when the previous convergence marker was already+# written. This is the race the resourceVersion+key snapshot closes.+printf 'old-key' >"${STATE_DIR}/mcp.key"+rm -f "${STATE_DIR}/mcp.applied" "${STATE_DIR}/source-json-snapshot.count"+: >"${LOG_FILE}"+if TEST_ROTATE_ON_SOURCE_SNAPSHOT=3 run_sync env \+ TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=chatllm \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ echo 'FAIL: auth-key sync certified a source that rotated after consumer proof' >&2+ exit 1+fi+grep -q 'auth source rotated during convergence' "${OUT_FILE}" || {+ cat "${OUT_FILE}" >&2+ echo 'FAIL: auth-key sync did not report the final source rotation' >&2+ exit 1+}+echo "PASS: auth-key convergence is durable, resumable, consumer-proven, and idempotent"diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.shindex 08d0f320f..5c02f7175 100755--- a/scripts/tests/test-minikube-sync-auth-key.sh+++ b/scripts/tests/test-minikube-sync-auth-key.sh@@ -10,11 +10,38 @@ LOG_FILE="${TMP_DIR}/kubectl.log"OUT_FILE="${TMP_DIR}/sync-auth-key.out"mkdir -p "${STATE_DIR}"+# The production helper requires a live parent T2 lease. Keep the behavior+# tests isolated with a lease owned by this test shell rather than adding a+# production-only bypass.+T2_LOCK_ROOT="${TMP_DIR}/locks"+T2_LOCK_TOKEN='auth-sync-test-token'+T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"+T2_PROCESS_START=unavailable+mkdir -p "${T2_LOCK_DIR}"+cat >"${T2_LOCK_DIR}/owner.env" <<EOF+REPOSITORY=${ROOT}+PROFILE=fake+CONTEXT=fake+TOKEN=${T2_LOCK_TOKEN}+PID=$$+PROCESS_START=${T2_PROCESS_START}+EOF+export T2_PROJECT_DIR="${ROOT}" T2_PROFILE=fake T2_CONTEXT=fake T2_LOCK_ROOT T2_LOCK_TOKEN+cat >"${TMP_DIR}/sleep" <<'SH'#!/usr/bin/env bashexit 0SH+cat >"${TMP_DIR}/ps" <<'SH'+#!/usr/bin/env bash+case " $* " in+ *' -o state='*) printf 'R\n' ;;+ *' -o lstart='*) printf 'unavailable\n' ;;+ *) exit 1 ;;+esac+SH+cat >"${TMP_DIR}/kubectl" <<'SH'#!/usr/bin/env bashset -euo pipefail@@ -49,7 +76,13 @@ restart_count() {if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "rpc-proxy-secrets" ]]; then[[ "${TEST_SOURCE_SECRET_PRESENT:-1}" == "1" ]] || exit 1- if has_output_flag "$@"; then+ if [[ "$*" == *'.metadata.resourceVersion'* ]]; then+ printf '42'+ elif [[ "$*" == *'-o json' ]]; then+ source_b64='cHVibGljLWtleQ=='+ [[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] && source_b64=''+ printf '{"metadata":{"resourceVersion":"42"},"data":{"RPC_PROXY_JWT_PUBLIC_KEY":"%s"}}' "$source_b64"+ elif has_output_flag "$@"; then[[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] || printf 'cHVibGljLWtleQ=='fiexit 0@@ -57,7 +90,9 @@ fiif [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-config" ]]; thenif has_output_flag "$@"; then- if [[ "$*" == *auth-key-applied-sha256* ]]; then+ if [[ "$*" == *'-o json' ]]; then+ printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"CLERUM_AUTH_JWT_PUBLIC_KEY":"old-key"}}'+ elif [[ "$*" == *auth-key-applied-sha256* ]]; thenprintf '%s' "${TEST_MCP_APPLIED_HASH:-old-hash}"elseprintf 'old-key'@@ -72,7 +107,9 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]exit 1fiif has_output_flag "$@"; then- if [[ "$*" == *auth-key-applied-sha256* ]]; then+ if [[ "$*" == *'-o json' ]]; then+ printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"jwt-public-key":"old-gfs-key"}}'+ elif [[ "$*" == *auth-key-applied-sha256* ]]; thenprintf '%s' "${TEST_GFS_APPLIED_HASH:-old-hash}"elseprintf 'old-gfs-key'@@ -112,6 +149,7 @@ if [[ "${1:-}" == "get" && "${2:-}" == deployment/* ]]; thenexit 1fifi+ [[ "$*" == *'.spec.replicas'* ]] && printf '1'exit 0fi@@ -176,7 +214,7 @@ echo "unexpected kubectl invocation: $*" >&2exit 99SH-chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/sleep"+chmod +x "${TMP_DIR}/kubectl" "${TMP_DIR}/sleep" "${TMP_DIR}/ps"if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" PATH="${TMP_DIR}:$PATH" \bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake >"${OUT_FILE}" 2>&1; then@@ -311,6 +349,34 @@ if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFIfigrep -q 'patch configmap gfs-config' "${LOG_FILE}"+: >"${LOG_FILE}"+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ echo "FAIL: strict GFS auth sync accepted an empty DSN before reconciliation" >&2+ exit 1+fi+grep -q 'gfs-controller-db.connection-string is empty' "${OUT_FILE}"++: >"${LOG_FILE}"+if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \+ GFS_AUTH_SYNC_ALLOW_STAGED=true \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo "FAIL: staged GFS auth sync did not defer proof until reconciliation" >&2+ exit 1+fi+grep -q 'Deferring active gfsc auth proof until GFS credentials are reconciled' "${OUT_FILE}"+if grep -q 'rollout restart deployment/gfsc-' "${LOG_FILE}"; then+ echo "FAIL: staged GFS auth sync restarted a consumer before DSN provisioning" >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi+: >"${LOG_FILE}"if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DSN_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 TEST_GFS_STATUS_FAILURE=42 \diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex c09cbeabf..a96e25360 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -35,7 +35,24 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" "$T0" \"$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"; dobash -n "$file"done-python3 -m py_compile "$GFS_FILTER"+python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'+import os+import py_compile+import sys+import tempfile++source = sys.argv[1]+tmp_root = sys.argv[2]+fd, compiled = tempfile.mkstemp(prefix="evenfire-filter-gfs-", suffix=".pyc", dir=tmp_root)+os.close(fd)+try:+ py_compile.compile(source, cfile=compiled, doraise=True)+finally:+ try:+ os.unlink(compiled)+ except FileNotFoundError:+ pass+PYgrep -Fq 't0.sh' "$T2"grep -Fq 'T0_SHELLCHECK=PASS' "$T0"grep -Fq 'npm run build' "$T0"@@ -64,15 +81,17 @@ figrep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON"grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON"grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"+grep -Fq 'T2_REQUIRED_DEPLOYMENTS' "$COMMON"grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"grep -Fq 'postgres:16-alpine' "$T1"-grep -Fq 'control-api/test/*realPostgres*.test.ts' "$T1"+grep -Fq '*realPostgres*.test.ts' "$T1"+grep -Fq 'is_isolated_control_api_file' "$T1"grep -Fq 'start_isolated_postgres' "$T1"grep -Fq 'require_isolated_control_api_files' "$T1"grep -Fq 'run_suite control-api isolated "$ISOLATED_DSN"' "$T1"if grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"; then- echo 'FAIL: control-api Real PostgreSQL suites must not use shared control-postgres' >&2+ echo 'FAIL: control-api Real PostgreSQL suites still target the shared profile database' >&2exit 1fiif grep -Fq 'run_suite control-api isolated "$ADMIN_DSN"' "$T1"; then@@ -217,21 +236,23 @@ grep -Fq 'full-bootstrap' "$PREFLIGHT" "$T2"grep -Fq 'run_pre_gate' "$T2"grep -Fq 'run_targeted_sync' "$T2"grep -Fq 'full-reconcile' "$T2"-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"+grep -Fq 'T2_SKIP_LOCK=true' "$T2"+grep -Fq 'T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"grep -Fq 'REUSE_DB=true' "$T2"grep -Fq 'T2_T0_STATUS=NOT_RUN' "$T2"grep -Fq 'T2_T1_STATUS=NOT_RUN' "$T2"grep -Fq 'T2_NP08_HCC_AUTHORIZATION_STATUS=NOT_RUN' "$T2"-grep -Fq "T2_T0_STATUS\" != PASS" "$T2"-grep -Fq "T2_T1_STATUS\" != PASS" "$T2"+grep -Fq 't2_lane_completed "$T2_T0_STATUS"' "$T2"+grep -Fq 't2_lane_completed "$T2_T1_STATUS"' "$T2"grep -Fq 'T2_HEALTHCHECK_COMMAND' "$T2"grep -Fq 'run_np08_hcc_authorization' "$T2"grep -Fq "CLERUM_PROFILE_PORTS_ENV=\"\$T2_PORTS_ENV\"" "$T2"grep -Fq 'NP08_HCC_AUTHORIZATION PASS' "$T2"grep -Fq "NP08_HCC_AUTHORIZATION=\$T2_NP08_HCC_AUTHORIZATION_STATUS" "$T2"grep -Fq 'already-synced' "$T2" "$COMMON"-grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"-grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"+grep -Fq 'T2_LOCK_TOKEN="$T2_LOCK_TOKEN"' "$T2"+grep -Fq 'T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"+grep -Fq 'T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"grep -Fq 'T2_PLAN_MODE=false' "$PREFLIGHT"grep -Fq -- '--skip-port-forwards' "$T2"grep -Fq 't2_lane_completed' "$T2"@@ -318,6 +339,13 @@ bash -c 'if T2_SKIP_LOCK=true T2_LOCK_TOKEN=wrong bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; thenexit 1fi+ saved_token="$T2_LOCK_TOKEN"+ T2_LOCK_TOKEN=wrong+ if t2_lock_validate_inherited; then+ echo "inherited lock validation returned success after a guard fired" >&2+ exit 1+ fi+ T2_LOCK_TOKEN="$saved_token"t2_lock_release 0' bash "$COMMON"if [ -e "$tmp/fenced-locks/clerum-fence.lock" ]; thendiff --git a/scripts/tests/test-minikube-t2-public-boundary.sh b/scripts/tests/test-minikube-t2-public-boundary.shindex d3e2d631a..78ceef2a1 100755--- a/scripts/tests/test-minikube-t2-public-boundary.sh+++ b/scripts/tests/test-minikube-t2-public-boundary.sh@@ -29,6 +29,12 @@ git -C "$ROOT" ls-files --others --exclude-standard -z |while IFS= read -r -d '' path; do[ -f "$ROOT/$path" ] || continueprintf '+++ b/%s\n' "$path"+ # A repository-local binary cannot be made line-safe by BSD sed. Keep the+ # path header (so sensitive filenames are still rejected), but skip binary+ # contents instead of allowing the scanner itself to fail by locale.+ if ! LC_ALL=C grep -Iq . "$ROOT/$path"; then+ continue+ fised 's/^/+/' "$ROOT/$path"done >>"$tmp"@@ -91,7 +97,12 @@ for line in diff.splitlines():match = re.search(expression, value)if match and reason == "private key" and "evidence-scanner" in current:continue- if match and reason == "credential assignment" and match.group(1) and safe_fixture.search(match.group(1)):+ if (+ match+ and reason == "credential assignment"+ and match.group(1)+ and (safe_fixture.search(match.group(1)) or "$" in match.group(1))+ ):continueif match:bad.append((current or "<unknown>", reason))diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex 568eb76bd..a75dc5be5 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -65,6 +65,7 @@ repo_env=(T2_PROFILE_ROOT="$tmp/profiles"T2_PROFILE_ENV="$profile_root/profile.env"T2_PORTS_ENV="$profile_root/ports.env"+ T2_REQUIRED_DEPLOYMENTS=gfs/gfsc-readerT2_BRANCH=feat/scenarioT2_HEAD="$feature_sha"T2_LOCK_ROOT="$tmp/locks"@@ -119,6 +120,20 @@ env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \bash -c 'source "$1"; t2_context_check' bash "$COMMON"env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \bash -c 'source "$1"; t2_context_check' bash "$COMMON"+expect_code DEVELOPMENT_SCOPE_REQUIRED public-endpoint public-endpoint \+ env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=8.8.8.8 \+ bash -c 'source "$1"; t2_context_check' bash "$COMMON"+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=192.168.1.99 \+ bash -c 'source "$1"; t2_context_check' bash "$COMMON"++identity_json='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"PROFILE"}},"status":{"addresses":[{"type":"InternalIP","address":"172.17.0.3"}]}}]}'+identity_ok="$(env "${repo_env[@]}" IDENTITY_JSON="$identity_json" \+ bash -c 'source "$1"; T2_BOOTSTRAP_REQUIRED=false; t2_mk(){ printf 172.17.0.3; }; t2_kc(){ printf "%s" "$IDENTITY_JSON" | sed "s/PROFILE/$T2_PROFILE/g"; }; t2_profile_context_identity_check; printf "%s" "$T2_CONTEXT_IDENTITY_VERIFIED"' bash "$COMMON")"+[ "$identity_ok" = true ] || fail "exact Minikube profile/node identity did not verify"+identity_wrong='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}},"status":{"addresses":[{"type":"InternalIP","address":"172.17.0.3"}]}}]}'+expect_code DEVELOPMENT_SCOPE_REQUIRED wrong-profile-identity wrong-profile-identity \+ env "${repo_env[@]}" IDENTITY_JSON="$identity_wrong" \+ bash -c 'source "$1"; T2_BOOTSTRAP_REQUIRED=false; t2_mk(){ printf 172.17.0.3; }; t2_kc(){ printf "%s" "$IDENTITY_JSON"; }; t2_profile_context_identity_check' bash "$COMMON"missing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"[ "$missing_profile_state" = full-bootstrap ] || fail "missing profile selected $missing_profile_state instead of full-bootstrap"@@ -255,6 +270,14 @@ case "$planner_unready" in*) fail 'planner full-reconcile reason does not name the unready deployment' ;;esac+ready_json='{"items":[{"metadata":{"namespace":"gfs","name":"gfsc-reader","generation":7},"spec":{"replicas":1},"status":{"observedGeneration":7,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}]}'+env "${repo_env[@]}" READY_JSON="$ready_json" \+ bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$READY_JSON"; }; t2_deployment_check' bash "$COMMON"+stale_generation_json='{"items":[{"metadata":{"namespace":"gfs","name":"gfsc-reader","generation":8},"spec":{"replicas":1},"status":{"observedGeneration":7,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}]}'+expect_code PROFILE_UNHEALTHY stale-generation stale-generation \+ env "${repo_env[@]}" INVALID_INVENTORY="$stale_generation_json" \+ bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"+# Negative pin: the standalone preflight and the final exact-head check# (T2_PLAN_MODE=false) stay fail-loud on an unready deployment.expect_code PROFILE_UNHEALTHY unready-final-preflight unready-final-preflight \@@ -404,6 +427,11 @@ if ! run_process_check "$tmp/ps-ef-wrap.out" "$tmp/pf-wrap.err"; thenfail 't2_process_check failed on a wrapper that only mentions kubectl (comm=bash)'fi+printf '%s\n' "$pf_ctl" >"$tmp/ps-ef-contextless.out"+if ! run_process_check "$tmp/ps-ef-contextless.out" "$tmp/pf-contextless.err"; then+ fail 't2_process_check rejected an unrelated context-less port-forward'+fi+grep -Fq 'REUSE_DB=true' "$ROOT/scripts/minikube/t2.sh"grep -Fq 'CONTROL_DB_RESET_PVC_UID' "$ROOT/scripts/minikube/t2.sh"@@ -414,7 +442,7 @@ cat >"$forward_tmp/bin/ps" <<'EOF'printf '%s\n' 'root 4242 1 0 00:00 ? 00:00:00 kubectl --context=profile-a -n control-plane port-forward svc/control-api 30100:8090'EOFchmod +x "$forward_tmp/bin/ps"-printf '4242\n' >"/tmp/pf-profile-a-control-api.pid"+printf '4242\nPROCESS_START=unavailable\n' >"/tmp/pf-profile-a-control-api.pid"forward_check="$(env PATH="$forward_tmp/bin:$PATH" T2_PROFILE=profile-a T2_CONTEXT=profile-a \T2_PROFILE_ROOT="$forward_tmp/profile-cache" bash -c 'source "$1"; t2_process_check; printf PASS' bash "$COMMON")"rm -f "/tmp/pf-profile-a-control-api.pid"