Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: harden final T2 recovery paths

Alfredo Lopez59e5aa79fecb
diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.md
index a691dc48d..0f3501e94 100644
--- a/docs/testing/minikube-t2-runbook.md
+++ b/docs/testing/minikube-t2-runbook.md
@@ -93,7 +93,9 @@ profile-level lock prevents two processes from mutating one profile at the
same time. The lock record contains only local metadata and is removed by
success, failure, timeout, and interrupt traps. A live owner produces
`PROFILE_BUSY`; stale metadata may be reclaimed only after its recorded PID is
-no longer running.
+no longer running. Reclaimers serialize through an atomic `.reclaim` directory
+inside that exact profile lock. A concurrent loser fails closed and never
+removes either the stale directory or the winner's replacement lock.
The pre-gate marker must contain the current worktree identifier, exact `HEAD`,
cluster fingerprint, and image coordinate. A mismatch stops with a stable
@@ -101,9 +103,12 @@ error code instead of allowing a mixed-commit run.
### Orphaned lock recovery
-If `PROFILE_BUSY` reports that the lock has no valid owner PID, first verify
-that no T2 process owns the profile. After that check, remove only the exact
-profile lock directory and retry the same command:
+If `PROFILE_BUSY` reports that the lock has no valid owner PID, or that a stale
+reclaim is already in progress, first verify that neither the recorded owner
+nor a reclaimer process owns the profile. A `.reclaim` directory can remain
+after a reclaimer is killed and is intentionally not auto-reclaimed. After
+those checks, remove only the exact profile lock directory and retry the same
+command:
```bash
rm -rf -- "$T2_LOCK_ROOT/<profile>.lock"
@@ -157,7 +162,12 @@ live non-terminating unready reader pod). A reader pod also fails closed
when `gfs-config.jwt-public-key` is empty — the overlay re-applies the base
ConfigMap with an empty value — so `full-setup.sh` and `pre-gate-sync` re-run
`scripts/minikube/sync-auth-key.sh` before each GFS reconcile; otherwise no
-new reader pod can start and the readiness wait can only time out.
+new reader pod can start and the readiness wait can only time out. Auth sync
+commits a SHA-256 convergence annotation only after every active managed
+mcp-host and the exact `gfsc-writer`/`gfsc-reader` consumers prove—through a
+stdin-only in-process check—that they loaded the target public key. A matching
+ConfigMap without that annotation is an interrupted rollout and is resumed;
+it is never treated as converged.
`pre-gate-sync` provisions GFS serving with the same opt-ins only in the
`minikube-t2` transition (including its "no cluster sync required" fast path).
Other platform security gates refresh MCP auth with `--skip-gfs` and do not
diff --git a/scripts/minikube/pre-gate-incremental.sh b/scripts/minikube/pre-gate-incremental.sh
index e67616797..a8e20094b 100755
--- a/scripts/minikube/pre-gate-incremental.sh
+++ b/scripts/minikube/pre-gate-incremental.sh
@@ -141,7 +141,10 @@ incremental_classify_path() {
rpc-proxy/*) incremental_add_target rpc-proxy rpc-proxy rpc-proxy ;;
mcp-host/*) incremental_add_target mcp-host mcp-host chatllm ;;
host-context-controller/*) incremental_add_target host-context-controller control-plane host-context-controller ;;
- gfs-controller/*) incremental_add_target gfs-controller control-plane host-context-controller ;;
+ gfs-controller/*)
+ incremental_add_target gfs-controller gfs gfsc-writer
+ incremental_add_target gfs-controller gfs gfsc-reader
+ ;;
workflow-recipes/*|packages/workflow-runtime-core/*|packages/workflow-sdk/*)
incremental_add_target workflow control-plane workflow-recipes
;;
@@ -385,7 +388,7 @@ incremental_abort_unshadowable() {
}
incremental_build_images() {
- local target selector
+ local target selector built="|"
if [[ "${IMAGE_SOURCE}" == "ghcr" ]]; then
incremental_build_images_ghcr
@@ -409,6 +412,8 @@ incremental_build_images() {
for target in "${INCREMENTAL_TARGETS[@]}"; do
selector="${target%%|*}"
+ [[ "${built}" == *"|${selector}|"* ]] && continue
+ built+="${selector}|"
log "Building only image selector ${selector}"
MINIKUBE_PROFILE="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/build-images.sh" "--only=${selector}"
@@ -416,7 +421,7 @@ incremental_build_images() {
}
incremental_build_images_ghcr() {
- local target selector
+ local target selector built="|"
if [[ "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ]]; then
incremental_abort_unshadowable
@@ -444,6 +449,8 @@ incremental_build_images_ghcr() {
for target in "${INCREMENTAL_TARGETS[@]}"; do
selector="${target%%|*}"
+ [[ "${built}" == *"|${selector}|"* ]] && continue
+ built+="${selector}|"
log "Building only image selector ${selector}"
MINIKUBE_PROFILE="${PROFILE}" \
bash "${PROJECT_DIR}/scripts/minikube/build-images.sh" "--only=${selector}"
@@ -454,7 +461,7 @@ incremental_build_images_ghcr() {
}
incremental_restart_targets() {
- local target selector remainder namespace deployment deployment_key
+ local target selector remainder namespace deployment deployment_key deployment_probe
local restarted="|"
for target in "${INCREMENTAL_TARGETS[@]}"; do
@@ -466,13 +473,27 @@ incremental_restart_targets() {
[[ "${restarted}" == *"${deployment_key}"* ]] && continue
restarted+="${namespace}/${deployment}|"
- if ! ${KC} get deployment "${deployment}" -n "${namespace}" >/dev/null 2>&1; then
- log "Skipping absent ${namespace}/${deployment} for image selector ${selector}"
- continue
+ deployment_probe=""
+ if ! deployment_probe="$(${KC} get deployment "${deployment}" -n "${namespace}" 2>&1)"; then
+ if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
+ log "Skipping absent ${namespace}/${deployment} for image selector ${selector}"
+ continue
+ fi
+ log "ERROR: unable to inspect ${namespace}/${deployment} for image selector ${selector}: ${deployment_probe}"
+ return 1
fi
${KC} rollout restart "deployment/${deployment}" -n "${namespace}" >/dev/null
log "Restarted ${namespace}/${deployment} for image selector ${selector}"
- rollout_if_present "${namespace}" "${deployment}"
+ if [[ "${namespace}/${deployment}" == "gfs/gfsc-reader" ]]; then
+ # HCC owns the reader template and strips kubectl's restartedAt
+ # annotation. The canonical shim judges Ready replicas/pods instead of
+ # waiting forever on a deployment generation HCC can rewrite.
+ PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
+ CONTEXT="${PROFILE}" ${KC} rollout status "deployment/${deployment}" \
+ -n "${namespace}" --timeout=120s >/dev/null
+ else
+ rollout_if_present "${namespace}" "${deployment}"
+ fi
done
}
diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.sh
index a375528f7..d189ec3f4 100755
--- a/scripts/minikube/pre-gate-sync.sh
+++ b/scripts/minikube/pre-gate-sync.sh
@@ -193,20 +193,51 @@ restore_control_api() {
}
restore_pre_gate_writers() {
- local status=$?
+ local status=0
# Restore Control API first so it can serve the final readiness checks; WRC
# remains last because it is the consumer that must stay stopped until every
# migration, writer restart, and legacy-policy gate has completed.
restore_control_api || status=1
restore_workflow_reconciler || status=1
- t2_lock_release "$status" || status=$?
return "$status"
}
+finalize_pre_gate_sync() {
+ local status=$? restore_status=0
+
+ # An EXIT trap's return value does not replace the script's exit status. Exit
+ # explicitly so a failed writer restore cannot turn a successful sync into a
+ # green exact-head attestation. Disable the trap first to avoid recursion.
+ trap - EXIT
+ if ! restore_pre_gate_writers; then
+ restore_status=1
+ fi
+ if [[ "${status}" -eq 0 && "${restore_status}" -ne 0 ]]; then
+ status=1
+ fi
+ # t2_lock_release returns the status passed to it; pass zero so an existing
+ # non-zero script status is preserved rather than misclassified as a lock
+ # cleanup failure.
+ t2_lock_release 0 || status=1
+ exit "${status}"
+}
+
+commit_cluster_sync_state() {
+ local cluster_fingerprint="$1" infra_fingerprint="$2"
+
+ # The exact-head marker is a certification artifact, not merely progress.
+ # Restore every fenced writer before publishing it so a cleanup failure can
+ # never leave a marker that claims the cluster is ready at this HEAD.
+ restore_pre_gate_writers || return 1
+ persist_cluster_marker "${cluster_fingerprint}" "${infra_fingerprint}" || return 1
+ persist_state cluster "${cluster_fingerprint}" || return 1
+ persist_state infra "${infra_fingerprint}" || return 1
+}
+
if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
t2_mutation_lock
fi
-trap restore_pre_gate_writers EXIT
+trap finalize_pre_gate_sync EXIT
fingerprint_dir() {
pre_gate_marker_fingerprint_dir "${PROJECT_DIR}" "$1"
@@ -740,9 +771,7 @@ if [[ "${cluster_changed}" == "true" ]]; then
log "ERROR: unable to recompute the infrastructure fingerprint after deployment; refusing to stamp the marker"
exit 1
fi
- persist_cluster_marker "${cluster_fingerprint}" "${infra_fingerprint}"
- persist_state cluster "${cluster_fingerprint}"
- persist_state infra "${infra_fingerprint}"
+ commit_cluster_sync_state "${cluster_fingerprint}" "${infra_fingerprint}"
log "Pre-gate cluster sync complete"
else
log "No cluster sync required before ${GATE_NAME}"
diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.sh
index ad9168795..d4514e386 100755
--- a/scripts/minikube/sync-auth-key.sh
+++ b/scripts/minikube/sync-auth-key.sh
@@ -58,6 +58,11 @@ CONFIG_NAMESPACE="mcp-host"
SOURCE_KEY="RPC_PROXY_JWT_PUBLIC_KEY"
TARGET_KEY="CLERUM_AUTH_JWT_PUBLIC_KEY"
MCP_HOST_ROLLOUT_TIMEOUT="180s"
+# This ConfigMap annotation is the durable commit record for a completed
+# rollout. It is written only after every active consumer proves its in-process
+# environment contains the source key; matching data without this hash resumes.
+APPLIED_HASH_ANNOTATION="clerum.io/auth-key-applied-sha256"
+MCP_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"
# gfsc verifies gfs access tokens with the SAME platform public key (open core
# reuses one keypair). gfsc fails closed without it (empty key ⇒ no read serving),
@@ -65,7 +70,6 @@ MCP_HOST_ROLLOUT_TIMEOUT="180s"
GFS_CONFIGMAP="gfs-config"
GFS_NAMESPACE="gfs"
GFS_TARGET_KEY="jwt-public-key"
-GFS_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"
log() { printf '[sync-auth-key] %s\n' "$*"; }
die() {
@@ -74,18 +78,19 @@ die() {
}
rollout_restart_with_retry() {
- local deployment="$1"
+ local namespace="$1"
+ local deployment="$2"
local attempt
local output
for attempt in 1 2 3; do
- if output="$("${KCTL[@]}" rollout restart "deployment/${deployment}" -n "${CONFIG_NAMESPACE}" 2>&1)"; then
+ if output="$("${KCTL[@]}" rollout restart "deployment/${deployment}" -n "${namespace}" 2>&1)"; then
[[ -n "${output}" ]] && printf '%s\n' "${output}"
return 0
fi
if [[ "${output}" == *"within the past second"* && "${attempt}" != "3" ]]; then
- log "Retrying rollout restart for ${CONFIG_NAMESPACE}/${deployment} after recent restart" >&2
+ log "Retrying rollout restart for ${namespace}/${deployment} after recent restart" >&2
sleep 2
continue
fi
@@ -105,6 +110,72 @@ print(json.dumps({"data": {os.environ["TARGET_KEY"]: os.environ["SOURCE_VALUE"]}
PY
}
+make_applied_hash_patch() {
+ APPLIED_HASH="$1" APPLIED_HASH_ANNOTATION="${APPLIED_HASH_ANNOTATION}" python3 - <<'PY'
+import json
+import os
+
+print(json.dumps({
+ "metadata": {
+ "annotations": {
+ os.environ["APPLIED_HASH_ANNOTATION"]: os.environ["APPLIED_HASH"]
+ }
+ }
+}))
+PY
+}
+
+read_applied_hash() {
+ local namespace="$1" configmap="$2"
+ "${KCTL[@]}" get configmap "${configmap}" -n "${namespace}" \
+ -o 'jsonpath={.metadata.annotations.clerum\.io/auth-key-applied-sha256}'
+}
+
+mark_key_applied() {
+ local namespace="$1" configmap="$2"
+ "${KCTL[@]}" patch configmap "${configmap}" -n "${namespace}" --type=merge \
+ -p "$(make_applied_hash_patch "${source_hash}")" >/dev/null
+}
+
+consumer_pods_use_key() {
+ local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"
+ local rows pod ready deleting pod_count=0
+
+ if ! rows="$("${KCTL[@]}" get pods -n "${namespace}" -l "${selector}" -o \
+ 'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' 2>&1)"; then
+ log "cannot inspect ${consumer} pods while checking auth-key convergence: ${rows}" >&2
+ return 1
+ fi
+
+ while IFS='|' read -r pod ready deleting; do
+ [[ -n "${pod}" ]] || continue
+ [[ -z "${deleting}" ]] || continue
+ if [[ "${ready}" != "True" ]]; then
+ log "${consumer} pod ${namespace}/${pod} is not Ready while checking auth-key convergence" >&2
+ return 1
+ fi
+ pod_count=$((pod_count + 1))
+ if ! printf '%s' "${source_key}" | "${KCTL[@]}" exec -i "${pod}" -n "${namespace}" -c "${container}" -- \
+ node -e 'const fs=require("fs");const expected=fs.readFileSync(0,"utf8");process.exit(process.env[process.argv[1]]===expected?0:42)' \
+ "${env_name}" >/dev/null; then
+ log "${consumer} pod ${namespace}/${pod} has not consumed the target auth key" >&2
+ return 1
+ fi
+ done <<<"${rows}"
+
+ if [[ "${pod_count}" -eq 0 ]]; then
+ log "${consumer} has a deployment but no active pod proved consumption of the target auth key" >&2
+ return 1
+ fi
+}
+
+verify_consumer_pods_use_key() {
+ local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"
+ if ! consumer_pods_use_key "${namespace}" "${selector}" "${container}" "${env_name}" "${consumer}"; then
+ die "${consumer} did not prove consumption of the target auth key after rollout"
+ fi
+}
+
# The public key is the SAME for every consumer (one platform keypair), so it is
# fetched once and synced into each consumer's ConfigMap below.
source_probe=""
@@ -128,6 +199,7 @@ if [[ -z "${source_key}" ]]; then
fi
die "auth source key ${SOURCE_KEY} is empty; refusing to mutate consumers"
fi
+source_hash="$(printf '%s' "${source_key}" | shasum -a 256 | awk '{print $1}')"
if [[ "${REQUIRE_GFS}" == "true" ]] && \
! gfs_target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; then
die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable: ${gfs_target_probe}"
@@ -150,27 +222,60 @@ sync_mcp_host() {
fi
die "cannot inspect auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${target_probe}"
fi
- local current_key
+ local current_key applied_hash
if ! current_key="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" -o "jsonpath={.data.${TARGET_KEY}}" 2>&1)"; then
die "cannot read auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${current_key}"
fi
- if [[ "${source_key}" == "${current_key}" ]]; then
- log "${CONFIGMAP}.${TARGET_KEY} already matches source; no patch needed"
- return 0
+ if ! applied_hash="$(read_applied_hash "${CONFIG_NAMESPACE}" "${CONFIGMAP}" 2>&1)"; then
+ die "cannot read auth convergence marker from ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${applied_hash}"
fi
- log "Patching ${CONFIGMAP}.${TARGET_KEY}"
- "${KCTL[@]}" patch configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${TARGET_KEY}")" >/dev/null
- local found=0 deployment
- for deployment in chatllm mcp-host; do
- if "${KCTL[@]}" get "deployment/${deployment}" -n "${CONFIG_NAMESPACE}" >/dev/null 2>&1; then
- found=1
- log "Restarting ${CONFIG_NAMESPACE}/${deployment} after auth key drift"
- rollout_restart_with_retry "${deployment}" >/dev/null
- "${KCTL[@]}" rollout status "deployment/${deployment}" -n "${CONFIG_NAMESPACE}" --timeout="${MCP_HOST_ROLLOUT_TIMEOUT}"
+ local force_rollout=false
+ if [[ "${source_key}" != "${current_key}" ]]; then
+ log "Patching ${CONFIGMAP}.${TARGET_KEY}"
+ "${KCTL[@]}" patch configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${TARGET_KEY}")" >/dev/null
+ force_rollout=true
+ else
+ if [[ "${applied_hash}" == "${source_hash}" ]]; then
+ log "${CONFIGMAP}.${TARGET_KEY} matches source; checking active consumers before declaring convergence"
+ else
+ log "${CONFIGMAP}.${TARGET_KEY} matches source but consumer attestation is pending; resuming convergence"
fi
- done
- if [[ "${found}" == "0" ]]; then
+ fi
+ local deployment_resources deployment resource_name
+ local deployments=()
+ if ! deployment_resources="$("${KCTL[@]}" get deployment -l "${MCP_DEPLOY_SELECTOR}" \
+ -n "${CONFIG_NAMESPACE}" -o name 2>&1)"; then
+ die "cannot enumerate mcp-host consumers before auth key rollout: ${deployment_resources}"
+ fi
+ while IFS= read -r resource_name; do
+ [[ -n "${resource_name}" ]] || continue
+ case "${resource_name}" in
+ deployment/*|deployment.apps/*) deployment="${resource_name#*/}" ;;
+ *) die "unexpected mcp-host consumer resource from Kubernetes: ${resource_name}" ;;
+ esac
+ [[ -n "${deployment}" ]] || die "Kubernetes returned an empty mcp-host deployment name"
+ deployments+=("${deployment}")
+ done <<<"${deployment_resources}"
+
+ if (( ${#deployments[@]} == 0 )); then
log "No mcp-host deployments exist yet; auth key is synced for future pods"
+ else
+ for deployment in "${deployments[@]}"; do
+ if [[ "${force_rollout}" != true ]] && \
+ consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \
+ "${TARGET_KEY}" "mcp-host deployment/${deployment}"; then
+ log "${CONFIG_NAMESPACE}/${deployment} already consumes the target auth key"
+ continue
+ fi
+ log "Restarting ${CONFIG_NAMESPACE}/${deployment} after auth key drift"
+ rollout_restart_with_retry "${CONFIG_NAMESPACE}" "${deployment}" >/dev/null
+ "${KCTL[@]}" rollout status "deployment/${deployment}" -n "${CONFIG_NAMESPACE}" --timeout="${MCP_HOST_ROLLOUT_TIMEOUT}"
+ verify_consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \
+ "${TARGET_KEY}" "mcp-host deployment/${deployment}"
+ done
+ fi
+ if [[ "${applied_hash}" != "${source_hash}" ]]; then
+ mark_key_applied "${CONFIG_NAMESPACE}" "${CONFIGMAP}"
fi
}
@@ -187,53 +292,92 @@ sync_gfs() {
fi
die "cannot inspect auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${target_probe}"
fi
- local current_key
+ local current_key applied_hash
if ! current_key="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" -o "jsonpath={.data.${GFS_TARGET_KEY}}" 2>&1)"; then
die "cannot read auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${current_key}"
fi
- if [[ "${source_key}" == "${current_key}" ]]; then
- log "${GFS_CONFIGMAP}.${GFS_TARGET_KEY} already matches source; no patch needed"
+ if ! applied_hash="$(read_applied_hash "${GFS_NAMESPACE}" "${GFS_CONFIGMAP}" 2>&1)"; then
+ die "cannot read auth convergence marker from ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${applied_hash}"
+ fi
+ local force_rollout=false
+ if [[ "${source_key}" != "${current_key}" ]]; then
+ log "Patching ${GFS_CONFIGMAP}.${GFS_TARGET_KEY}"
+ "${KCTL[@]}" patch configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${GFS_TARGET_KEY}")" >/dev/null
+ force_rollout=true
+ else
+ if [[ "${applied_hash}" == "${source_hash}" ]]; then
+ log "${GFS_CONFIGMAP}.${GFS_TARGET_KEY} matches source; checking active consumers before declaring convergence"
+ else
+ log "${GFS_CONFIGMAP}.${GFS_TARGET_KEY} matches source but consumer attestation is pending; resuming convergence"
+ fi
+ fi
+
+ local deployments=() rollout_deployments=() deployment deployment_probe role selector rollout_output dsn_encoded dsn
+ for deployment in gfsc-writer gfsc-reader; do
+ deployment_probe=""
+ if deployment_probe="$("${KCTL[@]}" get "deployment/${deployment}" -n "${GFS_NAMESPACE}" 2>&1)"; then
+ deployments+=("${deployment}")
+ elif [[ "${deployment_probe}" != *NotFound* && "${deployment_probe}" != *"not found"* ]]; then
+ die "cannot inspect ${GFS_NAMESPACE}/deployment/${deployment} before auth key rollout: ${deployment_probe}"
+ fi
+ done
+
+ if (( ${#deployments[@]} == 0 )); then
+ log "No gfsc deployments exist yet; auth key is synced for future pods"
+ if [[ "${applied_hash}" != "${source_hash}" ]]; then
+ mark_key_applied "${GFS_NAMESPACE}" "${GFS_CONFIGMAP}"
+ fi
return 0
fi
- log "Patching ${GFS_CONFIGMAP}.${GFS_TARGET_KEY}"
- "${KCTL[@]}" patch configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${GFS_TARGET_KEY}")" >/dev/null
- local dsn_encoded dsn
- if ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then
- if [[ "${dsn_encoded}" == *NotFound* || "${dsn_encoded}" == *"not found"* ]]; then
- log "Skipping gfsc restart after auth key drift (gfs-controller-db Secret is not populated yet)"
- return 0
+
+ for deployment in "${deployments[@]}"; do
+ role="${deployment#gfsc-}"
+ selector="app=gfs-controller,clerum.io/gfsc-role=${role}"
+ if [[ "${force_rollout}" != true ]] && \
+ consumer_pods_use_key "${GFS_NAMESPACE}" "${selector}" gfsc \
+ GFS_JWT_PUBLIC_KEY "gfsc deployment/${deployment}"; then
+ log "${GFS_NAMESPACE}/${deployment} already consumes the target auth key"
+ continue
+ fi
+ rollout_deployments+=("${deployment}")
+ done
+
+ if (( ${#rollout_deployments[@]} == 0 )); then
+ log "All active gfsc deployments already consume the target auth key"
+ if [[ "${applied_hash}" != "${source_hash}" ]]; then
+ mark_key_applied "${GFS_NAMESPACE}" "${GFS_CONFIGMAP}"
fi
+ return 0
+ fi
+
+ if ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then
die "cannot inspect ${GFS_NAMESPACE}/gfs-controller-db before gfsc restart: ${dsn_encoded}"
fi
if [[ -z "${dsn_encoded}" ]]; then
- log "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"
- return 0
+ die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is empty; refusing to attest gfsc auth convergence"
fi
if ! dsn="$(printf '%s' "${dsn_encoded}" | base64 -d 2>&1)"; then
- die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is not valid base64: ${dsn}"
+ die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is not valid base64"
fi
if [[ -z "${dsn}" ]]; then
- log "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"
- return 0
+ die "${GFS_NAMESPACE}/gfs-controller-db.connection-string decodes empty; refusing to attest gfsc auth convergence"
fi
- # gfsc reads the key at boot; restart the writer+reader so the new key takes
- # effect (label-based — the reconciler owns the deployment names).
- local deployments deployment rollout_output
- if ! deployments="$("${KCTL[@]}" get deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" -o name 2>&1)"; then
- die "cannot inspect gfsc deployments after auth key drift: ${deployments}"
- fi
- if [[ -n "${deployments}" ]]; then
- log "Restarting gfsc (${GFS_DEPLOY_SELECTOR}) after auth key drift"
- "${KCTL[@]}" rollout restart deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" >/dev/null
- while IFS= read -r deployment; do
- [[ -n "${deployment}" ]] || continue
- if ! rollout_output="$(PATH="${SCRIPT_DIR}/gfs-rollout-shim:${PATH}" CONTEXT="${PROFILE}" \
- "${KCTL[@]}" rollout status "${deployment}" -n "${GFS_NAMESPACE}" --timeout=90s 2>&1)"; then
- die "gfsc deployment ${deployment} did not become Ready after auth key drift: ${rollout_output}"
- fi
- done <<<"${deployments}"
- else
- log "No gfsc deployments exist yet; auth key is synced for future pods"
+ unset dsn dsn_encoded
+
+ for deployment in "${rollout_deployments[@]}"; do
+ role="${deployment#gfsc-}"
+ selector="app=gfs-controller,clerum.io/gfsc-role=${role}"
+ log "Restarting ${GFS_NAMESPACE}/${deployment} after auth key drift"
+ rollout_restart_with_retry "${GFS_NAMESPACE}" "${deployment}" >/dev/null
+ if ! rollout_output="$(PATH="${SCRIPT_DIR}/gfs-rollout-shim:${PATH}" CONTEXT="${PROFILE}" \
+ "${KCTL[@]}" rollout status "deployment/${deployment}" -n "${GFS_NAMESPACE}" --timeout=90s 2>&1)"; then
+ die "gfsc deployment deployment/${deployment} did not become Ready after auth key drift: ${rollout_output}"
+ fi
+ verify_consumer_pods_use_key "${GFS_NAMESPACE}" "${selector}" gfsc \
+ GFS_JWT_PUBLIC_KEY "gfsc deployment/${deployment}"
+ done
+ if [[ "${applied_hash}" != "${source_hash}" ]]; then
+ mark_key_applied "${GFS_NAMESPACE}" "${GFS_CONFIGMAP}"
fi
}
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 45dbc7627..99a215739 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -643,6 +643,7 @@ t2_lock_profile_id_check() {
if [[ ! "$T2_PROFILE" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
T2_NEXT_COMMAND='use the generated profile name without path separators or shell metacharacters'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'profile lock identifier is not a safe local profile name'
+ return 1
fi
}
@@ -664,7 +665,7 @@ t2_lock_process_matches() {
t2_lock_acquire() {
local process_start
- t2_lock_profile_id_check
+ t2_lock_profile_id_check || return 1
mkdir -p "$T2_LOCK_ROOT"
T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
T2_LOCK_KEY="$(t2_lock_key)"
@@ -672,24 +673,58 @@ t2_lock_acquire() {
if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ]; then
T2_NEXT_COMMAND='remove the unsafe profile-lock path manually after verifying the profile owner'
t2_fail PROFILE_BUSY "profile lock path is not a directory: $T2_LOCK_DIR"
+ return 1
fi
- local existing_pid existing_token existing_start
+ local existing_pid existing_token existing_start reclaim_dir
existing_pid="$(t2_lock_owner_value PID || true)"
existing_token="$(t2_lock_owner_value TOKEN || true)"
existing_start="$(t2_lock_owner_value PROCESS_START || true)"
if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; then
T2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
t2_fail PROFILE_BUSY "profile $T2_PROFILE has an orphaned lock without a valid owner identity"
+ return 1
fi
if t2_lock_process_matches "$existing_pid"; then
T2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"
t2_fail PROFILE_BUSY "profile $T2_PROFILE is locked by PID $existing_pid"
+ return 1
+ fi
+
+ # Reclaim ownership is itself acquired atomically inside the stale lock.
+ # With two concurrent reclaimers, exactly one can create this directory;
+ # every loser fails closed before it can remove either the stale lock or a
+ # replacement lock created by the winner. If a reclaimer is killed while
+ # holding this claim, the marker deliberately remains and requires the
+ # documented orphan-lock recovery rather than permitting an unsafe retry.
+ reclaim_dir="$T2_LOCK_DIR/.reclaim"
+ if ! mkdir "$reclaim_dir" 2>/dev/null; then
+ T2_NEXT_COMMAND='wait for the stale-lock reclaimer to finish; if it died, follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
+ t2_fail PROFILE_BUSY "profile $T2_PROFILE stale lock is already being reclaimed"
+ return 1
+ fi
+
+ # Revalidate after winning the claim. A changed or live owner means the
+ # original stale observation is no longer authoritative.
+ existing_pid="$(t2_lock_owner_value PID || true)"
+ existing_token="$(t2_lock_owner_value TOKEN || true)"
+ existing_start="$(t2_lock_owner_value PROCESS_START || true)"
+ if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; then
+ T2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
+ t2_fail PROFILE_BUSY "profile $T2_PROFILE owner changed while its stale lock was being reclaimed"
+ return 1
+ fi
+ if t2_lock_process_matches "$existing_pid"; then
+ rmdir "$reclaim_dir" 2>/dev/null || true
+ T2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"
+ t2_fail PROFILE_BUSY "profile $T2_PROFILE became live while its stale lock was being reclaimed"
+ return 1
fi
rm -rf -- "$T2_LOCK_DIR"
fi
mkdir "$T2_LOCK_DIR" 2>/dev/null || {
T2_NEXT_COMMAND='retry after the profile lock is released'
t2_fail PROFILE_BUSY "unable to acquire the profile lock: $T2_PROFILE"
+ return 1
}
umask 077
T2_LOCK_TOKEN="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"
@@ -760,7 +795,7 @@ t2_mutation_lock() {
}
t2_lock_release() {
- local incoming_status=$? status owner_pid owner_token
+ local incoming_status=$? status owner_pid owner_token cleanup_status=0
if [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fi
if [ "$T2_LOCK_RELEASED" = true ]; then
return "$status"
@@ -770,13 +805,16 @@ t2_lock_release() {
owner_pid="$(t2_lock_owner_value PID || true)"
owner_token="$(t2_lock_owner_value TOKEN || true)"
if [ "$owner_pid" = "$$" ] && [ -n "$T2_LOCK_TOKEN" ] && [ "$owner_token" = "$T2_LOCK_TOKEN" ]; then
- rm -rf -- "$T2_LOCK_DIR"
+ rm -rf -- "$T2_LOCK_DIR" || cleanup_status=1
fi
fi
T2_LOCK_HELD=false
if [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; then
t2_evidence_write failure FAIL "$T2_ERROR_CODE"
fi
+ if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
+ status="$cleanup_status"
+ fi
return "$status"
}
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index 7fb2fa672..30712ea71 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -34,11 +34,22 @@ T2_FINAL_PREFLIGHT_OUTPUT="$T2_TMP_ROOT/evenfire-t2-final-preflight.$$.out"
export T2_PROJECT_DIR T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_CONTEXT T2_LOCK_TOKEN
cleanup_plan() {
- local status=$?
- if [ -n "$T2_PLAN_TMP" ] && [ -f "$T2_PLAN_TMP" ]; then rm -f "$T2_PLAN_TMP"; fi
- if [ -f "$T2_FINAL_PREFLIGHT_OUTPUT" ]; then rm -f "$T2_FINAL_PREFLIGHT_OUTPUT"; fi
- if [ -f "$T2_T1_OUTPUT" ]; then rm -f "$T2_T1_OUTPUT"; fi
- t2_lock_release "$status"
+ local status=$? cleanup_status=0
+ trap - EXIT
+ if [ -n "$T2_PLAN_TMP" ] && [ -f "$T2_PLAN_TMP" ]; then
+ rm -f "$T2_PLAN_TMP" || cleanup_status=1
+ fi
+ if [ -f "$T2_FINAL_PREFLIGHT_OUTPUT" ]; then
+ rm -f "$T2_FINAL_PREFLIGHT_OUTPUT" || cleanup_status=1
+ fi
+ if [ -f "$T2_T1_OUTPUT" ]; then
+ rm -f "$T2_T1_OUTPUT" || cleanup_status=1
+ fi
+ t2_lock_release "$status" || cleanup_status=1
+ if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
+ status=1
+ fi
+ exit "$status"
}
trap cleanup_plan EXIT
@@ -157,37 +168,12 @@ run_bootstrap_or_reconcile() {
}
run_targeted_sync() {
- local changed path selected=false
- changed="$(git -C "$T2_PROJECT_DIR" diff --name-only "$T2_ORIGIN_DEV...$T2_HEAD")"
- while IFS= read -r path; do
- [ -z "$path" ] && continue
- case "$path" in
- control-api/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=control-api NS=control-plane MINIKUBE_DEPLOYMENT=control-api
- selected=true ;;
- gfs-controller/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=gfs-controller NS=gfs MINIKUBE_DEPLOYMENT=gfs-controller
- selected=true ;;
- external-rest-api/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=external-rest-api NS=profiles MINIKUBE_DEPLOYMENT=external-rest-api
- selected=true ;;
- rpc-proxy/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=rpc-proxy NS=rpc-proxy MINIKUBE_DEPLOYMENT=rpc-proxy
- selected=true ;;
- mcp-host/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=mcp-host NS=mcp-host MINIKUBE_DEPLOYMENT=chatllm
- selected=true ;;
- control-ui/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=control-ui NS=control-plane MINIKUBE_DEPLOYMENT=control-ui
- selected=true ;;
- profile-ui/*)
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-deploy-service SVC=profile-ui NS=profiles MINIKUBE_DEPLOYMENT=profile-ui
- selected=true ;;
- esac
- done <<< "$changed"
- if [ "$selected" != true ]; then
- printf '[minikube-t2] no service image requires a targeted deployment\n'
- fi
+ # pre-gate-sync owns the targeted plan and mutation. Keeping a second mapping
+ # here caused the T2 transition to deploy twice and, for GFS, referenced a
+ # deployment that does not exist. The canonical path below classifies the
+ # exact diff, builds each image selector once, and restarts its real consumer
+ # deployment(s) before publishing the exact-head marker.
+ printf '[minikube-t2] targeted mutation delegated to canonical pre-gate-sync\n'
}
run_pre_gate() {
diff --git a/scripts/tests/test-minikube-pre-gate-restore.sh b/scripts/tests/test-minikube-pre-gate-restore.sh
index 865fb65ce..e4e3b80af 100755
--- a/scripts/tests/test-minikube-pre-gate-restore.sh
+++ b/scripts/tests/test-minikube-pre-gate-restore.sh
@@ -60,4 +60,58 @@ CONTROL_API_FENCED=true
restore_control_api || fail 'Control API restore rejected a successful scale and rollout'
[ "$CONTROL_API_FENCED" = false ] || fail 'Control API fence remained armed after a successful restore'
-printf 'PASS: pre-gate restores fail closed and preserve writer fences\n'
+# EXIT traps do not propagate their return value in Bash. The pre-gate
+# finalizer must therefore exit explicitly when a last-chance restore fails.
+set +e
+PRE_GATE_SYNC_CONFIG_ONLY=true MINIKUBE_PROFILE=restore-contract IMAGE_SOURCE=local \
+ bash -c '
+ script="$1"; set --; source "$script" >/dev/null
+ restore_pre_gate_writers() { return 31; }
+ ' bash "$ROOT/scripts/minikube/pre-gate-sync.sh" >/dev/null 2>&1
+exit_status=$?
+set -e
+[ "$exit_status" -ne 0 ] || fail 'EXIT cleanup failure was hidden behind a zero pre-gate status'
+
+set +e
+PRE_GATE_SYNC_CONFIG_ONLY=true MINIKUBE_PROFILE=restore-contract IMAGE_SOURCE=local \
+ bash -c '
+ script="$1"; set --; source "$script" >/dev/null
+ restore_pre_gate_writers() { return 0; }
+ t2_lock_release() { return 41; }
+ ' bash "$ROOT/scripts/minikube/pre-gate-sync.sh" >/dev/null 2>&1
+exit_status=$?
+set -e
+[ "$exit_status" -ne 0 ] || fail 'EXIT lock cleanup failure was hidden behind a zero pre-gate status'
+
+set +e
+PRE_GATE_SYNC_CONFIG_ONLY=true MINIKUBE_PROFILE=restore-contract IMAGE_SOURCE=local \
+ bash -c '
+ script="$1"; set --; source "$script" >/dev/null
+ restore_pre_gate_writers() { return 0; }
+ exit 23
+ ' bash "$ROOT/scripts/minikube/pre-gate-sync.sh" >/dev/null 2>&1
+exit_status=$?
+set -e
+[ "$exit_status" -eq 23 ] || fail "EXIT cleanup replaced original failure status 23 with ${exit_status}"
+
+# Exact-head state is committed only after both fenced writers are restored.
+events=""
+restore_pre_gate_writers() { events+="restore "; return 1; }
+persist_cluster_marker() { events+="marker "; }
+persist_state() { events+="state-$1 "; }
+if commit_cluster_sync_state cluster-fingerprint infra-fingerprint; then
+ fail 'cluster sync state committed despite writer restore failure'
+fi
+[ "$events" = "restore " ] || fail "marker/state mutation ran after failed restore: ${events}"
+
+events=""
+restore_pre_gate_writers() { events+="restore "; return 0; }
+commit_cluster_sync_state cluster-fingerprint infra-fingerprint || \
+ fail 'cluster sync state rejected a successful writer restore'
+[ "$events" = "restore marker state-cluster state-infra " ] || \
+ fail "cluster marker was not persisted after restore in the required order: ${events}"
+
+# Leave the sourced script's EXIT finalizer with a successful cleanup stub.
+restore_pre_gate_writers() { return 0; }
+
+printf 'PASS: pre-gate cleanup propagates failures and exact-head state commits only after restore\n'
diff --git a/scripts/tests/test-minikube-pre-gate-sync-state.sh b/scripts/tests/test-minikube-pre-gate-sync-state.sh
index 0affe5460..82efc12be 100755
--- a/scripts/tests/test-minikube-pre-gate-sync-state.sh
+++ b/scripts/tests/test-minikube-pre-gate-sync-state.sh
@@ -250,7 +250,9 @@ if contains 'fence_control_api()' &&
contains 'restore_control_api()' &&
contains 'fence_workflow_reconciler' &&
contains 'fence_control_api' &&
- contains 'trap restore_pre_gate_writers EXIT' &&
+ contains 'trap finalize_pre_gate_sync EXIT' &&
+ contains 'restore_pre_gate_writers || return 1' &&
+ contains 'commit_cluster_sync_state "${cluster_fingerprint}" "${infra_fingerprint}"' &&
grep -Fq 'type: Recreate' deploy/base/control-plane/control-api.yaml; then
pass "pre-gate and Deployment enforce a no-overlap Control API writer window"
else
@@ -292,7 +294,8 @@ fi
if contains 'fence_workflow_reconciler()' &&
contains 'restore_workflow_reconciler()' &&
- contains 'trap restore_pre_gate_writers EXIT' &&
+ contains 'trap finalize_pre_gate_sync EXIT' &&
+ contains 'restore_pre_gate_writers || return 1' &&
contains 'fence_workflow_reconciler' &&
contains 'run-control-api-db-migration.sh' &&
contains 'make minikube-deploy-all'; then
diff --git a/scripts/tests/test-minikube-sync-auth-key-durable.sh b/scripts/tests/test-minikube-sync-auth-key-durable.sh
new file mode 100755
index 000000000..115ef791a
--- /dev/null
+++ b/scripts/tests/test-minikube-sync-auth-key-durable.sh
@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "${TMP_DIR}"' EXIT
+
+STATE_DIR="${TMP_DIR}/state"
+LOG_FILE="${TMP_DIR}/kubectl.log"
+OUT_FILE="${TMP_DIR}/sync.out"
+SOURCE_HASH="$(printf 'public-key' | shasum -a 256 | awk '{print $1}')"
+mkdir -p "${STATE_DIR}"
+printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"
+
+cat >"${TMP_DIR}/kubectl" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+
+while [[ "${1:-}" == --context* ]]; do
+ if [[ "${1}" == "--context" ]]; then shift 2; else shift; fi
+done
+while [[ "${1:-}" == -n || "${1:-}" == --namespace ]]; do shift 2; done
+printf '%s\n' "$*" >>"${TEST_KUBECTL_LOG}"
+
+increment() {
+ local name="$1" file count=0
+ file="${TEST_STATE_DIR}/${name}.count"
+ [[ -f "${file}" ]] && count="$(cat "${file}")"
+ printf '%s' "$((count + 1))" >"${file}"
+}
+
+if [[ "${1:-}" == get && "${2:-}" == secret && "${3:-}" == rpc-proxy-secrets ]]; then
+ [[ "$*" == *jsonpath* ]] && printf 'cHVibGljLWtleQ=='
+ exit 0
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == mcp-host-config ]]; then
+ if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ [[ -f "${TEST_STATE_DIR}/mcp.applied" ]] && cat "${TEST_STATE_DIR}/mcp.applied"
+ elif [[ "$*" == *jsonpath* ]]; then
+ cat "${TEST_STATE_DIR}/mcp.key"
+ fi
+ exit 0
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == configmap && "${3:-}" == gfs-config ]]; then
+ [[ "${TEST_GFS_PRESENT:-0}" == 1 ]] || {
+ echo 'Error from server (NotFound): configmaps "gfs-config" not found' >&2
+ exit 1
+ }
+ if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ [[ -f "${TEST_STATE_DIR}/gfs.applied" ]] && cat "${TEST_STATE_DIR}/gfs.applied"
+ elif [[ "$*" == *jsonpath* ]]; then
+ cat "${TEST_STATE_DIR}/gfs.key"
+ fi
+ exit 0
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == deployment && "${3:-}" == -l ]]; then
+ case "${TEST_MCP_DEPLOYMENTS:-chatllm}" in
+ none) ;;
+ custom) printf 'deployment.apps/custom-host\n' ;;
+ *) printf 'deployment.apps/chatllm\n' ;;
+ esac
+ exit 0
+fi
+
+if [[ "${1:-}" == patch && "${2:-}" == configmap ]]; then
+ target="${3}"
+ patch_payload=""
+ while [[ $# -gt 0 ]]; do
+ if [[ "$1" == -p ]]; then patch_payload="${2:-}"; break; fi
+ shift
+ done
+ case "${target}:${patch_payload}" in
+ mcp-host-config:*'"data"'*) printf 'public-key' >"${TEST_STATE_DIR}/mcp.key"; increment mcp-data-patch ;;
+ mcp-host-config:*auth-key-applied-sha256*) printf '%s' "${TEST_SOURCE_HASH}" >"${TEST_STATE_DIR}/mcp.applied"; increment mcp-marker-patch ;;
+ gfs-config:*'"data"'*) printf 'public-key' >"${TEST_STATE_DIR}/gfs.key"; increment gfs-data-patch ;;
+ gfs-config:*auth-key-applied-sha256*) printf '%s' "${TEST_SOURCE_HASH}" >"${TEST_STATE_DIR}/gfs.applied"; increment gfs-marker-patch ;;
+ *) echo "unexpected ConfigMap patch: ${target}" >&2; exit 91 ;;
+ esac
+ exit 0
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == deployment/* ]]; then
+ case "${2}" in
+ deployment/chatllm|deployment/custom-host) exit 0 ;;
+ deployment/mcp-host) echo 'Error from server (NotFound): deployments.apps "mcp-host" not found' >&2; exit 1 ;;
+ deployment/gfsc-writer|deployment/gfsc-reader)
+ if [[ "${TEST_GFS_DEPLOYMENTS:-0}" == 1 ]]; then exit 0; fi
+ printf 'Error from server (NotFound): deployments.apps "%s" not found\n' "${2#deployment/}" >&2
+ exit 1 ;;
+ *) exit 1 ;;
+ esac
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == deployment && "${3:-}" == gfsc-reader ]]; then
+ if [[ "$*" == *'.spec.replicas'* || "$*" == *'.status.readyReplicas'* ]]; then
+ printf '1'
+ exit 0
+ fi
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == secret && "${3:-}" == gfs-controller-db ]]; then
+ printf 'ZHNuLXZhbHVl'
+ exit 0
+fi
+
+if [[ "${1:-}" == rollout && "${2:-}" == restart && "${3:-}" == deployment/* ]]; then
+ increment "${3#deployment/}-restart"
+ exit 0
+fi
+
+if [[ "${1:-}" == rollout && "${2:-}" == status && "${3:-}" == deployment/* ]]; then
+ exit 0
+fi
+
+if [[ "${1:-}" == get && "${2:-}" == pods ]]; then
+ if [[ "$*" != *'.metadata.name'* && "$*" == *gfsc-role=reader* ]]; then
+ printf 'True|\n'
+ exit 0
+ fi
+ case "$*" in
+ *app=chatllm*) printf 'chatllm-pod|True|\n' ;;
+ *app=custom-host*) printf 'custom-host-pod|True|\n' ;;
+ *gfsc-role=writer*) printf 'gfsc-writer-pod|True|\n' ;;
+ *gfsc-role=reader*) printf 'gfsc-reader-pod|True|\n' ;;
+ *) echo "unexpected pod selector: $*" >&2; exit 92 ;;
+ esac
+ exit 0
+fi
+
+if [[ "${1:-}" == exec ]]; then
+ payload="$(cat)"
+ expected_key="${TEST_RUNTIME_KEY}"
+ [[ "$*" == *GFS_JWT_PUBLIC_KEY* ]] && expected_key="${TEST_RUNTIME_KEY_GFS:-${expected_key}}"
+ [[ "$*" == *CLERUM_AUTH_JWT_PUBLIC_KEY* ]] && expected_key="${TEST_RUNTIME_KEY_MCP:-${expected_key}}"
+ [[ "${payload}" == "${expected_key}" ]]
+ exit
+fi
+
+echo "unexpected kubectl invocation: $*" >&2
+exit 99
+SH
+chmod +x "${TMP_DIR}/kubectl"
+
+run_sync() {
+ TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
+ TEST_SOURCE_HASH="${SOURCE_HASH}" PATH="${TMP_DIR}:$PATH" \
+ "$@"
+}
+
+# Fault after the ConfigMap patch: the active pod still has the old key. The
+# durable marker must remain absent so a retry cannot misclassify this as done.
+if run_sync env TEST_RUNTIME_KEY=old-key \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: MCP sync accepted a pod that had not consumed the target key" >&2
+ exit 1
+fi
+[[ "$(cat "${STATE_DIR}/mcp.key")" == public-key ]]
+[[ ! -e "${STATE_DIR}/mcp.applied" ]]
+grep -q 'has not consumed the target auth key' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo "FAIL: interrupted MCP sync did not report failed consumer proof" >&2
+ exit 1
+}
+
+# The key now matches, but the absent marker must resume—not skip—consumer
+# convergence. A live pod that already consumed the target key need not restart.
+: >"${LOG_FILE}"
+run_sync env TEST_RUNTIME_KEY=public-key \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1
+[[ "$(cat "${STATE_DIR}/mcp.applied")" == "${SOURCE_HASH}" ]]
+[[ "$(cat "${STATE_DIR}/mcp-data-patch.count")" == 1 ]]
+[[ "$(cat "${STATE_DIR}/chatllm-restart.count")" == 1 ]]
+grep -q 'matches source but consumer attestation is pending; resuming convergence' "${OUT_FILE}"
+
+# A fully attested rerun is idempotent and must not touch the deployment.
+: >"${LOG_FILE}"
+# A converged pod must report the target key; an arbitrary value would
+# correctly trigger a recovery rollout rather than exercise idempotence.
+run_sync env TEST_RUNTIME_KEY=public-key \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1
+[[ "$(cat "${STATE_DIR}/chatllm-restart.count")" == 1 ]]
+if grep -Eq '^(rollout|patch)' "${LOG_FILE}"; then
+ echo "FAIL: converged MCP sync was not idempotent" >&2
+ cat "${LOG_FILE}" >&2
+ exit 1
+fi
+
+# A previous run may have attested the ConfigMap before HCC created a new
+# managed deployment. A later run must inspect that new pod instead of
+# returning on the old ConfigMap annotation alone.
+: >"${LOG_FILE}"
+if run_sync env TEST_RUNTIME_KEY=old-key TEST_MCP_DEPLOYMENTS=custom \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: newly created MCP consumer with stale key was accepted" >&2
+ exit 1
+fi
+[[ "$(cat "${STATE_DIR}/custom-host-restart.count")" == 1 ]]
+grep -q 'has not consumed the target auth key' "${OUT_FILE}"
+
+run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=custom \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \
+ >"${OUT_FILE}" 2>&1
+[[ "$(cat "${STATE_DIR}/custom-host-restart.count")" == 1 ]]
+
+# Repeat the interrupted-then-resumed contract for the two real GFSC
+# deployments. This also prevents a regression to deployment/gfs-controller.
+printf 'old-gfs-key' >"${STATE_DIR}/gfs.key"
+: >"${LOG_FILE}"
+if run_sync env TEST_RUNTIME_KEY=public-key TEST_RUNTIME_KEY_MCP=public-key TEST_RUNTIME_KEY_GFS=old-key \
+ TEST_GFS_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-gfs \
+ >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: GFS sync accepted a writer that had not consumed the target key" >&2
+ exit 1
+fi
+[[ "$(cat "${STATE_DIR}/gfs.key")" == public-key ]] || {
+ cat "${OUT_FILE}" >&2
+ cat "${LOG_FILE}" >&2
+ echo "FAIL: GFS ConfigMap was not patched before consumer proof" >&2
+ exit 1
+}
+[[ ! -e "${STATE_DIR}/gfs.applied" ]]
+
+: >"${LOG_FILE}"
+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_RUNTIME_KEY_MCP=public-key TEST_RUNTIME_KEY_GFS=public-key \
+ TEST_GFS_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \
+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-gfs \
+ >"${OUT_FILE}" 2>&1; then
+ cat "${OUT_FILE}" >&2
+ cat "${LOG_FILE}" >&2
+ echo "FAIL: GFS sync did not resume after an interrupted consumer proof" >&2
+ exit 1
+fi
+[[ "$(cat "${STATE_DIR}/gfs.applied")" == "${SOURCE_HASH}" ]]
+[[ "$(cat "${STATE_DIR}/gfsc-writer-restart.count")" == 1 ]]
+[[ ! -e "${STATE_DIR}/gfsc-reader-restart.count" ]]
+if grep -q 'rollout restart deployment/gfsc-' "${LOG_FILE}"; then
+ echo "FAIL: already-converged GFSC pods were restarted unnecessarily" >&2
+ cat "${LOG_FILE}" >&2
+ exit 1
+fi
+if grep -q 'deployment/gfs-controller' "${LOG_FILE}"; then
+ echo "FAIL: GFS auth sync targeted the nonexistent deployment/gfs-controller" >&2
+ exit 1
+fi
+
+echo "PASS: auth-key convergence is durable, resumable, consumer-proven, and idempotent"
diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.sh
index 8bd6bff5a..08d0f320f 100755
--- a/scripts/tests/test-minikube-sync-auth-key.sh
+++ b/scripts/tests/test-minikube-sync-auth-key.sh
@@ -57,7 +57,11 @@ fi
if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-config" ]]; then
if has_output_flag "$@"; then
- printf 'old-key'
+ if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ printf '%s' "${TEST_MCP_APPLIED_HASH:-old-hash}"
+ else
+ printf 'old-key'
+ fi
fi
exit 0
fi
@@ -68,7 +72,11 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]
exit 1
fi
if has_output_flag "$@"; then
- printf 'old-gfs-key'
+ if [[ "$*" == *auth-key-applied-sha256* ]]; then
+ printf '%s' "${TEST_GFS_APPLIED_HASH:-old-hash}"
+ else
+ printf 'old-gfs-key'
+ fi
fi
exit 0
fi
@@ -93,16 +101,40 @@ if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "gfs-controller-d
fi
if [[ "${1:-}" == "get" && "${2:-}" == "deployment" && "${3:-}" == "-l" ]]; then
- [[ "${TEST_GFS_DEPLOYMENTS:-0}" == "1" ]] || exit 0
- printf 'deployment.apps/gfsc-writer\ndeployment.apps/gfsc-reader\n'
+ printf 'deployment.apps/chatllm\ndeployment.apps/mcp-host\ndeployment.apps/custom-host\n'
exit 0
fi
if [[ "${1:-}" == "get" && "${2:-}" == deployment/* ]]; then
+ if [[ "${2:-}" == "deployment/gfsc-writer" || "${2:-}" == "deployment/gfsc-reader" ]]; then
+ if [[ "${TEST_GFS_DEPLOYMENTS:-0}" != "1" ]]; then
+ printf 'Error from server (NotFound): deployments.apps "%s" not found\n' "${2#deployment/}" >&2
+ exit 1
+ fi
+ fi
+ exit 0
+fi
+
+if [[ "${1:-}" == "get" && "${2:-}" == "pods" ]]; then
+ case "$*" in
+ *app=chatllm*) printf 'chatllm-pod|True|\n' ;;
+ *app=mcp-host*) printf 'mcp-host-pod|True|\n' ;;
+ *app=custom-host*) printf 'custom-host-pod|True|\n' ;;
+ *gfsc-role=writer*) printf 'gfsc-writer-pod|True|\n' ;;
+ *gfsc-role=reader*) printf 'gfsc-reader-pod|True|\n' ;;
+ *) exit 99 ;;
+ esac
exit 0
fi
-if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment" && "${4:-}" == "-l" ]]; then
+if [[ "${1:-}" == "exec" ]]; then
+ payload="$(cat)"
+ [[ "${TEST_CONSUMER_KEY_MATCH:-1}" == "1" && "${payload}" == "public-key" ]]
+ exit
+fi
+
+if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && \
+ ( "${3:-}" == "deployment/gfsc-writer" || "${3:-}" == "deployment/gfsc-reader" ) ]]; then
[[ "${TEST_GFS_DSN_PRESENT:-0}" == "1" ]] || {
echo "unexpected gfsc rollout restart before DSN provisioning" >&2
exit 98
@@ -126,6 +158,12 @@ if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/
exit 0
fi
+if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/custom-host" ]]; then
+ restart_count custom-host >/dev/null
+ echo 'deployment.apps/custom-host restarted'
+ exit 0
+fi
+
if [[ "${1:-}" == "rollout" && "${2:-}" == "status" && "${3:-}" == deployment/* ]]; then
if [[ "${3:-}" == *gfsc-writer* && "${TEST_GFS_STATUS_FAILURE:-0}" != "0" ]]; then
echo 'simulated gfsc writer rollout failure' >&2
@@ -159,9 +197,16 @@ if [[ "$(cat "${STATE_DIR}/mcp-host.restart-count")" != "1" ]]; then
exit 1
fi
+if [[ "$(cat "${STATE_DIR}/custom-host.restart-count")" != "1" ]]; then
+ echo "FAIL: managed custom mcp-host deployment was not restarted" >&2
+ cat "${OUT_FILE}" >&2
+ exit 1
+fi
+
grep -q 'patch configmap mcp-host-config' "${LOG_FILE}"
grep -q 'rollout status deployment/chatllm -n mcp-host --timeout=180s' "${LOG_FILE}"
grep -q 'rollout status deployment/mcp-host -n mcp-host --timeout=180s' "${LOG_FILE}"
+grep -q 'rollout status deployment/custom-host -n mcp-host --timeout=180s' "${LOG_FILE}"
: >"${LOG_FILE}"
if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFIG_PRESENT=1 PATH="${TMP_DIR}:$PATH" \
@@ -172,9 +217,9 @@ if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFI
fi
grep -q 'patch configmap gfs-config' "${LOG_FILE}"
-grep -q 'Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)' "${OUT_FILE}"
-if grep -q 'rollout restart deployment -l' "${LOG_FILE}"; then
- echo "FAIL: gfsc restarted before gfs-controller-db DSN was populated" >&2
+grep -q 'No gfsc deployments exist yet; auth key is synced for future pods' "${OUT_FILE}"
+if grep -q 'rollout restart deployment/gfsc-' "${LOG_FILE}"; then
+ echo "FAIL: absent gfsc deployments were restarted" >&2
cat "${OUT_FILE}" >&2
exit 1
fi
@@ -274,10 +319,10 @@ if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
echo "FAIL: GFS auth sync swallowed a failed rollout status" >&2
exit 1
fi
-grep -q 'gfsc deployment deployment.apps/gfsc-writer did not become Ready after auth key drift' "${OUT_FILE}" || {
+grep -q 'gfsc deployment deployment/gfsc-writer did not become Ready after auth key drift' "${OUT_FILE}" || {
cat "${OUT_FILE}" >&2
echo "FAIL: GFS rollout failure did not produce a stable diagnostic" >&2
exit 1
}
-echo "PASS: auth-key sync retries transient rollout restart race and skips gfsc restart until DSN provisioning"
+echo "PASS: auth-key sync retries transient rollout restart race and handles absent gfsc consumers safely"
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 6cecc55f2..c09cbeabf 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -26,7 +26,12 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" "$T0" \
"$ROOT/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh" \
"$ROOT/scripts/tests/test-minikube-gfs-rollout-shim.sh" \
"$ROOT/scripts/tests/test-minikube-gfs-provision-order.sh" \
+ "$ROOT/scripts/tests/test-minikube-pre-gate-sync-state.sh" \
"$ROOT/scripts/tests/test-minikube-pre-gate-restore.sh" \
+ "$ROOT/scripts/tests/test-minikube-sync-auth-key.sh" \
+ "$ROOT/scripts/tests/test-minikube-sync-auth-key-durable.sh" \
+ "$ROOT/scripts/tests/test-minikube-targeted-gfs-sync.sh" \
+ "$ROOT/scripts/tests/test-minikube-t2-lock-race.sh" \
"$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"; do
bash -n "$file"
done
@@ -520,7 +525,12 @@ bash "$ROOT/scripts/tests/test-minikube-t2-scenarios.sh"
bash "$ROOT/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh"
bash "$ROOT/scripts/tests/test-minikube-gfs-rollout-shim.sh"
bash "$ROOT/scripts/tests/test-minikube-gfs-provision-order.sh"
+bash "$ROOT/scripts/tests/test-minikube-pre-gate-sync-state.sh"
bash "$ROOT/scripts/tests/test-minikube-pre-gate-restore.sh"
+bash "$ROOT/scripts/tests/test-minikube-sync-auth-key.sh"
+bash "$ROOT/scripts/tests/test-minikube-sync-auth-key-durable.sh"
+bash "$ROOT/scripts/tests/test-minikube-targeted-gfs-sync.sh"
+bash "$ROOT/scripts/tests/test-minikube-t2-lock-race.sh"
bash "$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"
printf 'PASS: local Minikube T0/T1/T2 contract checks\n'
diff --git a/scripts/tests/test-minikube-t2-lock-race.sh b/scripts/tests/test-minikube-t2-lock-race.sh
new file mode 100755
index 000000000..ff7bfc8b4
--- /dev/null
+++ b/scripts/tests/test-minikube-t2-lock-race.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "${TMP_DIR}"' EXIT
+
+LOCK_ROOT="${TMP_DIR}/locks"
+LOCK_DIR="${LOCK_ROOT}/race-profile.lock"
+READY_FILE="${TMP_DIR}/ready"
+RESULT_FILE="${TMP_DIR}/result"
+RELEASE_FILE="${TMP_DIR}/release"
+mkdir -p "${LOCK_DIR}"
+: >"${READY_FILE}"
+: >"${RESULT_FILE}"
+
+owner_token_key=TOKEN
+{
+ printf '%s=%s\n' "${owner_token_key}" stale-owner-token
+ printf 'PID=999999\n'
+ printf 'PROCESS_START=stale-process-start\n'
+} >"${LOCK_DIR}/owner.env"
+
+cat >"${TMP_DIR}/worker.sh" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=/dev/null
+source "${TEST_COMMON}"
+T2_PROJECT_DIR="${TEST_PROJECT_DIR}"
+T2_BRANCH=feat/lock-race
+T2_HEAD=1111111111111111111111111111111111111111
+T2_PROFILE=race-profile
+T2_CONTEXT=race-profile
+T2_WORKTREE_ID=lock-race-worktree
+
+# Force both contenders to classify the same original owner as stale before
+# either can claim it. A replacement owner is judged by liveness so the loser
+# cannot remove the winner's newly acquired lock.
+t2_lock_process_matches() {
+ local pid="$1" deadline lines
+ if [[ "${pid}" != 999999 ]]; then
+ kill -0 "${pid}" >/dev/null 2>&1
+ return
+ fi
+ printf '%s\n' "$$" >>"${TEST_READY_FILE}"
+ deadline=$((SECONDS + 10))
+ while :; do
+ lines="$(wc -l <"${TEST_READY_FILE}" | tr -d ' ')"
+ [[ "${lines}" -ge 2 ]] && break
+ [[ "${SECONDS}" -lt "${deadline}" ]] || exit 70
+ sleep 0.02
+ done
+ return 1
+}
+
+t2_lock_acquire
+printf 'ACQUIRED %s\n' "$$" >>"${TEST_RESULT_FILE}"
+deadline=$((SECONDS + 10))
+while [[ ! -e "${TEST_RELEASE_FILE}" ]]; do
+ [[ "${SECONDS}" -lt "${deadline}" ]] || exit 71
+ sleep 0.02
+done
+t2_lock_release 0
+SH
+chmod +x "${TMP_DIR}/worker.sh"
+
+worker_env=(
+ TEST_COMMON="${ROOT}/scripts/minikube/t2-common.sh"
+ TEST_PROJECT_DIR="${ROOT}"
+ TEST_READY_FILE="${READY_FILE}"
+ TEST_RESULT_FILE="${RESULT_FILE}"
+ TEST_RELEASE_FILE="${RELEASE_FILE}"
+ T2_LOCK_ROOT="${LOCK_ROOT}"
+ T2_PROFILE=race-profile
+ T2_CONTEXT=race-profile
+ MINIKUBE_PROFILE=race-profile
+)
+
+env "${worker_env[@]}" bash "${TMP_DIR}/worker.sh" >"${TMP_DIR}/worker-1.out" 2>&1 &
+worker_1=$!
+env "${worker_env[@]}" bash "${TMP_DIR}/worker.sh" >"${TMP_DIR}/worker-2.out" 2>&1 &
+worker_2=$!
+
+deadline=$((SECONDS + 10))
+while [[ "$(wc -l <"${RESULT_FILE}" | tr -d ' ')" -lt 1 ]]; do
+ if ! kill -0 "${worker_1}" >/dev/null 2>&1 && ! kill -0 "${worker_2}" >/dev/null 2>&1; then
+ cat "${TMP_DIR}/worker-1.out" "${TMP_DIR}/worker-2.out" >&2
+ echo 'FAIL: both stale-lock reclaimers exited before acquiring the lock' >&2
+ exit 1
+ fi
+ [[ "${SECONDS}" -lt "${deadline}" ]] || {
+ echo 'FAIL: stale-lock race did not produce a winner' >&2
+ exit 1
+ }
+ sleep 0.02
+done
+
+[[ "$(wc -l <"${RESULT_FILE}" | tr -d ' ')" -eq 1 ]] || {
+ cat "${RESULT_FILE}" >&2
+ echo 'FAIL: two stale-lock reclaimers entered the critical section' >&2
+ exit 1
+}
+winner_pid="$(awk '{print $2}' "${RESULT_FILE}")"
+[[ "$(sed -n 's/^PID=//p' "${LOCK_DIR}/owner.env")" == "${winner_pid}" ]] || {
+ echo 'FAIL: active lock owner does not match the sole reclaimer' >&2
+ exit 1
+}
+[[ ! -e "${LOCK_DIR}/.reclaim" ]] || {
+ echo 'FAIL: losing reclaimer left a claim inside the active replacement lock' >&2
+ exit 1
+}
+
+: >"${RELEASE_FILE}"
+set +e
+wait "${worker_1}"; rc_1=$?
+wait "${worker_2}"; rc_2=$?
+set -e
+if ! { [[ "${rc_1}" -eq 0 && "${rc_2}" -ne 0 ]] || [[ "${rc_2}" -eq 0 && "${rc_1}" -ne 0 ]]; }; then
+ cat "${TMP_DIR}/worker-1.out" "${TMP_DIR}/worker-2.out" >&2
+ echo "FAIL: expected one successful reclaimer and one fail-closed contender (rc=${rc_1},${rc_2})" >&2
+ exit 1
+fi
+[[ ! -e "${LOCK_DIR}" ]] || {
+ echo 'FAIL: winning reclaimer did not release its lock' >&2
+ exit 1
+}
+
+cat "${TMP_DIR}/worker-1.out" "${TMP_DIR}/worker-2.out" | \
+ grep -Eq 'stale lock is already being reclaimed|became live while its stale lock was being reclaimed|unable to acquire the profile lock' || {
+ echo 'FAIL: losing reclaimer did not report a fail-closed lock conflict' >&2
+ exit 1
+ }
+
+printf 'PASS: concurrent stale-lock reclaim admits exactly one owner and preserves its replacement lock\n'
diff --git a/scripts/tests/test-minikube-targeted-gfs-sync.sh b/scripts/tests/test-minikube-targeted-gfs-sync.sh
new file mode 100755
index 000000000..157ad474c
--- /dev/null
+++ b/scripts/tests/test-minikube-targeted-gfs-sync.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "${TMP_DIR}"' EXIT
+LOG_FILE="${TMP_DIR}/calls.log"
+
+fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
+
+cat >"${TMP_DIR}/kubectl" <<'SH'
+#!/usr/bin/env bash
+set -euo pipefail
+while [[ "${1:-}" == --context* ]]; do
+ if [[ "${1}" == --context ]]; then shift 2; else shift; fi
+done
+while [[ "${1:-}" == -n || "${1:-}" == --namespace ]]; do shift 2; done
+printf '%s\n' "$*" >>"${TEST_CALL_LOG}"
+case "$*" in
+ get\ deployment\ gfsc-writer\ -n\ gfs|get\ deployment\ gfsc-reader\ -n\ gfs) exit 0 ;;
+ rollout\ restart\ deployment/gfsc-writer\ -n\ gfs|rollout\ restart\ deployment/gfsc-reader\ -n\ gfs) exit 0 ;;
+ get\ deployment\ gfsc-reader\ -o*spec.replicas*) printf '1'; exit 0 ;;
+ get\ deployment\ gfsc-reader\ -o*status.readyReplicas*) printf '1'; exit 0 ;;
+ get\ pods\ -l*gfsc-role=reader*) printf 'True|\n'; exit 0 ;;
+ *) echo "unexpected kubectl call: $*" >&2; exit 90 ;;
+esac
+SH
+chmod +x "${TMP_DIR}/kubectl"
+
+PROJECT_DIR="${ROOT}"
+PROFILE=fake
+KC="kubectl --context=fake"
+FORCE_CLUSTER_SYNC=false
+FORCE_RESTART=false
+IMAGE_SOURCE=local
+IMAGE_TAG=test
+IMAGES_GENERATED_AT=test
+log() { :; }
+rollout_if_present() { printf 'wait %s/%s\n' "$1" "$2" >>"${LOG_FILE}"; }
+export TEST_CALL_LOG="${LOG_FILE}"
+export PATH="${TMP_DIR}:${PATH}"
+
+# shellcheck source=/dev/null
+source "${ROOT}/scripts/minikube/pre-gate-incremental.sh"
+incremental_classify_path gfs-controller/src/index.ts
+
+expected_targets=$'gfs-controller|gfs|gfsc-writer\ngfs-controller|gfs|gfsc-reader'
+actual_targets="$(printf '%s\n' "${INCREMENTAL_TARGETS[@]}")"
+[[ "${actual_targets}" == "${expected_targets}" ]] || \
+ fail "GFS source mapped to the wrong consumers: ${actual_targets}"
+
+# Two deployments consume one image selector; the image must be built once.
+bash() { printf 'build %s\n' "$*" >>"${LOG_FILE}"; }
+incremental_build_images
+[[ "$(grep -c '^build .*--only=gfs-controller$' "${LOG_FILE}")" -eq 1 ]] || \
+ fail 'gfs-controller image was not built exactly once for writer+reader'
+
+incremental_restart_targets
+grep -q '^rollout restart deployment/gfsc-writer -n gfs$' "${LOG_FILE}" || \
+ fail 'targeted GFS sync did not restart gfsc-writer'
+grep -q '^rollout restart deployment/gfsc-reader -n gfs$' "${LOG_FILE}" || \
+ fail 'targeted GFS sync did not restart gfsc-reader'
+grep -q '^wait gfs/gfsc-writer$' "${LOG_FILE}" || fail 'gfsc-writer readiness was not awaited'
+grep -q '^get deployment gfsc-reader -o jsonpath={.status.readyReplicas}$' "${LOG_FILE}" || \
+ fail 'gfsc-reader did not use the HCC-safe readiness wait'
+if grep -q '^wait gfs/gfsc-reader$' "${LOG_FILE}"; then
+ fail 'gfsc-reader used the generation-based generic rollout wait'
+fi
+
+if grep -Eq 'deployment/gfs-controller|control-plane/host-context-controller' "${LOG_FILE}"; then
+ fail 'targeted GFS sync used the obsolete controller deployment path'
+fi
+if grep -Fq 'MINIKUBE_DEPLOYMENT=gfs-controller' "${ROOT}/scripts/minikube/t2.sh"; then
+ fail 'T2 still directly targets nonexistent deployment/gfs-controller'
+fi
+grep -Fq 'targeted mutation delegated to canonical pre-gate-sync' "${ROOT}/scripts/minikube/t2.sh" || \
+ fail 'T2 targeted transition does not delegate to canonical pre-gate-sync'
+
+printf 'PASS: targeted GFS sync builds once and restarts gfsc-writer/gfsc-reader only\n'