Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: settle a Ready gfsc-reader leftover rollout before T2 reconcile
Alfredo Lopezf89e04fcfb67
diff --git a/.cursor/rules/minikube-t2-harness.mdc b/.cursor/rules/minikube-t2-harness.mdcindex fa122a8fb..114df0b9c 100644--- a/.cursor/rules/minikube-t2-harness.mdc+++ b/.cursor/rules/minikube-t2-harness.mdc@@ -42,6 +42,9 @@ Invariants that must never regress:reconcile calls — setup runs before pre-gate-sync and must not abort on aNOLOGIN reader or a leftover `rollout-running` claim from a dead priorsetup. The T2 profile lock makes that abandoned-state recover safe.+ When `gfsc-reader` is already Ready, `settle-gfs-reader-rollout.sh`+ marks that leftover claim ready before reconcile so a second+ `rollout restart` cannot race HCC's gfsReconciler.- T1 stays fail-loud (no green on unavailable DSN or zero tests); the JSONreporter is the suite verdict, not a leftover Vitest process exit after acomplete green reporter. Role-reset suites keep the throwaway Postgres 16.diff --git a/.cursor/skills/minikube-t0-t1-t2/reference.md b/.cursor/skills/minikube-t0-t1-t2/reference.mdindex 6f3e34ea4..e0b8e40a2 100644--- a/.cursor/skills/minikube-t0-t1-t2/reference.md+++ b/.cursor/skills/minikube-t0-t1-t2/reference.md@@ -64,7 +64,9 @@ The harness owns NOLOGIN recovery and abandoned-rollout resume: T1 restoresbranch-profile GFS credentials on exit, and `pre-gate-sync` provisionsserving with `GFS_RESTORE_ACTIVE_NOLOGIN=true` and`GFS_RECOVER_ABANDONED_STATE=true` in every plan and restarts an unready-`gfsc-reader` after restore. Do not run+`gfsc-reader` after restore. A Ready leftover `rollout-running` claim is+settled by `scripts/minikube/settle-gfs-reader-rollout.sh` before+reconcile so HCC's gfsReconciler cannot race a second restart. Do not run`reconcile-gfs-deploy-credentials.sh` or `kubectl rollout restartdeploy/gfsc-reader` by hand as a T2-repair step — re-run the entry point.diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.mdindex cfeb57b11..959885f95 100644--- a/docs/testing/minikube-t2-runbook.md+++ b/docs/testing/minikube-t2-runbook.md@@ -140,7 +140,12 @@ or interrupt) using the canonical`GFS_RESTORE_ACTIVE_NOLOGIN=true` and `GFS_RECOVER_ABANDONED_STATE=true` —the same contract as the standalone GFS T1 gate, plus resume of a leftover`rollout-running` claim from a timed-out prior setup. The T2 profile lock-makes that recover safe: the prior process is dead. `pre-gate-sync`+makes that recover safe: the prior process is dead. When `gfsc-reader`+is already Ready, `scripts/minikube/settle-gfs-reader-rollout.sh` marks+the leftover reader claim ready before reconcile so a second+`rollout restart` cannot race HCC's gfsReconciler (that patch loop+makes `kubectl rollout status` wait for a spec update until timeout).+`pre-gate-sync`provisions GFS serving with the same opt-ins in every sync plan (including"no cluster sync required") and restarts an unready `gfsc-reader` after asuccessful restore. `full-setup.sh` on the REUSE_DB / T2 full-reconcile pathdiff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 7a1d128b9..580e0ed5a 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -985,6 +985,10 @@ else# reader Secret in rollout-running. This path holds the T2 profile lock,# so the prior process is dead; resume that claim instead of asking for# a manual retry flag between runs.+ # If gfsc-reader is already Ready, settle the leftover claim first so+ # reconcile does not rollout restart and race HCC's gfsReconciler.+ CONTEXT="${PROFILE}" \+ bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"@@ -1049,6 +1053,8 @@ else# the split writer/reader templates; reconciling before that lands leaves# the staged reader credential rollout-pending and fails the final verify.CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/wait-gfsc-secret-references.sh"+ CONTEXT="${PROFILE}" \+ bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.shindex 11251c569..ce96c2584 100755--- a/scripts/minikube/pre-gate-sync.sh+++ b/scripts/minikube/pre-gate-sync.sh@@ -333,6 +333,10 @@ provision_gfs_serving() {# credential cannot authenticate; no password is invented here.# GFS_RECOVER_ABANDONED_STATE: a dead prior setup can leave# rollout-running; this sync holds the T2 lock, so resume that claim.+ # Settle a Ready reader first so reconcile does not restart it and+ # race HCC's gfsReconciler during kubectl rollout status.+ CONTEXT="${PROFILE}" \+ bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"if ! GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; thendiff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.shnew file mode 100755index 000000000..97b0140fc--- /dev/null+++ b/scripts/minikube/settle-gfs-reader-rollout.sh@@ -0,0 +1,50 @@+#!/usr/bin/env bash+# If gfsc-reader is already Ready, clear a leftover rollout-running claim+# on the reader Secret so the next reconcile does not rollout restart.+# HCC's gfsReconciler patches gfsc-reader; a second restart races+# `kubectl rollout status` ("Waiting for deployment spec update to be+# observed") until the wait times out.+set -euo pipefail++CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"+GFS_NS="${GFS_NS:-gfs}"+DEPLOY="${GFS_READER_DEPLOYMENT:-gfsc-reader}"+SECRET="${GFS_READER_DB_SECRET:-gfs-controller-reader-db}"++kc() { kubectl --context="$CONTEXT" "$@"; }+log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }++if ! kc -n "$GFS_NS" get deployment "$DEPLOY" >/dev/null 2>&1; then+ exit 0+fi++desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.spec.replicas}')"+ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.status.readyReplicas}')"+if [ -z "$desired" ] || [ "$desired" = "0" ] || [ "${ready:-0}" != "$desired" ]; then+ exit 0+fi++if ! kc -n "$GFS_NS" get secret "$SECRET" >/dev/null 2>&1; then+ exit 0+fi++state="$(kc -n "$GFS_NS" get secret "$SECRET" \+ -o jsonpath='{.metadata.annotations.clerum\.io/gfs-dsn-state}')"+[ "$state" = rollout-running ] || exit 0++rotated="$(kc -n "$GFS_NS" get secret "$SECRET" \+ -o jsonpath='{.metadata.annotations.clerum\.io/gfs-dsn-rotated-at}')"+if [ -z "$rotated" ]; then+ log "reader Secret is rollout-running without rotated-at; leaving it for reconcile"+ exit 0+fi++log "gfsc-reader is Ready with an interrupted rollout claim; marking the reader Secret ready without restart"+python3 -c 'import json, sys+state, stamp = sys.argv[1:]+print(json.dumps([+ {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": state},+ {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-rotated-at", "value": stamp},+ {"op": "replace", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": "ready"},+]))' "$state" "$rotated" \+ | kc -n "$GFS_NS" patch secret "$SECRET" --type=json --patch-file=/dev/stdin >/dev/nulldiff --git a/scripts/tests/test-minikube-full-setup.sh b/scripts/tests/test-minikube-full-setup.shindex f6aab554c..870f2c78d 100755--- a/scripts/tests/test-minikube-full-setup.sh+++ b/scripts/tests/test-minikube-full-setup.sh@@ -283,6 +283,7 @@ assert_gfs_provisioning_follows_migrations_and_core_readiness() {grep -q 'scale deployment/host-context-controller --replicas="\$RESET_HCC_REPLICAS"' <<<"$reset_block" && \! grep -q 'delete pvc control-postgres-data' scripts/minikube/full-setup.sh && \[[ "$(grep -c 'GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true' scripts/minikube/full-setup.sh)" -ge 2 ]] && \+ [[ "$(grep -c 'scripts/minikube/settle-gfs-reader-rollout.sh' scripts/minikube/full-setup.sh)" -ge 2 ]] && \[[ "$core_block" != *"gfs-controller"* ]]; thenpass "full-setup centralizes explicit UID-bound reset and never auto-deletes on WAL logs"elsediff --git a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh b/scripts/tests/test-minikube-settle-gfs-reader-rollout.shnew file mode 100755index 000000000..6ca7cfc9b--- /dev/null+++ b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh@@ -0,0 +1,69 @@+#!/usr/bin/env bash+# Stubbed contract for scripts/minikube/settle-gfs-reader-rollout.sh+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+SCRIPT="$ROOT/scripts/minikube/settle-gfs-reader-rollout.sh"+fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }++bash -n "$SCRIPT"++run_settle() {+ local fake_dir+ fake_dir="$(mktemp -d)"+ cat >"$fake_dir/kubectl" <<'STUB'+#!/usr/bin/env bash+set -euo pipefail+printf '%s\n' "$*" >>"${FAKE_KUBECTL_LOG:?}"+case " $* " in+ *' get deployment gfsc-reader '*'-o jsonpath={.spec.replicas}'*)+ printf '%s' "${FAKE_DESIRED:-1}"; exit 0 ;;+ *' get deployment gfsc-reader '*'-o jsonpath={.status.readyReplicas}'*)+ printf '%s' "${FAKE_READY:-1}"; exit 0 ;;+ *' get deployment gfsc-reader'*)+ [ "${FAKE_DEPLOY_EXISTS:-1}" = 1 ] || exit 1+ exit 0 ;;+ *' get secret gfs-controller-reader-db '*gfs-dsn-state*)+ printf '%s' "${FAKE_STATE:-rollout-running}"; exit 0 ;;+ *' get secret gfs-controller-reader-db '*gfs-dsn-rotated-at*)+ printf '%s' "${FAKE_ROTATED:-2026-01-01T00:00:00Z}"; exit 0 ;;+ *' get secret gfs-controller-reader-db'*)+ [ "${FAKE_SECRET_EXISTS:-1}" = 1 ] || exit 1+ exit 0 ;;+ *' patch secret gfs-controller-reader-db'*)+ cat >/dev/null+ [ "${FAKE_PATCH_OK:-1}" = 1 ] || exit 1+ exit 0 ;;+esac+exit 1+STUB+ chmod +x "$fake_dir/kubectl"+ PATH="$fake_dir:$PATH" CONTEXT=fake \+ FAKE_KUBECTL_LOG="$fake_dir/kubectl.log" \+ FAKE_DEPLOY_EXISTS="${1:-1}" \+ FAKE_DESIRED="${2:-1}" \+ FAKE_READY="${3:-1}" \+ FAKE_SECRET_EXISTS="${4:-1}" \+ FAKE_STATE="${5:-rollout-running}" \+ FAKE_ROTATED="${6:-2026-01-01T00:00:00Z}" \+ FAKE_PATCH_OK="${7:-1}" \+ bash "$SCRIPT"+ printf '%s' "$fake_dir"+}++dir="$(run_settle 1 1 1 1 rollout-running '2026-01-01T00:00:00Z' 1)"+grep -q 'patch secret gfs-controller-reader-db' "$dir/kubectl.log" \+ || fail 'Ready leftover claim did not patch the reader Secret to ready'+rm -rf "$dir"++dir="$(run_settle 1 1 0 1 rollout-running '2026-01-01T00:00:00Z' 1)"+grep -q 'patch secret' "$dir/kubectl.log" \+ && fail 'unready reader must not settle the leftover claim'+rm -rf "$dir"++dir="$(run_settle 1 1 1 1 ready '2026-01-01T00:00:00Z' 1)"+grep -q 'patch secret' "$dir/kubectl.log" \+ && fail 'ready Secret state must not be patched again'+rm -rf "$dir"++printf 'PASS: settle-gfs-reader-rollout leftover Ready claim\n'diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex 947aaabd5..7a3b270b3 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -16,8 +16,10 @@ set -ufor file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \"$ROOT/scripts/e2e/e2e-np08-hcc-authorization.sh" \+ "$ROOT/scripts/minikube/settle-gfs-reader-rollout.sh" \"$ROOT/scripts/tests/test-minikube-t2-public-boundary.sh" \- "$ROOT/scripts/tests/test-minikube-t2-scenarios.sh"; do+ "$ROOT/scripts/tests/test-minikube-t2-scenarios.sh" \+ "$ROOT/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh"; dobash -n "$file"done@@ -102,6 +104,10 @@ if ! sed -n '/^provision_gfs_serving()/,/^}/p' "$PRE_GATE" | grep -Fq 'GFS_RECOVecho 'FAIL: provision_gfs_serving does not resume an interrupted gfsc-reader rollout claim' >&2exit 1fi+if ! sed -n '/^provision_gfs_serving()/,/^}/p' "$PRE_GATE" | grep -Fq 'settle-gfs-reader-rollout.sh'; then+ echo 'FAIL: provision_gfs_serving does not settle a Ready gfsc-reader leftover rollout claim before reconcile' >&2+ exit 1+fiif ! sed -n '/No cluster sync required before/,$p' "$PRE_GATE" | grep -Fq 'provision_gfs_serving'; thenecho 'FAIL: pre-gate-sync skips GFS serving convergence when no cluster sync is required' >&2exit 1@@ -111,6 +117,10 @@ if [[ "$(grep -c 'GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=trecho 'FAIL: full-setup REUSE_DB path must restore a NOLOGIN GFS role and resume an abandoned reader rollout before and after overlay' >&2exit 1fi+if [[ "$(grep -c 'scripts/minikube/settle-gfs-reader-rollout.sh' "$ROOT/scripts/minikube/full-setup.sh")" -lt 2 ]]; then+ echo 'FAIL: full-setup REUSE_DB path must settle a Ready gfsc-reader leftover rollout claim before both reconciles' >&2+ exit 1+fiif grep -Eq 'make minikube-pre-gate-sync|pre-gate-sync\.sh' "$PREFLIGHT"; thenecho 'FAIL: preflight invokes pre-gate-sync' >&2@@ -256,5 +266,6 @@ if T2_PUBLIC_ROOT="$public_repo" T2_PUBLIC_BASE_REF="$public_base" \exit 1fibash "$ROOT/scripts/tests/test-minikube-t2-scenarios.sh"+bash "$ROOT/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh"printf 'PASS: local Minikube T0/T1/T2 contract checks\n'diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex 16d17d5a3..679fac3c7 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -278,7 +278,9 @@ grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$ROOT/scripts/e2e/minikube-real-postgrep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 GFS restore must resume an interrupted gfsc-reader rollout claim'grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$ROOT/scripts/minikube/pre-gate-sync.sh" || fail 'pre-gate-sync must restore a NOLOGIN GFS role from the committed Secret DSN'grep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$ROOT/scripts/minikube/pre-gate-sync.sh" || fail 'pre-gate-sync must resume an interrupted gfsc-reader rollout claim'+grep -Fq 'settle-gfs-reader-rollout.sh' "$ROOT/scripts/minikube/pre-gate-sync.sh" || fail 'pre-gate-sync must settle a Ready gfsc-reader leftover rollout claim before reconcile'[[ "$(grep -c 'GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true' "$ROOT/scripts/minikube/full-setup.sh")" -ge 2 ]] || fail 'full-setup REUSE_DB must restore a NOLOGIN GFS role and resume an abandoned reader rollout on both reconcile calls'+[[ "$(grep -c 'scripts/minikube/settle-gfs-reader-rollout.sh' "$ROOT/scripts/minikube/full-setup.sh")" -ge 2 ]] || fail 'full-setup REUSE_DB must settle a Ready gfsc-reader leftover rollout claim before both reconciles'grep -Fq 'T2_LOCK_ROOT' "$COMMON"grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"grep -Fq 'PORT_FORWARD_CONFLICT' "$COMMON"