Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): harden recovery and profile forwarding checks

Alfredo Lopez0bc39b850e03
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index ea64d749b..63be04220 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -1133,6 +1133,7 @@ WRITER_RECOVERY_STATE_ROOT="${PROJECT_DIR}/.local-notes/infra/runs/writer-recove
WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"
WRITER_RECOVERY_STATE_LOADED=false
WRITER_RECOVERY_STATE_PHASE=""
+WRITER_RECOVERY_STATE_HEAD=""
WRITER_RECOVERY_FENCE_PENDING=false
WRITER_RECOVERY_MIGRATIONS_COMPLETE=false
K8S_API_EGRESS_POLICY_FILE=""
@@ -1176,7 +1177,7 @@ writer_recovery_state_clear() {
writer_recovery_state_load() {
local state_output state_status=0
- state_output="$(writer_recovery_state_cli read)" || state_status=$?
+ state_output="$(writer_recovery_state_cli read --include-head)" || state_status=$?
if [ "$state_status" -ne 0 ]; then
err "Unable to validate the durable writer-recovery state; refusing profile mutation"
return 1
@@ -1186,7 +1187,7 @@ writer_recovery_state_load() {
fi
IFS='|' read -r WRITER_RECOVERY_STATE_PHASE PARTIAL_HCC_REPLICAS \
PARTIAL_WORKFLOW_REPLICAS PARTIAL_TRACE_REPLICAS \
- PARTIAL_CONTROL_API_REPLICAS <<<"$state_output"
+ PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD <<<"$state_output"
[ -n "$WRITER_RECOVERY_STATE_PHASE" ] &&
[[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] &&
[[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] &&
@@ -1259,7 +1260,11 @@ writer_recovery_state_load() {
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
- WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then
+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ else
+ log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"
+ fi
# The phase is written before the control-api restore. An interruption
# after scale but before the next phase must close every writer again.
WRITER_RECOVERY_FENCE_PENDING=true
@@ -1270,7 +1275,11 @@ writer_recovery_state_load() {
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
- WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then
+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ else
+ log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"
+ fi
# The overlay is rendered with all four database-writer Deployments at zero.
# Re-fence on every post-migration resume because an interrupted apply
# may have recreated a writer before its durable phase was updated.
diff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.sh
index adf77037e..b0461aff3 100755
--- a/scripts/minikube/pf-all-stack.sh
+++ b/scripts/minikube/pf-all-stack.sh
@@ -11,6 +11,8 @@ WORKTREE_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel 2>/dev/n
WORKTREE_ROOT="$(cd -- "${WORKTREE_ROOT}" && pwd -P)"
# shellcheck source=scripts/minikube/port-forward-owner.sh
source "${SCRIPT_DIR}/port-forward-owner.sh"
+# shellcheck source=scripts/minikube/profile-owner.sh
+source "${SCRIPT_DIR}/profile-owner.sh"
PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
CONTEXT="${KUBECONTEXT:-${PROFILE}}"
@@ -121,6 +123,29 @@ is_branch_profile() {
[[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]
}
+validate_branch_profile_ownership() {
+ [[ "${PROFILE}" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]] || return 0
+ local branch profile_env ports_env
+ branch="$(git -C "${WORKTREE_ROOT}" branch --show-current 2>/dev/null || true)"
+ [[ -n "${branch}" ]] || {
+ echo "ERROR: branch-owned port-forwards require a named Git branch" >&2
+ return 1
+ }
+ profile_env="${T2_PROFILE_ENV:-${PROFILE_PID_ROOT}/${PROFILE}/profile.env}"
+ ports_env="${CLERUM_PROFILE_PORTS_ENV:-${PROFILE_PID_ROOT}/${PROFILE}/ports.env}"
+ [[ -f "${profile_env}" && -f "${ports_env}" ]] || {
+ echo "ERROR: branch-owned profile metadata is missing for ${PROFILE}" >&2
+ return 1
+ }
+ if ! profile_owner_validate_selection "${profile_env}" "${ports_env}" \
+ "${WORKTREE_ROOT}" "${branch}" "${PROFILE}"; then
+ profile_owner_print_error
+ return 1
+ fi
+}
+
+validate_branch_profile_ownership
+
if [[ ! -f "${BRANCH_PROFILE_PORTS_ENV}" ]] && is_branch_profile; then
echo "ERROR: missing branch-scoped port cache for minikube profile: ${PROFILE}" >&2
echo "Expected ${BRANCH_PROFILE_PORTS_ENV}" >&2
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 98bfa860e..4b2bf3d71 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -386,28 +386,43 @@ PY
t2_profile_status() {
local status_text status_rc=0
- status_text="$(t2_mk status 2>/dev/null)" || status_rc=$?
- if [ -z "$status_text" ]; then
- T2_PROFILE_STATUS=missing
+ # Keep stderr so a missing profile remains distinguishable from a deadline,
+ # transport, or permission failure. An empty result from a failed bounded
+ # probe is not evidence that bootstrap is safe.
+ status_text="$(t2_mk status 2>&1)" || status_rc=$?
+ if [ -n "$status_text" ] && minikube_profile_status_is_missing_or_stopped "$status_text"; then
+ if [[ "$status_text" == *Nonexistent* ||
+ "$status_text" == *'does not exist'* ||
+ "$status_text" == *'not found'* ]]; then
+ T2_PROFILE_STATUS=missing
+ T2_PLAN_REASON='profile is missing or uninitialized'
+ else
+ T2_PROFILE_STATUS=stopped
+ T2_PLAN_REASON='profile is missing or stopped'
+ fi
T2_BOOTSTRAP_REQUIRED=true
T2_PROFILE_HEALTHY=false
T2_PLAN_STATE=full-bootstrap
- T2_PLAN_REASON='profile is missing or uninitialized'
return 0
fi
- if minikube_profile_status_is_missing_or_stopped "$status_text"; then
- T2_PROFILE_STATUS=stopped
- T2_BOOTSTRAP_REQUIRED=true
+ if [ "$status_rc" -ne 0 ]; then
+ T2_PROFILE_STATUS=unhealthy
T2_PROFILE_HEALTHY=false
- T2_PLAN_STATE=full-bootstrap
- T2_PLAN_REASON='profile is missing or stopped'
- return 0
+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"
+ t2_fail PROFILE_UNHEALTHY "Minikube status probe failed with exit $status_rc: $T2_PROFILE"
+ return 1
fi
- if [ "$status_rc" -ne 0 ] || ! minikube_profile_status_is_healthy "$status_text"; then
+ if [ -z "$status_text" ] || ! minikube_profile_status_is_healthy "$status_text"; then
T2_PROFILE_STATUS=unhealthy
T2_PROFILE_HEALTHY=false
- T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-start"
- t2_fail PROFILE_UNHEALTHY "Minikube profile is not healthy: $T2_PROFILE"
+ if [ -z "$status_text" ]; then
+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"
+ t2_fail PROFILE_UNHEALTHY "Minikube status probe returned no output: $T2_PROFILE"
+ else
+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-start"
+ t2_fail PROFILE_UNHEALTHY "Minikube profile is not healthy: $T2_PROFILE"
+ fi
+ return 1
fi
T2_PROFILE_STATUS=healthy
T2_PROFILE_HEALTHY=true
@@ -756,6 +771,31 @@ t2_port_forward_targets_context() {
[ "$port_forward" = true ] && [ "$context_matches" = true ]
}
+t2_port_forward_uses_profile_port() {
+ local command_line="$1" index arg local_port key port
+ local -a argv=()
+ read -r -a argv <<<"$command_line"
+ [ "${#argv[@]}" -gt 0 ] || return 1
+ for ((index = 1; index < ${#argv[@]}; index += 1)); do
+ [ "${argv[$index]}" = port-forward ] || continue
+ for ((index += 1; index < ${#argv[@]}; index += 1)); do
+ arg="${argv[$index]}"
+ if [[ "$arg" =~ ^([0-9]+):[0-9]+$ ]]; then
+ local_port="${BASH_REMATCH[1]}"
+ elif [[ "$arg" =~ ^[0-9]+$ ]]; then
+ local_port="$arg"
+ else
+ continue
+ fi
+ while IFS='=' read -r key port; do
+ [ "$port" = "$local_port" ] && return 0
+ done < <(awk -F= '/^(CONTROL_UI_PORT|PROFILE_UI_PORT|MCP_HOST_PORT|REGISTRY_API_PORT|CONTROL_API_PORT|EXTERNAL_REST_API_PORT|MEMBER_REGISTRATION_SERVICE_PORT|RPC_PROXY_PORT|WORKFLOW_APPROVAL_READER_PORT)=[0-9]+$/ { print $1 "=" $2 }' "$T2_PORTS_ENV" 2>/dev/null || true)
+ done
+ return 1
+ done
+ return 1
+}
+
t2_pid_file_matches_process() {
local pid_file="$1" pid="$2"
pf_owner_read_record "$pid_file" || return 1
@@ -796,9 +836,16 @@ t2_process_check() {
*) continue ;;
esac
# Context-less or differently pinned developer forwards are not
- # attributable to this profile. Exact-token matching prevents a profile
- # name that is merely a substring of another context from becoming ours.
+ # attributable to this profile. A forward on one of this profile's
+ # persisted local ports is still a conflict, even without a usable
+ # context, because it could make the user-facing health probe hit another
+ # cluster. Unrelated profile ports remain eligible for parallel lanes.
if ! t2_port_forward_targets_context "$command_line"; then
+ if t2_port_forward_uses_profile_port "$command_line"; then
+ T2_NEXT_COMMAND='stop the contextless or foreign port-forward occupying a persisted profile port'
+ t2_fail PORT_FORWARD_CONFLICT 'a contextless or foreign port-forward occupies a port owned by this profile'
+ return 1
+ fi
continue
fi
matching_records=0
diff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.py
index 5932c6515..669a53f25 100755
--- a/scripts/minikube/writer-recovery-state.py
+++ b/scripts/minikube/writer-recovery-state.py
@@ -140,7 +140,10 @@ def read_state(args: argparse.Namespace) -> None:
str(data.get("traceReplicas")), "trace-maintenance-worker", allow_zero=True
)
control_api = validate_replica(str(data.get("controlApiReplicas")), "control-api")
- print(f"{phase}|{hcc}|{workflow}|{trace}|{control_api}")
+ output = f"{phase}|{hcc}|{workflow}|{trace}|{control_api}"
+ if args.include_head:
+ output += f"|{stored_head}"
+ print(output)
def clear_state(args: argparse.Namespace) -> None:
@@ -167,7 +170,8 @@ def parser() -> argparse.ArgumentParser:
write.add_argument("--workflow", required=True)
write.add_argument("--trace", required=True)
write.add_argument("--control-api", required=True)
- subparsers.add_parser("read", parents=[common])
+ read = subparsers.add_parser("read", parents=[common])
+ read.add_argument("--include-head", action="store_true")
subparsers.add_parser("clear", parents=[common])
return result
diff --git a/scripts/tests/test-minikube-full-setup.sh b/scripts/tests/test-minikube-full-setup.sh
index daeeaad40..f5246728f 100755
--- a/scripts/tests/test-minikube-full-setup.sh
+++ b/scripts/tests/test-minikube-full-setup.sh
@@ -1380,6 +1380,18 @@ STUB
rm -rf "$d"
}
+assert_recovery_migrations_are_head_bound() {
+ local current_head_checks
+ current_head_checks="$(grep -Fc '[ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]' scripts/minikube/full-setup.sh)"
+ if grep -Fq 'writer_recovery_state_cli read --include-head' scripts/minikube/full-setup.sh && \
+ grep -Fq 'PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD' scripts/minikube/full-setup.sh && \
+ [ "$current_head_checks" -ge 2 ]; then
+ pass "recovery resumes with writers fenced and reruns migrations after a HEAD change"
+ else
+ fail "recovery can reuse a historical migration boundary after HEAD changes"
+ fi
+}
+
# The guard that keeps this file honest: a case defined but never added to the
# call block below reports nothing at all, which reads as a green run.
assert_every_defined_case_is_invoked() {
@@ -1430,6 +1442,7 @@ assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_tree
assert_the_unoverridden_path_generates_the_api_ip_patch_once
assert_trace_writer_fence_closes_and_proves_zero_replicas
assert_trace_writer_fence_is_behaviorally_fail_closed
+assert_recovery_migrations_are_head_bound
assert_skip_build_follows_the_recorded_image_source
assert_skip_build_says_it_is_following_the_cluster
assert_an_acquiring_run_still_honours_image_source
diff --git a/scripts/tests/test-minikube-port-forward-owner.sh b/scripts/tests/test-minikube-port-forward-owner.sh
index 4c9ab7836..2a0d6bad0 100755
--- a/scripts/tests/test-minikube-port-forward-owner.sh
+++ b/scripts/tests/test-minikube-port-forward-owner.sh
@@ -17,6 +17,8 @@ pass() {
}
[[ -f "${OWNER}" ]] || fail 'port-forward-owner.sh is missing'
+grep -Fq 'profile_owner_validate_selection' "${PF_ALL}" ||
+ fail 'branch-owned pf-all-stack does not validate worktree/profile ownership'
# shellcheck source=scripts/minikube/port-forward-owner.sh
source "${OWNER}"
diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.sh
index 22626f406..94e1387dc 100755
--- a/scripts/tests/test-minikube-sync-auth-key.sh
+++ b/scripts/tests/test-minikube-sync-auth-key.sh
@@ -19,7 +19,14 @@ T2_LOCK_ROOT="${TMP_DIR}/locks"
T2_LOCK_TOKEN='auth-sync-test-token'
T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"
T2_PROCESS_START=unavailable
+# GitHub Actions checks out a detached commit. The fixture still needs a
+# non-empty logical lane name because the production lease contract rejects
+# owner records without a branch identity; use the PR source ref when it is
+# available and a test-only identity otherwise.
T2_TEST_BRANCH="$(git -C "${ROOT}" branch --show-current)"
+if [[ -z "${T2_TEST_BRANCH}" ]]; then
+ T2_TEST_BRANCH="${GITHUB_HEAD_REF:-detached-ci-test}"
+fi
T2_TEST_HEAD="$(git -C "${ROOT}" rev-parse --verify HEAD)"
T2_TEST_WORKTREE_ID="$(printf '%s' "${ROOT}" | shasum | awk '{print $1}')"
T2_TEST_LOCK_KEY="$(printf '%s\0%s\0%s\0%s\0%s' "${ROOT}" "${T2_TEST_BRANCH}" "${T2_TEST_HEAD}" fake fake | shasum | awk '{print $1}')"
diff --git a/scripts/tests/test-minikube-t2-process-owner.sh b/scripts/tests/test-minikube-t2-process-owner.sh
index 484c3242e..a309f2ac5 100755
--- a/scripts/tests/test-minikube-t2-process-owner.sh
+++ b/scripts/tests/test-minikube-t2-process-owner.sh
@@ -21,6 +21,18 @@ PS_BIN="$TEST_DIR/bin"
PS_EF="$PS_DATA/ef"
RECORD="$PID_DIR/control-ui.pid"
mkdir -p "$PID_DIR" "$PS_DATA" "$PS_BIN"
+PORTS_ENV="$TEST_DIR/ports.env"
+cat >"$PORTS_ENV" <<'EOF_PORTS'
+CONTROL_UI_PORT=3000
+PROFILE_UI_PORT=3001
+MCP_HOST_PORT=3080
+REGISTRY_API_PORT=3085
+CONTROL_API_PORT=3090
+EXTERNAL_REST_API_PORT=3091
+MEMBER_REGISTRATION_SERVICE_PORT=3092
+RPC_PROXY_PORT=3094
+WORKFLOW_APPROVAL_READER_PORT=3098
+EOF_PORTS
cat >"$PS_BIN/ps" <<'EOF_PS'
#!/usr/bin/env bash
@@ -62,7 +74,7 @@ run_process_check() {
local output="$1"
T2_PS_DATA_DIR="$PS_DATA" PATH="$PS_BIN:$PATH" \
T2_PROFILE="$PROFILE" T2_CONTEXT="$PROFILE" MINIKUBE_PROFILE="$PROFILE" \
- T2_PROFILE_ROOT="$PROFILE_ROOT" T2_PROJECT_DIR="$ROOT" \
+ T2_PROFILE_ROOT="$PROFILE_ROOT" T2_PORTS_ENV="$PORTS_ENV" T2_PROJECT_DIR="$ROOT" \
bash -c 'source "$1"; t2_process_check' _ "$COMMON" >"$output" 2>&1
}
@@ -78,10 +90,21 @@ set_process_fixture bash "bash -c echo kubectl --context=$PROFILE port-forward"
run_process_check "$TEST_DIR/wrapper.out"
set_process_fixture kubectl 'kubectl -n control-plane port-forward svc/control-ui 3000:3000'
-run_process_check "$TEST_DIR/contextless.out"
+if run_process_check "$TEST_DIR/contextless.out"; then
+ printf 'FAIL: contextless forward on a persisted profile port was accepted\n' >&2
+ exit 1
+fi
+grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/contextless.out"
set_process_fixture kubectl 'kubectl --context=another-profile -n control-plane port-forward svc/control-ui 3000:3000'
-run_process_check "$TEST_DIR/other-context.out"
+if run_process_check "$TEST_DIR/other-context.out"; then
+ printf 'FAIL: foreign-context forward on a persisted profile port was accepted\n' >&2
+ exit 1
+fi
+grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/other-context.out"
+
+set_process_fixture kubectl 'kubectl -n control-plane port-forward svc/control-ui 3999:3999'
+run_process_check "$TEST_DIR/unrelated-port.out"
# shellcheck source=scripts/minikube/port-forward-owner.sh
source "$OWNER"
diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.sh
index b9667efc3..5a00d5e17 100755
--- a/scripts/tests/test-minikube-t2-scenarios.sh
+++ b/scripts/tests/test-minikube-t2-scenarios.sh
@@ -166,6 +166,9 @@ expect_code DEVELOPMENT_SCOPE_REQUIRED wrong-profile-identity wrong-profile-iden
missing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"
[ "$missing_profile_state" = full-bootstrap ] || fail "missing profile selected $missing_profile_state instead of full-bootstrap"
+expect_code PROFILE_UNHEALTHY status-probe-timeout status-probe-timeout \
+ env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ return 124; }; t2_profile_status' bash "$COMMON"
+
healthy_profile_state="$(env T2_PROJECT_DIR="$repo" MINIKUBE_PROFILE="$profile" T2_CONTEXT="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" T2_PROFILE_ROOT="$tmp/profiles" T2_PROFILE_ENV="$profile_root/profile.env" T2_PORTS_ENV="$profile_root/ports.env" T2_REQUIRED_DEPLOYMENTS=gfs/gfsc-reader T2_BRANCH=feat/scenario T2_HEAD="$feature_sha" T2_LOCK_ROOT="$tmp/locks" T2_EVIDENCE_ROOT="$tmp/evidence" bash -c 'source "$1"; t2_mk(){ printf "%s" "host: Running\nkubelet: Running\napiserver: Running"; }; t2_profile_status; printf "%s:%s" "$T2_PROFILE_STATUS" "$T2_PROFILE_HEALTHY"' bash "$COMMON")"
[ "$healthy_profile_state" = healthy:true ] || fail "healthy profile selected $healthy_profile_state instead of healthy:true"
diff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.sh
index ab75c45a4..00f687bf5 100755
--- a/scripts/tests/test-minikube-writer-recovery-state.sh
+++ b/scripts/tests/test-minikube-writer-recovery-state.sh
@@ -48,6 +48,9 @@ for phase in api-restoring overlay-applying; do
done
python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 | grep -Fxq 'overlay-applying|1|2|1|1' ||
fail 'state did not resume under a different HEAD on the same owned lane'
+python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 --include-head |
+ grep -Fxq 'overlay-applying|1|2|1|1|0123456789abcdef' ||
+ fail 'state did not expose its historical HEAD for migration freshness checks'
[[ -s "$STATE" ]] || fail 'identity mismatch destroyed durable state'
pass 'writer recovery state preserves lane identity while retaining historical HEAD'