Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(minikube): harden recovery and profile forwarding checks
Alfredo Lopez0bc39b850e03
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex ea64d749b..63be04220 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -1133,6 +1133,7 @@ WRITER_RECOVERY_STATE_ROOT="${PROJECT_DIR}/.local-notes/infra/runs/writer-recoveWRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"WRITER_RECOVERY_STATE_LOADED=falseWRITER_RECOVERY_STATE_PHASE=""+WRITER_RECOVERY_STATE_HEAD=""WRITER_RECOVERY_FENCE_PENDING=falseWRITER_RECOVERY_MIGRATIONS_COMPLETE=falseK8S_API_EGRESS_POLICY_FILE=""@@ -1176,7 +1177,7 @@ writer_recovery_state_clear() {writer_recovery_state_load() {local state_output state_status=0- state_output="$(writer_recovery_state_cli read)" || state_status=$?+ state_output="$(writer_recovery_state_cli read --include-head)" || state_status=$?if [ "$state_status" -ne 0 ]; thenerr "Unable to validate the durable writer-recovery state; refusing profile mutation"return 1@@ -1186,7 +1187,7 @@ writer_recovery_state_load() {fiIFS='|' read -r WRITER_RECOVERY_STATE_PHASE PARTIAL_HCC_REPLICAS \PARTIAL_WORKFLOW_REPLICAS PARTIAL_TRACE_REPLICAS \- PARTIAL_CONTROL_API_REPLICAS <<<"$state_output"+ PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD <<<"$state_output"[ -n "$WRITER_RECOVERY_STATE_PHASE" ] &&[[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] &&[[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] &&@@ -1259,7 +1260,11 @@ writer_recovery_state_load() {PARTIAL_WORKFLOW_FENCED=truePARTIAL_TRACE_FENCED=truePARTIAL_CONTROL_API_FENCED=true- WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ else+ log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"+ fi# The phase is written before the control-api restore. An interruption# after scale but before the next phase must close every writer again.WRITER_RECOVERY_FENCE_PENDING=true@@ -1270,7 +1275,11 @@ writer_recovery_state_load() {PARTIAL_HCC_FENCED=truePARTIAL_WORKFLOW_FENCED=truePARTIAL_CONTROL_API_FENCED=true- WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ else+ log "Recovery state belongs to historical HEAD ${WRITER_RECOVERY_STATE_HEAD}; re-running migrations for ${T2_HEAD} while writers remain fenced"+ fi# The overlay is rendered with all four database-writer Deployments at zero.# Re-fence on every post-migration resume because an interrupted apply# may have recreated a writer before its durable phase was updated.diff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.shindex adf77037e..b0461aff3 100755--- a/scripts/minikube/pf-all-stack.sh+++ b/scripts/minikube/pf-all-stack.sh@@ -11,6 +11,8 @@ WORKTREE_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel 2>/dev/nWORKTREE_ROOT="$(cd -- "${WORKTREE_ROOT}" && pwd -P)"# shellcheck source=scripts/minikube/port-forward-owner.shsource "${SCRIPT_DIR}/port-forward-owner.sh"+# shellcheck source=scripts/minikube/profile-owner.sh+source "${SCRIPT_DIR}/profile-owner.sh"PROFILE="${MINIKUBE_PROFILE:-clerum-test}"CONTEXT="${KUBECONTEXT:-${PROFILE}}"@@ -121,6 +123,29 @@ is_branch_profile() {[[ "$PROFILE" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]}+validate_branch_profile_ownership() {+ [[ "${PROFILE}" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]] || return 0+ local branch profile_env ports_env+ branch="$(git -C "${WORKTREE_ROOT}" branch --show-current 2>/dev/null || true)"+ [[ -n "${branch}" ]] || {+ echo "ERROR: branch-owned port-forwards require a named Git branch" >&2+ return 1+ }+ profile_env="${T2_PROFILE_ENV:-${PROFILE_PID_ROOT}/${PROFILE}/profile.env}"+ ports_env="${CLERUM_PROFILE_PORTS_ENV:-${PROFILE_PID_ROOT}/${PROFILE}/ports.env}"+ [[ -f "${profile_env}" && -f "${ports_env}" ]] || {+ echo "ERROR: branch-owned profile metadata is missing for ${PROFILE}" >&2+ return 1+ }+ if ! profile_owner_validate_selection "${profile_env}" "${ports_env}" \+ "${WORKTREE_ROOT}" "${branch}" "${PROFILE}"; then+ profile_owner_print_error+ return 1+ fi+}++validate_branch_profile_ownership+if [[ ! -f "${BRANCH_PROFILE_PORTS_ENV}" ]] && is_branch_profile; thenecho "ERROR: missing branch-scoped port cache for minikube profile: ${PROFILE}" >&2echo "Expected ${BRANCH_PROFILE_PORTS_ENV}" >&2diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex 98bfa860e..4b2bf3d71 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -386,28 +386,43 @@ PYt2_profile_status() {local status_text status_rc=0- status_text="$(t2_mk status 2>/dev/null)" || status_rc=$?- if [ -z "$status_text" ]; then- T2_PROFILE_STATUS=missing+ # Keep stderr so a missing profile remains distinguishable from a deadline,+ # transport, or permission failure. An empty result from a failed bounded+ # probe is not evidence that bootstrap is safe.+ status_text="$(t2_mk status 2>&1)" || status_rc=$?+ if [ -n "$status_text" ] && minikube_profile_status_is_missing_or_stopped "$status_text"; then+ if [[ "$status_text" == *Nonexistent* ||+ "$status_text" == *'does not exist'* ||+ "$status_text" == *'not found'* ]]; then+ T2_PROFILE_STATUS=missing+ T2_PLAN_REASON='profile is missing or uninitialized'+ else+ T2_PROFILE_STATUS=stopped+ T2_PLAN_REASON='profile is missing or stopped'+ fiT2_BOOTSTRAP_REQUIRED=trueT2_PROFILE_HEALTHY=falseT2_PLAN_STATE=full-bootstrap- T2_PLAN_REASON='profile is missing or uninitialized'return 0fi- if minikube_profile_status_is_missing_or_stopped "$status_text"; then- T2_PROFILE_STATUS=stopped- T2_BOOTSTRAP_REQUIRED=true+ if [ "$status_rc" -ne 0 ]; then+ T2_PROFILE_STATUS=unhealthyT2_PROFILE_HEALTHY=false- T2_PLAN_STATE=full-bootstrap- T2_PLAN_REASON='profile is missing or stopped'- return 0+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"+ t2_fail PROFILE_UNHEALTHY "Minikube status probe failed with exit $status_rc: $T2_PROFILE"+ return 1fi- if [ "$status_rc" -ne 0 ] || ! minikube_profile_status_is_healthy "$status_text"; then+ if [ -z "$status_text" ] || ! minikube_profile_status_is_healthy "$status_text"; thenT2_PROFILE_STATUS=unhealthyT2_PROFILE_HEALTHY=false- T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-start"- t2_fail PROFILE_UNHEALTHY "Minikube profile is not healthy: $T2_PROFILE"+ if [ -z "$status_text" ]; then+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"+ t2_fail PROFILE_UNHEALTHY "Minikube status probe returned no output: $T2_PROFILE"+ else+ T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-start"+ t2_fail PROFILE_UNHEALTHY "Minikube profile is not healthy: $T2_PROFILE"+ fi+ return 1fiT2_PROFILE_STATUS=healthyT2_PROFILE_HEALTHY=true@@ -756,6 +771,31 @@ t2_port_forward_targets_context() {[ "$port_forward" = true ] && [ "$context_matches" = true ]}+t2_port_forward_uses_profile_port() {+ local command_line="$1" index arg local_port key port+ local -a argv=()+ read -r -a argv <<<"$command_line"+ [ "${#argv[@]}" -gt 0 ] || return 1+ for ((index = 1; index < ${#argv[@]}; index += 1)); do+ [ "${argv[$index]}" = port-forward ] || continue+ for ((index += 1; index < ${#argv[@]}; index += 1)); do+ arg="${argv[$index]}"+ if [[ "$arg" =~ ^([0-9]+):[0-9]+$ ]]; then+ local_port="${BASH_REMATCH[1]}"+ elif [[ "$arg" =~ ^[0-9]+$ ]]; then+ local_port="$arg"+ else+ continue+ fi+ while IFS='=' read -r key port; do+ [ "$port" = "$local_port" ] && return 0+ done < <(awk -F= '/^(CONTROL_UI_PORT|PROFILE_UI_PORT|MCP_HOST_PORT|REGISTRY_API_PORT|CONTROL_API_PORT|EXTERNAL_REST_API_PORT|MEMBER_REGISTRATION_SERVICE_PORT|RPC_PROXY_PORT|WORKFLOW_APPROVAL_READER_PORT)=[0-9]+$/ { print $1 "=" $2 }' "$T2_PORTS_ENV" 2>/dev/null || true)+ done+ return 1+ done+ return 1+}+t2_pid_file_matches_process() {local pid_file="$1" pid="$2"pf_owner_read_record "$pid_file" || return 1@@ -796,9 +836,16 @@ t2_process_check() {*) continue ;;esac# Context-less or differently pinned developer forwards are not- # attributable to this profile. Exact-token matching prevents a profile- # name that is merely a substring of another context from becoming ours.+ # attributable to this profile. A forward on one of this profile's+ # persisted local ports is still a conflict, even without a usable+ # context, because it could make the user-facing health probe hit another+ # cluster. Unrelated profile ports remain eligible for parallel lanes.if ! t2_port_forward_targets_context "$command_line"; then+ if t2_port_forward_uses_profile_port "$command_line"; then+ T2_NEXT_COMMAND='stop the contextless or foreign port-forward occupying a persisted profile port'+ t2_fail PORT_FORWARD_CONFLICT 'a contextless or foreign port-forward occupies a port owned by this profile'+ return 1+ ficontinuefimatching_records=0diff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.pyindex 5932c6515..669a53f25 100755--- a/scripts/minikube/writer-recovery-state.py+++ b/scripts/minikube/writer-recovery-state.py@@ -140,7 +140,10 @@ def read_state(args: argparse.Namespace) -> None:str(data.get("traceReplicas")), "trace-maintenance-worker", allow_zero=True)control_api = validate_replica(str(data.get("controlApiReplicas")), "control-api")- print(f"{phase}|{hcc}|{workflow}|{trace}|{control_api}")+ output = f"{phase}|{hcc}|{workflow}|{trace}|{control_api}"+ if args.include_head:+ output += f"|{stored_head}"+ print(output)def clear_state(args: argparse.Namespace) -> None:@@ -167,7 +170,8 @@ def parser() -> argparse.ArgumentParser:write.add_argument("--workflow", required=True)write.add_argument("--trace", required=True)write.add_argument("--control-api", required=True)- subparsers.add_parser("read", parents=[common])+ read = subparsers.add_parser("read", parents=[common])+ read.add_argument("--include-head", action="store_true")subparsers.add_parser("clear", parents=[common])return resultdiff --git a/scripts/tests/test-minikube-full-setup.sh b/scripts/tests/test-minikube-full-setup.shindex daeeaad40..f5246728f 100755--- a/scripts/tests/test-minikube-full-setup.sh+++ b/scripts/tests/test-minikube-full-setup.sh@@ -1380,6 +1380,18 @@ STUBrm -rf "$d"}+assert_recovery_migrations_are_head_bound() {+ local current_head_checks+ current_head_checks="$(grep -Fc '[ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]' scripts/minikube/full-setup.sh)"+ if grep -Fq 'writer_recovery_state_cli read --include-head' scripts/minikube/full-setup.sh && \+ grep -Fq 'PARTIAL_CONTROL_API_REPLICAS WRITER_RECOVERY_STATE_HEAD' scripts/minikube/full-setup.sh && \+ [ "$current_head_checks" -ge 2 ]; then+ pass "recovery resumes with writers fenced and reruns migrations after a HEAD change"+ else+ fail "recovery can reuse a historical migration boundary after HEAD changes"+ fi+}+# The guard that keeps this file honest: a case defined but never added to the# call block below reports nothing at all, which reads as a green run.assert_every_defined_case_is_invoked() {@@ -1430,6 +1442,7 @@ assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_treeassert_the_unoverridden_path_generates_the_api_ip_patch_onceassert_trace_writer_fence_closes_and_proves_zero_replicasassert_trace_writer_fence_is_behaviorally_fail_closed+assert_recovery_migrations_are_head_boundassert_skip_build_follows_the_recorded_image_sourceassert_skip_build_says_it_is_following_the_clusterassert_an_acquiring_run_still_honours_image_sourcediff --git a/scripts/tests/test-minikube-port-forward-owner.sh b/scripts/tests/test-minikube-port-forward-owner.shindex 4c9ab7836..2a0d6bad0 100755--- a/scripts/tests/test-minikube-port-forward-owner.sh+++ b/scripts/tests/test-minikube-port-forward-owner.sh@@ -17,6 +17,8 @@ pass() {}[[ -f "${OWNER}" ]] || fail 'port-forward-owner.sh is missing'+grep -Fq 'profile_owner_validate_selection' "${PF_ALL}" ||+ fail 'branch-owned pf-all-stack does not validate worktree/profile ownership'# shellcheck source=scripts/minikube/port-forward-owner.shsource "${OWNER}"diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.shindex 22626f406..94e1387dc 100755--- a/scripts/tests/test-minikube-sync-auth-key.sh+++ b/scripts/tests/test-minikube-sync-auth-key.sh@@ -19,7 +19,14 @@ T2_LOCK_ROOT="${TMP_DIR}/locks"T2_LOCK_TOKEN='auth-sync-test-token'T2_LOCK_DIR="${T2_LOCK_ROOT}/fake.lock"T2_PROCESS_START=unavailable+# GitHub Actions checks out a detached commit. The fixture still needs a+# non-empty logical lane name because the production lease contract rejects+# owner records without a branch identity; use the PR source ref when it is+# available and a test-only identity otherwise.T2_TEST_BRANCH="$(git -C "${ROOT}" branch --show-current)"+if [[ -z "${T2_TEST_BRANCH}" ]]; then+ T2_TEST_BRANCH="${GITHUB_HEAD_REF:-detached-ci-test}"+fiT2_TEST_HEAD="$(git -C "${ROOT}" rev-parse --verify HEAD)"T2_TEST_WORKTREE_ID="$(printf '%s' "${ROOT}" | shasum | awk '{print $1}')"T2_TEST_LOCK_KEY="$(printf '%s\0%s\0%s\0%s\0%s' "${ROOT}" "${T2_TEST_BRANCH}" "${T2_TEST_HEAD}" fake fake | shasum | awk '{print $1}')"diff --git a/scripts/tests/test-minikube-t2-process-owner.sh b/scripts/tests/test-minikube-t2-process-owner.shindex 484c3242e..a309f2ac5 100755--- a/scripts/tests/test-minikube-t2-process-owner.sh+++ b/scripts/tests/test-minikube-t2-process-owner.sh@@ -21,6 +21,18 @@ PS_BIN="$TEST_DIR/bin"PS_EF="$PS_DATA/ef"RECORD="$PID_DIR/control-ui.pid"mkdir -p "$PID_DIR" "$PS_DATA" "$PS_BIN"+PORTS_ENV="$TEST_DIR/ports.env"+cat >"$PORTS_ENV" <<'EOF_PORTS'+CONTROL_UI_PORT=3000+PROFILE_UI_PORT=3001+MCP_HOST_PORT=3080+REGISTRY_API_PORT=3085+CONTROL_API_PORT=3090+EXTERNAL_REST_API_PORT=3091+MEMBER_REGISTRATION_SERVICE_PORT=3092+RPC_PROXY_PORT=3094+WORKFLOW_APPROVAL_READER_PORT=3098+EOF_PORTScat >"$PS_BIN/ps" <<'EOF_PS'#!/usr/bin/env bash@@ -62,7 +74,7 @@ run_process_check() {local output="$1"T2_PS_DATA_DIR="$PS_DATA" PATH="$PS_BIN:$PATH" \T2_PROFILE="$PROFILE" T2_CONTEXT="$PROFILE" MINIKUBE_PROFILE="$PROFILE" \- T2_PROFILE_ROOT="$PROFILE_ROOT" T2_PROJECT_DIR="$ROOT" \+ T2_PROFILE_ROOT="$PROFILE_ROOT" T2_PORTS_ENV="$PORTS_ENV" T2_PROJECT_DIR="$ROOT" \bash -c 'source "$1"; t2_process_check' _ "$COMMON" >"$output" 2>&1}@@ -78,10 +90,21 @@ set_process_fixture bash "bash -c echo kubectl --context=$PROFILE port-forward"run_process_check "$TEST_DIR/wrapper.out"set_process_fixture kubectl 'kubectl -n control-plane port-forward svc/control-ui 3000:3000'-run_process_check "$TEST_DIR/contextless.out"+if run_process_check "$TEST_DIR/contextless.out"; then+ printf 'FAIL: contextless forward on a persisted profile port was accepted\n' >&2+ exit 1+fi+grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/contextless.out"set_process_fixture kubectl 'kubectl --context=another-profile -n control-plane port-forward svc/control-ui 3000:3000'-run_process_check "$TEST_DIR/other-context.out"+if run_process_check "$TEST_DIR/other-context.out"; then+ printf 'FAIL: foreign-context forward on a persisted profile port was accepted\n' >&2+ exit 1+fi+grep -Fq PORT_FORWARD_CONFLICT "$TEST_DIR/other-context.out"++set_process_fixture kubectl 'kubectl -n control-plane port-forward svc/control-ui 3999:3999'+run_process_check "$TEST_DIR/unrelated-port.out"# shellcheck source=scripts/minikube/port-forward-owner.shsource "$OWNER"diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex b9667efc3..5a00d5e17 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -166,6 +166,9 @@ expect_code DEVELOPMENT_SCOPE_REQUIRED wrong-profile-identity wrong-profile-idenmissing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"[ "$missing_profile_state" = full-bootstrap ] || fail "missing profile selected $missing_profile_state instead of full-bootstrap"+expect_code PROFILE_UNHEALTHY status-probe-timeout status-probe-timeout \+ env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ return 124; }; t2_profile_status' bash "$COMMON"+healthy_profile_state="$(env T2_PROJECT_DIR="$repo" MINIKUBE_PROFILE="$profile" T2_CONTEXT="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" T2_PROFILE_ROOT="$tmp/profiles" T2_PROFILE_ENV="$profile_root/profile.env" T2_PORTS_ENV="$profile_root/ports.env" T2_REQUIRED_DEPLOYMENTS=gfs/gfsc-reader T2_BRANCH=feat/scenario T2_HEAD="$feature_sha" T2_LOCK_ROOT="$tmp/locks" T2_EVIDENCE_ROOT="$tmp/evidence" bash -c 'source "$1"; t2_mk(){ printf "%s" "host: Running\nkubelet: Running\napiserver: Running"; }; t2_profile_status; printf "%s:%s" "$T2_PROFILE_STATUS" "$T2_PROFILE_HEALTHY"' bash "$COMMON")"[ "$healthy_profile_state" = healthy:true ] || fail "healthy profile selected $healthy_profile_state instead of healthy:true"diff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.shindex ab75c45a4..00f687bf5 100755--- a/scripts/tests/test-minikube-writer-recovery-state.sh+++ b/scripts/tests/test-minikube-writer-recovery-state.sh@@ -48,6 +48,9 @@ for phase in api-restoring overlay-applying; dodonepython3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 | grep -Fxq 'overlay-applying|1|2|1|1' ||fail 'state did not resume under a different HEAD on the same owned lane'+python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 --include-head |+ grep -Fxq 'overlay-applying|1|2|1|1|0123456789abcdef' ||+ fail 'state did not expose its historical HEAD for migration freshness checks'[[ -s "$STATE" ]] || fail 'identity mismatch destroyed durable state'pass 'writer recovery state preserves lane identity while retaining historical HEAD'