Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(minikube): fail closed on interrupted writer recovery
Alfredo Lopez5fd79ba90df8
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 6676d18c6..2eea890fc 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -683,11 +683,23 @@ ensure_control_postgres_ready() {# close all four writers again before the later recovery code resumes. The# journal is deliberately not advanced here: if this guard is interrupted, the# old phase remains conservative and the next run repeats the fence.+guard_interrupted_writer_deployment() {+ local deployment="$1" selector="$2" desired pods+ $KC -n control-plane scale "deployment/${deployment}" --replicas=0 >/dev/null || return 1+ desired="$($KC -n control-plane get "deployment/${deployment}" \+ -o 'jsonpath={.spec.replicas}')" || return 1+ [ "$desired" = 0 ] || { err "${deployment} did not converge to zero desired replicas before setup mutation"; return 1; }+ pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1+ [ -z "$pods" ] || $KC -n control-plane wait --for=delete pod \+ -l "$selector" --timeout=180s >/dev/null || return 1+ pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1+ [ -z "$pods" ] || { err "${deployment} pods remain after the pre-mutation fence"; return 1; }+}+guard_interrupted_writer_recovery() {local state_helper="${SCRIPT_DIR}/writer-recovery-state.py"local state_file="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery/${PROFILE}.json"local state_output state_status=0 phase hcc_replicas workflow_replicas trace_replicas control_api_replicas state_head- local pods[[ -f "${state_file}" ]] || return 0[[ -f "${state_helper}" ]] || {@@ -724,35 +736,10 @@ guard_interrupted_writer_recovery() {}log "Resuming interrupted writer recovery from phase ${phase} (recorded HEAD ${state_head}) — fencing all writers before setup mutations"- $KC -n control-plane scale deployment/host-context-controller --replicas=0 >/dev/null- pods="$($KC -n control-plane get pods -l app=host-context-controller -o name)" || return 1- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \- -l app=host-context-controller --timeout=180s >/dev/null-- $KC -n control-plane scale deployment/workflow-recipes --replicas=0 >/dev/null- pods="$($KC -n control-plane get pods -l app=workflow-recipes -o name)" || return 1- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \- -l app=workflow-recipes --timeout=180s >/dev/null-- $KC -n control-plane scale deployment/trace-maintenance-worker --replicas=0 >/dev/null- [ "$($KC -n control-plane get deployment/trace-maintenance-worker \- -o 'jsonpath={.spec.replicas}')" = 0 ] || {- err "trace-maintenance-worker did not converge to zero before setup mutation"- return 1- }- pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" || return 1- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \- -l app=trace-maintenance-worker --timeout=180s >/dev/null- pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" || return 1- [ -z "${pods}" ] || {- err "trace-maintenance-worker pods remain after the pre-mutation fence"- return 1- }-- $KC -n control-plane scale deployment/control-api --replicas=0 >/dev/null- pods="$($KC -n control-plane get pods -l 'app=control-api,!clerum.io/component' -o name)" || return 1- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \- -l 'app=control-api,!clerum.io/component' --timeout=180s >/dev/null+ guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller' || return 1+ guard_interrupted_writer_deployment workflow-recipes 'app=workflow-recipes' || return 1+ guard_interrupted_writer_deployment trace-maintenance-worker 'app=trace-maintenance-worker' || return 1+ guard_interrupted_writer_deployment control-api 'app=control-api,!clerum.io/component' || return 1}# ======================================================================@@ -1360,6 +1347,7 @@ writer_recovery_state_load() {PARTIAL_BOOTSTRAP_RECOVERY=truePARTIAL_HCC_FENCED=truePARTIAL_WORKFLOW_FENCED=true+ PARTIAL_TRACE_FENCED=truePARTIAL_CONTROL_API_FENCED=trueif [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; thenWRITER_RECOVERY_MIGRATIONS_COMPLETE=truediff --git a/scripts/tests/test-minikube-full-setup.sh b/scripts/tests/test-minikube-full-setup.shindex dd417fa21..d28f44c7a 100755--- a/scripts/tests/test-minikube-full-setup.sh+++ b/scripts/tests/test-minikube-full-setup.sh@@ -1400,6 +1400,88 @@ STUBrm -rf "$d"}+assert_interrupted_recovery_resume_restores_all_writer_fences() {+ local resume_block+ resume_block="$(sed -n '/^ api-restored|overlay-applying|overlay-applied)/,/^ ;;$/p' scripts/minikube/full-setup.sh)"+ if grep -Fq 'PARTIAL_HCC_FENCED=true' <<<"$resume_block" && \+ grep -Fq 'PARTIAL_WORKFLOW_FENCED=true' <<<"$resume_block" && \+ grep -Fq 'PARTIAL_TRACE_FENCED=true' <<<"$resume_block" && \+ grep -Fq 'PARTIAL_CONTROL_API_FENCED=true' <<<"$resume_block"; then+ pass "post-migration recovery resumes with all four writer fences active"+ else+ fail "post-migration recovery can omit a writer fence before restoring the journaled replicas"+ fi+}++assert_interrupted_recovery_fence_is_behaviorally_fail_closed() {+ local d helper run_script out rc problems=""+ d="$(mktemp -d)"+ helper="$(sed -n '/^guard_interrupted_writer_deployment()/,/^}/p' scripts/minikube/full-setup.sh)"++ mkdir -p "$d/bin"+ cat > "$d/bin/kubectl" <<'STUB'+#!/usr/bin/env bash+set -u+case "$*" in+ *"scale deployment/"*)+ [ "${TEST_FENCE_MODE:-}" = scale-fail ] && exit 1+ exit 0+ ;;+ *"get deployment/"*)+ printf '%s' "${TEST_FENCE_DESIRED:-0}"+ exit 0+ ;;+ *"get pods -l"*)+ printf '%s' "${TEST_FENCE_PODS:-}"+ exit 0+ ;;+ *"wait --for=delete pod"*)+ [ "${TEST_FENCE_MODE:-}" = wait-fail ] && exit 1+ exit 0+ ;;+esac+exit 0+STUB+ chmod +x "$d/bin/kubectl"++ run_script="$d/run.sh"+ {+ printf 'set -u\n'+ printf 'KC=%q\n' 'kubectl --context=test'+ printf 'log() { :; }\n'+ printf 'err() { printf "ERR:%%s\\n" "$*" >&2; }\n'+ printf '%s\n' "$helper"+ printf "guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller'\n"+ } >"$run_script"++ out="$(TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"+ rc=$?+ [ "$rc" -eq 0 ] || problems+="healthy zero-replica fence failed; "++ out="$(TEST_FENCE_DESIRED=1 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"+ rc=$?+ [ "$rc" -ne 0 ] || problems+="nonzero desired replicas were accepted; "+ grep -Fq 'did not converge to zero desired replicas' <<<"$out" \+ || problems+="nonzero desired replicas did not fail closed; "++ out="$(TEST_FENCE_MODE=scale-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= \+ PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"+ rc=$?+ [ "$rc" -ne 0 ] || problems+="scale failure was swallowed; "++ out="$(TEST_FENCE_MODE=wait-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS='pod/hcc-0' \+ PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"+ rc=$?+ [ "$rc" -ne 0 ] || problems+="wait failure was swallowed; "++ if [ -z "$problems" ]; then+ pass "interrupted recovery fencing fails closed on scale, convergence, and pod-wait failures"+ else+ fail "$problems"+ fi+ rm -rf "$d"+}+assert_recovery_migrations_are_head_bound() {local current_head_checkscurrent_head_checks="$(grep -Fc '[ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]' scripts/minikube/full-setup.sh)"@@ -1462,6 +1544,8 @@ assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_treeassert_the_unoverridden_path_generates_the_api_ip_patch_onceassert_trace_writer_fence_closes_and_proves_zero_replicasassert_trace_writer_fence_is_behaviorally_fail_closed+assert_interrupted_recovery_resume_restores_all_writer_fences+assert_interrupted_recovery_fence_is_behaviorally_fail_closedassert_recovery_migrations_are_head_boundassert_skip_build_follows_the_recorded_image_sourceassert_skip_build_says_it_is_following_the_cluster