Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): fail closed on interrupted writer recovery

Alfredo Lopez5fd79ba90df8
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 6676d18c6..2eea890fc 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -683,11 +683,23 @@ ensure_control_postgres_ready() {
# close all four writers again before the later recovery code resumes. The
# journal is deliberately not advanced here: if this guard is interrupted, the
# old phase remains conservative and the next run repeats the fence.
+guard_interrupted_writer_deployment() {
+ local deployment="$1" selector="$2" desired pods
+ $KC -n control-plane scale "deployment/${deployment}" --replicas=0 >/dev/null || return 1
+ desired="$($KC -n control-plane get "deployment/${deployment}" \
+ -o 'jsonpath={.spec.replicas}')" || return 1
+ [ "$desired" = 0 ] || { err "${deployment} did not converge to zero desired replicas before setup mutation"; return 1; }
+ pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1
+ [ -z "$pods" ] || $KC -n control-plane wait --for=delete pod \
+ -l "$selector" --timeout=180s >/dev/null || return 1
+ pods="$($KC -n control-plane get pods -l "$selector" -o name)" || return 1
+ [ -z "$pods" ] || { err "${deployment} pods remain after the pre-mutation fence"; return 1; }
+}
+
guard_interrupted_writer_recovery() {
local state_helper="${SCRIPT_DIR}/writer-recovery-state.py"
local state_file="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery/${PROFILE}.json"
local state_output state_status=0 phase hcc_replicas workflow_replicas trace_replicas control_api_replicas state_head
- local pods
[[ -f "${state_file}" ]] || return 0
[[ -f "${state_helper}" ]] || {
@@ -724,35 +736,10 @@ guard_interrupted_writer_recovery() {
}
log "Resuming interrupted writer recovery from phase ${phase} (recorded HEAD ${state_head}) — fencing all writers before setup mutations"
- $KC -n control-plane scale deployment/host-context-controller --replicas=0 >/dev/null
- pods="$($KC -n control-plane get pods -l app=host-context-controller -o name)" || return 1
- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \
- -l app=host-context-controller --timeout=180s >/dev/null
-
- $KC -n control-plane scale deployment/workflow-recipes --replicas=0 >/dev/null
- pods="$($KC -n control-plane get pods -l app=workflow-recipes -o name)" || return 1
- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \
- -l app=workflow-recipes --timeout=180s >/dev/null
-
- $KC -n control-plane scale deployment/trace-maintenance-worker --replicas=0 >/dev/null
- [ "$($KC -n control-plane get deployment/trace-maintenance-worker \
- -o 'jsonpath={.spec.replicas}')" = 0 ] || {
- err "trace-maintenance-worker did not converge to zero before setup mutation"
- return 1
- }
- pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" || return 1
- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \
- -l app=trace-maintenance-worker --timeout=180s >/dev/null
- pods="$($KC -n control-plane get pods -l app=trace-maintenance-worker -o name)" || return 1
- [ -z "${pods}" ] || {
- err "trace-maintenance-worker pods remain after the pre-mutation fence"
- return 1
- }
-
- $KC -n control-plane scale deployment/control-api --replicas=0 >/dev/null
- pods="$($KC -n control-plane get pods -l 'app=control-api,!clerum.io/component' -o name)" || return 1
- [ -z "${pods}" ] || $KC -n control-plane wait --for=delete pod \
- -l 'app=control-api,!clerum.io/component' --timeout=180s >/dev/null
+ guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller' || return 1
+ guard_interrupted_writer_deployment workflow-recipes 'app=workflow-recipes' || return 1
+ guard_interrupted_writer_deployment trace-maintenance-worker 'app=trace-maintenance-worker' || return 1
+ guard_interrupted_writer_deployment control-api 'app=control-api,!clerum.io/component' || return 1
}
# ======================================================================
@@ -1360,6 +1347,7 @@ writer_recovery_state_load() {
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_TRACE_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
if [ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]; then
WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
diff --git a/scripts/tests/test-minikube-full-setup.sh b/scripts/tests/test-minikube-full-setup.sh
index dd417fa21..d28f44c7a 100755
--- a/scripts/tests/test-minikube-full-setup.sh
+++ b/scripts/tests/test-minikube-full-setup.sh
@@ -1400,6 +1400,88 @@ STUB
rm -rf "$d"
}
+assert_interrupted_recovery_resume_restores_all_writer_fences() {
+ local resume_block
+ resume_block="$(sed -n '/^ api-restored|overlay-applying|overlay-applied)/,/^ ;;$/p' scripts/minikube/full-setup.sh)"
+ if grep -Fq 'PARTIAL_HCC_FENCED=true' <<<"$resume_block" && \
+ grep -Fq 'PARTIAL_WORKFLOW_FENCED=true' <<<"$resume_block" && \
+ grep -Fq 'PARTIAL_TRACE_FENCED=true' <<<"$resume_block" && \
+ grep -Fq 'PARTIAL_CONTROL_API_FENCED=true' <<<"$resume_block"; then
+ pass "post-migration recovery resumes with all four writer fences active"
+ else
+ fail "post-migration recovery can omit a writer fence before restoring the journaled replicas"
+ fi
+}
+
+assert_interrupted_recovery_fence_is_behaviorally_fail_closed() {
+ local d helper run_script out rc problems=""
+ d="$(mktemp -d)"
+ helper="$(sed -n '/^guard_interrupted_writer_deployment()/,/^}/p' scripts/minikube/full-setup.sh)"
+
+ mkdir -p "$d/bin"
+ cat > "$d/bin/kubectl" <<'STUB'
+#!/usr/bin/env bash
+set -u
+case "$*" in
+ *"scale deployment/"*)
+ [ "${TEST_FENCE_MODE:-}" = scale-fail ] && exit 1
+ exit 0
+ ;;
+ *"get deployment/"*)
+ printf '%s' "${TEST_FENCE_DESIRED:-0}"
+ exit 0
+ ;;
+ *"get pods -l"*)
+ printf '%s' "${TEST_FENCE_PODS:-}"
+ exit 0
+ ;;
+ *"wait --for=delete pod"*)
+ [ "${TEST_FENCE_MODE:-}" = wait-fail ] && exit 1
+ exit 0
+ ;;
+esac
+exit 0
+STUB
+ chmod +x "$d/bin/kubectl"
+
+ run_script="$d/run.sh"
+ {
+ printf 'set -u\n'
+ printf 'KC=%q\n' 'kubectl --context=test'
+ printf 'log() { :; }\n'
+ printf 'err() { printf "ERR:%%s\\n" "$*" >&2; }\n'
+ printf '%s\n' "$helper"
+ printf "guard_interrupted_writer_deployment host-context-controller 'app=host-context-controller'\n"
+ } >"$run_script"
+
+ out="$(TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
+ rc=$?
+ [ "$rc" -eq 0 ] || problems+="healthy zero-replica fence failed; "
+
+ out="$(TEST_FENCE_DESIRED=1 TEST_FENCE_PODS= PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
+ rc=$?
+ [ "$rc" -ne 0 ] || problems+="nonzero desired replicas were accepted; "
+ grep -Fq 'did not converge to zero desired replicas' <<<"$out" \
+ || problems+="nonzero desired replicas did not fail closed; "
+
+ out="$(TEST_FENCE_MODE=scale-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS= \
+ PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
+ rc=$?
+ [ "$rc" -ne 0 ] || problems+="scale failure was swallowed; "
+
+ out="$(TEST_FENCE_MODE=wait-fail TEST_FENCE_DESIRED=0 TEST_FENCE_PODS='pod/hcc-0' \
+ PATH="$d/bin:$PATH" bash "$run_script" 2>&1)"
+ rc=$?
+ [ "$rc" -ne 0 ] || problems+="wait failure was swallowed; "
+
+ if [ -z "$problems" ]; then
+ pass "interrupted recovery fencing fails closed on scale, convergence, and pod-wait failures"
+ else
+ fail "$problems"
+ fi
+ rm -rf "$d"
+}
+
assert_recovery_migrations_are_head_bound() {
local current_head_checks
current_head_checks="$(grep -Fc '[ "$WRITER_RECOVERY_STATE_HEAD" = "$T2_HEAD" ]' scripts/minikube/full-setup.sh)"
@@ -1462,6 +1544,8 @@ assert_the_tag_override_also_generates_the_api_ip_patch_in_the_working_tree
assert_the_unoverridden_path_generates_the_api_ip_patch_once
assert_trace_writer_fence_closes_and_proves_zero_replicas
assert_trace_writer_fence_is_behaviorally_fail_closed
+assert_interrupted_recovery_resume_restores_all_writer_fences
+assert_interrupted_recovery_fence_is_behaviorally_fail_closed
assert_recovery_migrations_are_head_bound
assert_skip_build_follows_the_recorded_image_source
assert_skip_build_says_it_is_following_the_cluster