Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(minikube): preserve writer fences across recovery
Alfredo Lopez0bf2684ce446
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.shindex 9ccab8510..df47ce0a7 100755--- a/scripts/e2e/minikube-real-postgres.sh+++ b/scripts/e2e/minikube-real-postgres.sh@@ -54,6 +54,7 @@ T1_DURATION_SECONDS=0T1_GFS_RESTORE_REQUIRED=falseT1_CLEANUP_DONE=falseT1_SIGNAL_STATUS=0+T1_PORT_FORWARD_CLEANUP_OK=truedie_t1() {local code="$1"@@ -186,6 +187,7 @@ cleanup_t1() {trap '' INT TERMif ! stop_control_postgres_forward; thenstatus=1+ T1_PORT_FORWARD_CLEANUP_OK=falsefiif ! stop_isolated_postgres; thenstatus=1@@ -197,7 +199,12 @@ cleanup_t1() {status=1fiT1_DURATION_SECONDS=$((SECONDS - T1_STARTED_SECONDS))- if [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi+ if [ "$T1_PORT_FORWARD_CLEANUP_OK" = true ] &&+ [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then+ rm -rf "$T1_TMP_DIR"+ elif [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then+ printf '[minikube-t1] preserving T1 temp directory because port-forward ownership cleanup was not proven: %s\n' "$T1_TMP_DIR" >&2+ fiif [ "$status" -eq 0 ] && [ "$T1_STATUS" = PASS ]; thent2_evidence_write T1 PASS "Real PostgreSQL suites passed; files=$T1_REPORTED_FILES/$T1_EXPECTED_FILES tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS duration=${T1_DURATION_SECONDS}s; cleanup=PASS"t2_evidence_write complete PASS "T1=PASS files=$T1_REPORTED_FILES/$T1_EXPECTED_FILES tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS duration=${T1_DURATION_SECONDS}s cleanup=PASS"diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 7df5ee6da..34b56f2f0 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -1127,8 +1127,10 @@ WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"WRITER_RECOVERY_STATE_LOADED=falseWRITER_RECOVERY_STATE_PHASE=""WRITER_RECOVERY_FENCE_PENDING=false+WRITER_RECOVERY_MIGRATIONS_COMPLETE=falseK8S_API_EGRESS_POLICY_FILE=""K8S_API_EGRESS_POLICY_DRIFT=false+RECOVERY_OVERLAY_FILE=""writer_recovery_state_cli() {local command="$1"@@ -1218,18 +1220,39 @@ writer_recovery_state_load() {PARTIAL_CONTROL_API_FENCED=trueWRITER_RECOVERY_FENCE_PENDING=true;;- policy-ready|roles-ready)+ policy-ready)WRITER_RECOVERY=truePARTIAL_BOOTSTRAP_RECOVERY=truePARTIAL_HCC_FENCED=truePARTIAL_WORKFLOW_FENCED=truePARTIAL_CONTROL_API_FENCED=true+ # A persisted policy phase is historical evidence only. Re-fence and+ # restage the current rendered policy before any writer is restored so a+ # Minikube API endpoint change cannot resurrect the original egress drift.+ WRITER_RECOVERY_FENCE_PENDING=true+ ;;+ roles-ready|api-restoring)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ PARTIAL_CONTROL_API_FENCED=true+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ # The phase is written before the control-api restore. An interruption+ # after scale but before the next phase must close every writer again.+ WRITER_RECOVERY_FENCE_PENDING=true;;- api-restored|overlay-applied)+ api-restored|overlay-applying|overlay-applied)WRITER_RECOVERY=truePARTIAL_BOOTSTRAP_RECOVERY=truePARTIAL_HCC_FENCED=truePARTIAL_WORKFLOW_FENCED=true+ PARTIAL_CONTROL_API_FENCED=true+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true+ # The overlay is rendered with all three writer Deployments at zero.+ # Re-fence on every post-migration resume because an interrupted apply+ # may have recreated a writer before its durable phase was updated.+ WRITER_RECOVERY_FENCE_PENDING=true;;*)err "Durable writer-recovery state has an unsupported phase: $WRITER_RECOVERY_STATE_PHASE"@@ -1466,11 +1489,13 @@ restore_partial_control_api() {return 1filog "Restoring control-api to ${PARTIAL_CONTROL_API_REPLICAS} replica(s) after runtime-role provisioning"+ writer_recovery_state_phase api-restoring || return 1$KC -n control-plane scale deployment/control-api \- --replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null- $KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null+ --replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null || return 1+ $KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null || return 1+ control_api_is_ready || return 1PARTIAL_CONTROL_API_FENCED=false- writer_recovery_state_phase api-restored+ writer_recovery_state_phase api-restored || return 1}restore_partial_non_api_writers() {@@ -1508,6 +1533,36 @@ reconcile_existing_gfs_credentials() {bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs}+render_fenced_recovery_overlay() {+ local rendered_file="${K8S_API_POLICY_TMP_DIR}/recovery-rendered.yaml"+ [ -n "${K8S_API_POLICY_TMP_DIR}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ] || {+ err "Recovery overlay workspace is unavailable; refusing writer recovery"+ return 1+ }+ log "Rendering the recovery overlay with HCC, workflow-recipes, and control-api fenced"+ if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | \+ RUBYOPT=--disable=gems ruby "${SCRIPT_DIR}/render-fenced-writer-deployments.rb" \+ --target control-plane/host-context-controller \+ --target control-plane/workflow-recipes \+ --target control-plane/control-api >"$rendered_file"; then+ err "Unable to render a complete fenced recovery overlay; refusing writer recovery"+ return 1+ fi+ [ -s "$rendered_file" ] || {+ err "Fenced recovery overlay is empty; refusing writer recovery"+ return 1+ }+ RECOVERY_OVERLAY_FILE="$rendered_file"+}++apply_fenced_recovery_overlay() {+ render_fenced_recovery_overlay || return 1+ writer_recovery_state_phase overlay-applying || return 1+ log "Applying the complete recovery overlay while all control-plane writers remain fenced"+ $KC apply -f "$RECOVERY_OVERLAY_FILE" >/dev/null || return 1+ writer_recovery_state_phase overlay-applied || return 1+}+writer_recovery_state_load || exit 1prepare_refreshed_k8s_api_network_policy || exit 1CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"@@ -1607,23 +1662,28 @@ if [ "$RESET_DB" = true ]; thenfiok "Control-api database and GFS roles converged after reset"else- ensure_control_postgres_ready- log "Applying control-api database migrations and runtime roles..."- CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \- bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \- --overlay "$ACTIVE_MINIKUBE_RENDER_DIR"- CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \- bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"- ok "Control-api database migrations and runtime roles applied"- if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then- writer_recovery_state_phase roles-ready- fi-- # The credential probe runs through control-api, so prove that deployment is- # live before normal staging on bootstrap or an idempotent upgrade. On the- # deferred recovery path this is also the point where the missing runtime- # Secret has been provisioned, so the previously blocked deployment can start.+ if [ "$WRITER_RECOVERY_MIGRATIONS_COMPLETE" != true ]; then+ ensure_control_postgres_ready+ log "Applying control-api database migrations and runtime roles..."+ CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \+ bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \+ --overlay "$ACTIVE_MINIKUBE_RENDER_DIR"+ CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \+ bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"+ ok "Control-api database migrations and runtime roles applied"+ if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then+ writer_recovery_state_phase roles-ready+ fi+ else+ log "Skipping migrations and runtime-role provisioning; durable recovery state already completed that boundary"+ fi+if [ "$WRITER_RECOVERY" = true ]; then+ # Keep every writer at zero while the full overlay is applied. The overlay+ # itself declares one replica for these Deployments, so applying it+ # directly would reopen the writer window before the explicit restore.+ apply_fenced_recovery_overlay+ log "Recovery overlay is applied — restoring control-api before GFS reconciliation"if ! restore_partial_control_api; thenif [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; thenerr "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"@@ -1640,20 +1700,15 @@ elsefiexit 1fi+ log "Control-api/runtime roles are Ready — reconciling GFS after the fenced overlay"+ reconcile_existing_gfs_credentials+ restore_partial_non_api_writers+ ok "Fenced recovery overlay applied after runtime-role readiness; writers restored explicitly"else$KC scale deployment/control-api -n control-plane --replicas=1 >/dev/null$KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/nullfi- if [ "$WRITER_RECOVERY" = true ]; then- log "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"- reconcile_existing_gfs_credentials- $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -- writer_recovery_state_phase overlay-applied- restore_partial_non_api_writers- ok "Deferred full kustomize overlay applied after fenced writer/runtime-role readiness"- fi-# On the upgrade path the full overlay may still cut HCC over to the# split writer/reader templates; reconciling after that lands leaves the# staged reader credential rollout-pending and fails the final verify.diff --git a/scripts/minikube/render-fenced-writer-deployments.rb b/scripts/minikube/render-fenced-writer-deployments.rbnew file mode 100755index 000000000..cc1c07f9b--- /dev/null+++ b/scripts/minikube/render-fenced-writer-deployments.rb@@ -0,0 +1,42 @@+#!/usr/bin/env ruby+# Render a complete Kubernetes manifest while preserving an explicit writer+# fence. Recovery must never apply the ordinary one-replica Deployment objects+# before the orchestrator has restored them deliberately.++require "optparse"+require "yaml"++targets = []+OptionParser.new do |parser|+ parser.on("--target NAMESPACE/NAME", "Deployment to keep at zero replicas") do |value|+ namespace, name = value.split("/", 2)+ abort "FENCED_RENDER_ERROR: target must be NAMESPACE/NAME" if namespace.nil? || name.nil? || namespace.empty? || name.empty?+ targets << [namespace, name]+ end+end.parse!++abort "FENCED_RENDER_ERROR: at least one target Deployment is required" if targets.empty?+abort "FENCED_RENDER_ERROR: duplicate target Deployment" unless targets.uniq.length == targets.length++documents = YAML.load_stream($stdin.read)+seen = Hash.new(0)++documents.each do |document|+ next unless document.is_a?(Hash) && document["kind"] == "Deployment"++ metadata = document["metadata"]+ namespace = metadata.is_a?(Hash) ? metadata["namespace"] : nil+ name = metadata.is_a?(Hash) ? metadata["name"] : nil+ key = [namespace, name]+ next unless targets.include?(key)++ spec = document["spec"]+ abort "FENCED_RENDER_ERROR: target #{namespace}/#{name} has no Deployment spec" unless spec.is_a?(Hash)+ spec["replicas"] = 0+ seen[key] += 1+end++missing = targets.reject { |target| seen[target] == 1 }+abort "FENCED_RENDER_ERROR: target Deployment missing or duplicated: #{missing.map { |namespace, name| "#{namespace}/#{name}" }.join(", ")}" unless missing.empty?++print YAML.dump_stream(*documents)diff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.pyindex f059053a6..200111f9c 100755--- a/scripts/minikube/writer-recovery-state.py+++ b/scripts/minikube/writer-recovery-state.py@@ -26,7 +26,9 @@ PHASES = {"api-fenced","policy-ready","roles-ready",+ "api-restoring","api-restored",+ "overlay-applying","overlay-applied",}diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex fee63107c..eb77d61fe 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -177,7 +177,7 @@ assert_minikube_upgrade_classifier() {fidoneif [[ "$path" == *"scripts/minikube/full-setup.sh" ]]; then- local readiness_probe writer_flag partial_flag fence_call policy_apply migration roles restore_api deferred_full_overlay post_ready post_reconcile+ local readiness_probe writer_flag partial_flag fence_call policy_apply migration roles recovery_overlay restore_api post_ready post_reconcile restore_writersreadiness_probe="$(grep -n 'if control_api_is_ready' <<<"$block" | head -1 | cut -d: -f1)"if [[ -n "$readiness_probe" ]]; thenreadiness_probe=$((block_start + readiness_probe - 1))@@ -192,9 +192,10 @@ assert_minikube_upgrade_classifier() {migration="$(grep -n 'run-control-api-db-migration.sh' "$path" | head -1 | cut -d: -f1)"roles="$(grep -n 'provision-control-api-runtime-roles.sh' "$path" | head -1 | cut -d: -f1)"restore_api="$(grep -nE '^[[:space:]]+restore_partial_control_api$' "$path" | tail -1 | cut -d: -f1)"- deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"+ recovery_overlay="$(grep -nE '^[[:space:]]+apply_fenced_recovery_overlay$' "$path" | tail -1 | cut -d: -f1)"post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"+ restore_writers="$(grep -nE '^[[:space:]]+restore_partial_non_api_writers$' "$path" | tail -1 | cut -d: -f1)"for line_name in writer_flag partial_flag fence_call policy_apply; doline_value="${!line_name}"if [[ -n "$line_value" ]]; then@@ -204,21 +205,31 @@ assert_minikube_upgrade_classifier() {[[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$writer_flag" && \-n "$partial_flag" && -n "$fence_call" && -n "$policy_apply" && \-n "$fallback" && -n "$fresh" && -n "$migration" && \- -n "$roles" && -n "$restore_api" && -n "$deferred_full_overlay" && \- -n "$post_ready" && -n "$post_reconcile" ]] \+ -n "$roles" && -n "$recovery_overlay" && -n "$restore_api" && \+ -n "$post_ready" && -n "$post_reconcile" && -n "$restore_writers" ]] \|| fail "$path recovery classifier is incomplete"[[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \"$readiness_probe" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \"$partial_flag" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \"$partial_flag" -lt "$fallback" && "$fallback" -lt "$fresh" && \- "$migration" -lt "$roles" && "$roles" -lt "$restore_api" && \- "$restore_api" -lt "$post_reconcile" && "$post_reconcile" -lt "$deferred_full_overlay" && \- "$post_ready" -lt "$deferred_full_overlay" ]] \- || fail "$path does not fence writers and defer the full overlay until runtime-role readiness"+ "$migration" -lt "$roles" && "$roles" -lt "$recovery_overlay" && \+ "$recovery_overlay" -lt "$restore_api" && "$restore_api" -lt "$post_reconcile" && \+ "$post_reconcile" -lt "$restore_writers" && "$post_ready" -lt "$post_reconcile" ]] \+ || fail "$path does not apply a fenced overlay before restoring API, GFS, and the remaining writers"grep -Fq 'Existing GFS writer detected but control-api is not Ready; refusing HCC cutover' "$path" \|| fail "$path no longer retains the fail-closed guard for an unrecognized unready upgrade"grep -Fq 'app=control-api,!clerum.io/component' "$path" \|| fail "$path does not exclude migration Jobs from the control-api writer fence"+ grep -Fq 'WRITER_RECOVERY_MIGRATIONS_COMPLETE=true' "$path" \+ || fail "$path does not mark post-migration recovery as complete"+ grep -Fq 'WRITER_RECOVERY_FENCE_PENDING=true' "$path" \+ || fail "$path does not re-fence every durable post-migration resume"+ grep -Fq 'render-fenced-writer-deployments.rb' "$path" \+ || fail "$path does not render a fenced recovery overlay"+ grep -Fq 'writer_recovery_state_phase overlay-applying' "$path" \+ || fail "$path does not persist the overlay-application boundary"+ grep -Fq 'writer_recovery_state_phase api-restoring' "$path" \+ || fail "$path does not persist the control-api restore boundary"else[[ -n "$dsn" && -n "$classify" && -n "$ready" && -n "$abort" && -n "$reconcile" && -n "$fallback" && -n "$fresh" ]] \|| fail "$path upgrade classifier is incomplete"diff --git a/scripts/tests/test-minikube-fenced-recovery-render.sh b/scripts/tests/test-minikube-fenced-recovery-render.shnew file mode 100755index 000000000..4b81515d1--- /dev/null+++ b/scripts/tests/test-minikube-fenced-recovery-render.sh@@ -0,0 +1,66 @@+#!/usr/bin/env bash+set -euo pipefail++ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+RENDERER="$ROOT/scripts/minikube/render-fenced-writer-deployments.rb"+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-fenced-render.XXXXXX")"+trap 'rm -rf -- "$TMP_DIR"' EXIT++cat >"$TMP_DIR/input.yaml" <<'EOF_YAML'+apiVersion: apps/v1+kind: Deployment+metadata:+ name: host-context-controller+ namespace: control-plane+spec:+ replicas: 1+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: workflow-recipes+ namespace: control-plane+spec:+ replicas: 2+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: control-api+ namespace: control-plane+spec:+ replicas: 1+---+apiVersion: v1+kind: Service+metadata:+ name: control-api+ namespace: control-plane+EOF_YAML++output="$TMP_DIR/output.yaml"+ruby "$RENDERER" \+ --target control-plane/host-context-controller \+ --target control-plane/workflow-recipes \+ --target control-plane/control-api <"$TMP_DIR/input.yaml" >"$output"++ruby -ryaml -e '+docs = YAML.load_stream(File.read(ARGV.fetch(0)))+targets = %w[host-context-controller workflow-recipes control-api]+targets.each do |name|+ deployment = docs.find { |doc| doc.is_a?(Hash) && doc["kind"] == "Deployment" && doc.dig("metadata", "namespace") == "control-plane" && doc.dig("metadata", "name") == name }+ abort "missing #{name}" unless deployment+ abort "#{name} was not fenced" unless deployment.dig("spec", "replicas") == 0+end+service = docs.find { |doc| doc.is_a?(Hash) && doc["kind"] == "Service" }+abort "non-target resource was changed" unless service.dig("metadata", "name") == "control-api"+' "$output"++if ruby "$RENDERER" --target control-plane/missing <"$TMP_DIR/input.yaml" >"$TMP_DIR/missing.yaml" 2>"$TMP_DIR/missing.err"; then+ echo 'FAIL: renderer accepted a missing writer target' >&2+ exit 1+fi+grep -Fq 'FENCED_RENDER_ERROR' "$TMP_DIR/missing.err"++printf 'PASS: recovery overlay keeps every writer Deployment at zero replicas\n'+printf 'PASS: recovery overlay fails closed when a writer target is missing\n'diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex 537ba7ac4..887ef9d9e 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -39,6 +39,7 @@ for file in "$MINIKUBE_DIR/profile-readiness.sh" "$ROOT/scripts/tests/test-minik"$ROOT/scripts/tests/test-real-postgres-local-preflight.sh" \"$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh" \"$ROOT/scripts/tests/test-minikube-t1-port-forward-owner.sh" \+ "$ROOT/scripts/tests/test-minikube-fenced-recovery-render.sh" \"$ROOT/scripts/tests/test-minikube-t2-process-owner.sh" \"$ROOT/scripts/tests/test-minikube-explicit-context.sh" \"$ROOT/scripts/tests/test-minikube-t2-evidence.sh" \@@ -54,6 +55,7 @@ done"$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh""$ROOT/scripts/tests/test-minikube-k8s-api-egress-policy.sh""$ROOT/scripts/tests/test-minikube-writer-recovery-state.sh"+"$ROOT/scripts/tests/test-minikube-fenced-recovery-render.sh"python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'import osimport py_compile@@ -156,6 +158,8 @@ figrep -Fq 'kubectl --context="$T2_CONTEXT" -n "$PG_NAMESPACE" port-forward' "$T1"grep -Fq 'pf_owner_record_process "$PORT_FORWARD_RECORD" "$PORT_FORWARD_PID"' "$T1"grep -Fq 'pf_owner_cleanup_record "$PORT_FORWARD_RECORD"' "$T1"+grep -Fq 'T1_PORT_FORWARD_CLEANUP_OK=false' "$T1"+grep -Fq 'preserving T1 temp directory because port-forward ownership cleanup was not proven' "$T1"grep -Fq 'set +x' "$T1" "$PREFLIGHT" "$T2"# T1 must restore the branch-profile GFS credentials on the way out with thediff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.shindex fcb3949b4..b2ef1c726 100755--- a/scripts/tests/test-minikube-writer-recovery-state.sh+++ b/scripts/tests/test-minikube-writer-recovery-state.sh@@ -37,6 +37,11 @@ pass 'writer recovery state is private and round-trips atomically'python3 "$HELPER" write "${identity[@]}" --phase api-fencing --hcc 1 --workflow 2 --control-api 1python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'api-fencing|1|2|1' ||fail 'fencing phase did not preserve original replicas'+for phase in api-restoring overlay-applying; do+ python3 "$HELPER" write "${identity[@]}" --phase "$phase" --hcc 1 --workflow 2 --control-api 1+ python3 "$HELPER" read "${identity[@]}" | grep -Fxq "${phase}|1|2|1" ||+ fail "${phase} phase did not round-trip"+donewrong_status=0python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 >/dev/null 2>&1 || wrong_status=$?[[ "$wrong_status" -ne 0 ]] || fail 'state was readable under a different HEAD'