Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): preserve writer fences across recovery

Alfredo Lopez0bf2684ce446
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.sh
index 9ccab8510..df47ce0a7 100755
--- a/scripts/e2e/minikube-real-postgres.sh
+++ b/scripts/e2e/minikube-real-postgres.sh
@@ -54,6 +54,7 @@ T1_DURATION_SECONDS=0
T1_GFS_RESTORE_REQUIRED=false
T1_CLEANUP_DONE=false
T1_SIGNAL_STATUS=0
+T1_PORT_FORWARD_CLEANUP_OK=true
die_t1() {
local code="$1"
@@ -186,6 +187,7 @@ cleanup_t1() {
trap '' INT TERM
if ! stop_control_postgres_forward; then
status=1
+ T1_PORT_FORWARD_CLEANUP_OK=false
fi
if ! stop_isolated_postgres; then
status=1
@@ -197,7 +199,12 @@ cleanup_t1() {
status=1
fi
T1_DURATION_SECONDS=$((SECONDS - T1_STARTED_SECONDS))
- if [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi
+ if [ "$T1_PORT_FORWARD_CLEANUP_OK" = true ] &&
+ [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then
+ rm -rf "$T1_TMP_DIR"
+ elif [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then
+ printf '[minikube-t1] preserving T1 temp directory because port-forward ownership cleanup was not proven: %s\n' "$T1_TMP_DIR" >&2
+ fi
if [ "$status" -eq 0 ] && [ "$T1_STATUS" = PASS ]; then
t2_evidence_write T1 PASS "Real PostgreSQL suites passed; files=$T1_REPORTED_FILES/$T1_EXPECTED_FILES tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS duration=${T1_DURATION_SECONDS}s; cleanup=PASS"
t2_evidence_write complete PASS "T1=PASS files=$T1_REPORTED_FILES/$T1_EXPECTED_FILES tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS duration=${T1_DURATION_SECONDS}s cleanup=PASS"
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 7df5ee6da..34b56f2f0 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -1127,8 +1127,10 @@ WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"
WRITER_RECOVERY_STATE_LOADED=false
WRITER_RECOVERY_STATE_PHASE=""
WRITER_RECOVERY_FENCE_PENDING=false
+WRITER_RECOVERY_MIGRATIONS_COMPLETE=false
K8S_API_EGRESS_POLICY_FILE=""
K8S_API_EGRESS_POLICY_DRIFT=false
+RECOVERY_OVERLAY_FILE=""
writer_recovery_state_cli() {
local command="$1"
@@ -1218,18 +1220,39 @@ writer_recovery_state_load() {
PARTIAL_CONTROL_API_FENCED=true
WRITER_RECOVERY_FENCE_PENDING=true
;;
- policy-ready|roles-ready)
+ policy-ready)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
PARTIAL_CONTROL_API_FENCED=true
+ # A persisted policy phase is historical evidence only. Re-fence and
+ # restage the current rendered policy before any writer is restored so a
+ # Minikube API endpoint change cannot resurrect the original egress drift.
+ WRITER_RECOVERY_FENCE_PENDING=true
+ ;;
+ roles-ready|api-restoring)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_CONTROL_API_FENCED=true
+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ # The phase is written before the control-api restore. An interruption
+ # after scale but before the next phase must close every writer again.
+ WRITER_RECOVERY_FENCE_PENDING=true
;;
- api-restored|overlay-applied)
+ api-restored|overlay-applying|overlay-applied)
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
PARTIAL_HCC_FENCED=true
PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_CONTROL_API_FENCED=true
+ WRITER_RECOVERY_MIGRATIONS_COMPLETE=true
+ # The overlay is rendered with all three writer Deployments at zero.
+ # Re-fence on every post-migration resume because an interrupted apply
+ # may have recreated a writer before its durable phase was updated.
+ WRITER_RECOVERY_FENCE_PENDING=true
;;
*)
err "Durable writer-recovery state has an unsupported phase: $WRITER_RECOVERY_STATE_PHASE"
@@ -1466,11 +1489,13 @@ restore_partial_control_api() {
return 1
fi
log "Restoring control-api to ${PARTIAL_CONTROL_API_REPLICAS} replica(s) after runtime-role provisioning"
+ writer_recovery_state_phase api-restoring || return 1
$KC -n control-plane scale deployment/control-api \
- --replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null
- $KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null
+ --replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null || return 1
+ $KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null || return 1
+ control_api_is_ready || return 1
PARTIAL_CONTROL_API_FENCED=false
- writer_recovery_state_phase api-restored
+ writer_recovery_state_phase api-restored || return 1
}
restore_partial_non_api_writers() {
@@ -1508,6 +1533,36 @@ reconcile_existing_gfs_credentials() {
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
}
+render_fenced_recovery_overlay() {
+ local rendered_file="${K8S_API_POLICY_TMP_DIR}/recovery-rendered.yaml"
+ [ -n "${K8S_API_POLICY_TMP_DIR}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ] || {
+ err "Recovery overlay workspace is unavailable; refusing writer recovery"
+ return 1
+ }
+ log "Rendering the recovery overlay with HCC, workflow-recipes, and control-api fenced"
+ if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | \
+ RUBYOPT=--disable=gems ruby "${SCRIPT_DIR}/render-fenced-writer-deployments.rb" \
+ --target control-plane/host-context-controller \
+ --target control-plane/workflow-recipes \
+ --target control-plane/control-api >"$rendered_file"; then
+ err "Unable to render a complete fenced recovery overlay; refusing writer recovery"
+ return 1
+ fi
+ [ -s "$rendered_file" ] || {
+ err "Fenced recovery overlay is empty; refusing writer recovery"
+ return 1
+ }
+ RECOVERY_OVERLAY_FILE="$rendered_file"
+}
+
+apply_fenced_recovery_overlay() {
+ render_fenced_recovery_overlay || return 1
+ writer_recovery_state_phase overlay-applying || return 1
+ log "Applying the complete recovery overlay while all control-plane writers remain fenced"
+ $KC apply -f "$RECOVERY_OVERLAY_FILE" >/dev/null || return 1
+ writer_recovery_state_phase overlay-applied || return 1
+}
+
writer_recovery_state_load || exit 1
prepare_refreshed_k8s_api_network_policy || exit 1
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
@@ -1607,23 +1662,28 @@ if [ "$RESET_DB" = true ]; then
fi
ok "Control-api database and GFS roles converged after reset"
else
- ensure_control_postgres_ready
- log "Applying control-api database migrations and runtime roles..."
- CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
- bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \
- --overlay "$ACTIVE_MINIKUBE_RENDER_DIR"
- CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
- bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"
- ok "Control-api database migrations and runtime roles applied"
- if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then
- writer_recovery_state_phase roles-ready
- fi
-
- # The credential probe runs through control-api, so prove that deployment is
- # live before normal staging on bootstrap or an idempotent upgrade. On the
- # deferred recovery path this is also the point where the missing runtime
- # Secret has been provisioned, so the previously blocked deployment can start.
+ if [ "$WRITER_RECOVERY_MIGRATIONS_COMPLETE" != true ]; then
+ ensure_control_postgres_ready
+ log "Applying control-api database migrations and runtime roles..."
+ CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
+ bash "${PROJECT_DIR}/deploy/scripts/run-control-api-db-migration.sh" \
+ --overlay "$ACTIVE_MINIKUBE_RENDER_DIR"
+ CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
+ bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"
+ ok "Control-api database migrations and runtime roles applied"
+ if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then
+ writer_recovery_state_phase roles-ready
+ fi
+ else
+ log "Skipping migrations and runtime-role provisioning; durable recovery state already completed that boundary"
+ fi
+
if [ "$WRITER_RECOVERY" = true ]; then
+ # Keep every writer at zero while the full overlay is applied. The overlay
+ # itself declares one replica for these Deployments, so applying it
+ # directly would reopen the writer window before the explicit restore.
+ apply_fenced_recovery_overlay
+ log "Recovery overlay is applied — restoring control-api before GFS reconciliation"
if ! restore_partial_control_api; then
if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
@@ -1640,20 +1700,15 @@ else
fi
exit 1
fi
+ log "Control-api/runtime roles are Ready — reconciling GFS after the fenced overlay"
+ reconcile_existing_gfs_credentials
+ restore_partial_non_api_writers
+ ok "Fenced recovery overlay applied after runtime-role readiness; writers restored explicitly"
else
$KC scale deployment/control-api -n control-plane --replicas=1 >/dev/null
$KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null
fi
- if [ "$WRITER_RECOVERY" = true ]; then
- log "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"
- reconcile_existing_gfs_credentials
- $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
- writer_recovery_state_phase overlay-applied
- restore_partial_non_api_writers
- ok "Deferred full kustomize overlay applied after fenced writer/runtime-role readiness"
- fi
-
# On the upgrade path the full overlay may still cut HCC over to the
# split writer/reader templates; reconciling after that lands leaves the
# staged reader credential rollout-pending and fails the final verify.
diff --git a/scripts/minikube/render-fenced-writer-deployments.rb b/scripts/minikube/render-fenced-writer-deployments.rb
new file mode 100755
index 000000000..cc1c07f9b
--- /dev/null
+++ b/scripts/minikube/render-fenced-writer-deployments.rb
@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+# Render a complete Kubernetes manifest while preserving an explicit writer
+# fence. Recovery must never apply the ordinary one-replica Deployment objects
+# before the orchestrator has restored them deliberately.
+
+require "optparse"
+require "yaml"
+
+targets = []
+OptionParser.new do |parser|
+ parser.on("--target NAMESPACE/NAME", "Deployment to keep at zero replicas") do |value|
+ namespace, name = value.split("/", 2)
+ abort "FENCED_RENDER_ERROR: target must be NAMESPACE/NAME" if namespace.nil? || name.nil? || namespace.empty? || name.empty?
+ targets << [namespace, name]
+ end
+end.parse!
+
+abort "FENCED_RENDER_ERROR: at least one target Deployment is required" if targets.empty?
+abort "FENCED_RENDER_ERROR: duplicate target Deployment" unless targets.uniq.length == targets.length
+
+documents = YAML.load_stream($stdin.read)
+seen = Hash.new(0)
+
+documents.each do |document|
+ next unless document.is_a?(Hash) && document["kind"] == "Deployment"
+
+ metadata = document["metadata"]
+ namespace = metadata.is_a?(Hash) ? metadata["namespace"] : nil
+ name = metadata.is_a?(Hash) ? metadata["name"] : nil
+ key = [namespace, name]
+ next unless targets.include?(key)
+
+ spec = document["spec"]
+ abort "FENCED_RENDER_ERROR: target #{namespace}/#{name} has no Deployment spec" unless spec.is_a?(Hash)
+ spec["replicas"] = 0
+ seen[key] += 1
+end
+
+missing = targets.reject { |target| seen[target] == 1 }
+abort "FENCED_RENDER_ERROR: target Deployment missing or duplicated: #{missing.map { |namespace, name| "#{namespace}/#{name}" }.join(", ")}" unless missing.empty?
+
+print YAML.dump_stream(*documents)
diff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.py
index f059053a6..200111f9c 100755
--- a/scripts/minikube/writer-recovery-state.py
+++ b/scripts/minikube/writer-recovery-state.py
@@ -26,7 +26,9 @@ PHASES = {
"api-fenced",
"policy-ready",
"roles-ready",
+ "api-restoring",
"api-restored",
+ "overlay-applying",
"overlay-applied",
}
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index fee63107c..eb77d61fe 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -177,7 +177,7 @@ assert_minikube_upgrade_classifier() {
fi
done
if [[ "$path" == *"scripts/minikube/full-setup.sh" ]]; then
- local readiness_probe writer_flag partial_flag fence_call policy_apply migration roles restore_api deferred_full_overlay post_ready post_reconcile
+ local readiness_probe writer_flag partial_flag fence_call policy_apply migration roles recovery_overlay restore_api post_ready post_reconcile restore_writers
readiness_probe="$(grep -n 'if control_api_is_ready' <<<"$block" | head -1 | cut -d: -f1)"
if [[ -n "$readiness_probe" ]]; then
readiness_probe=$((block_start + readiness_probe - 1))
@@ -192,9 +192,10 @@ assert_minikube_upgrade_classifier() {
migration="$(grep -n 'run-control-api-db-migration.sh' "$path" | head -1 | cut -d: -f1)"
roles="$(grep -n 'provision-control-api-runtime-roles.sh' "$path" | head -1 | cut -d: -f1)"
restore_api="$(grep -nE '^[[:space:]]+restore_partial_control_api$' "$path" | tail -1 | cut -d: -f1)"
- deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"
+ recovery_overlay="$(grep -nE '^[[:space:]]+apply_fenced_recovery_overlay$' "$path" | tail -1 | cut -d: -f1)"
post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"
post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"
+ restore_writers="$(grep -nE '^[[:space:]]+restore_partial_non_api_writers$' "$path" | tail -1 | cut -d: -f1)"
for line_name in writer_flag partial_flag fence_call policy_apply; do
line_value="${!line_name}"
if [[ -n "$line_value" ]]; then
@@ -204,21 +205,31 @@ assert_minikube_upgrade_classifier() {
[[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$writer_flag" && \
-n "$partial_flag" && -n "$fence_call" && -n "$policy_apply" && \
-n "$fallback" && -n "$fresh" && -n "$migration" && \
- -n "$roles" && -n "$restore_api" && -n "$deferred_full_overlay" && \
- -n "$post_ready" && -n "$post_reconcile" ]] \
+ -n "$roles" && -n "$recovery_overlay" && -n "$restore_api" && \
+ -n "$post_ready" && -n "$post_reconcile" && -n "$restore_writers" ]] \
|| fail "$path recovery classifier is incomplete"
[[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \
"$readiness_probe" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \
"$partial_flag" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \
"$partial_flag" -lt "$fallback" && "$fallback" -lt "$fresh" && \
- "$migration" -lt "$roles" && "$roles" -lt "$restore_api" && \
- "$restore_api" -lt "$post_reconcile" && "$post_reconcile" -lt "$deferred_full_overlay" && \
- "$post_ready" -lt "$deferred_full_overlay" ]] \
- || fail "$path does not fence writers and defer the full overlay until runtime-role readiness"
+ "$migration" -lt "$roles" && "$roles" -lt "$recovery_overlay" && \
+ "$recovery_overlay" -lt "$restore_api" && "$restore_api" -lt "$post_reconcile" && \
+ "$post_reconcile" -lt "$restore_writers" && "$post_ready" -lt "$post_reconcile" ]] \
+ || fail "$path does not apply a fenced overlay before restoring API, GFS, and the remaining writers"
grep -Fq 'Existing GFS writer detected but control-api is not Ready; refusing HCC cutover' "$path" \
|| fail "$path no longer retains the fail-closed guard for an unrecognized unready upgrade"
grep -Fq 'app=control-api,!clerum.io/component' "$path" \
|| fail "$path does not exclude migration Jobs from the control-api writer fence"
+ grep -Fq 'WRITER_RECOVERY_MIGRATIONS_COMPLETE=true' "$path" \
+ || fail "$path does not mark post-migration recovery as complete"
+ grep -Fq 'WRITER_RECOVERY_FENCE_PENDING=true' "$path" \
+ || fail "$path does not re-fence every durable post-migration resume"
+ grep -Fq 'render-fenced-writer-deployments.rb' "$path" \
+ || fail "$path does not render a fenced recovery overlay"
+ grep -Fq 'writer_recovery_state_phase overlay-applying' "$path" \
+ || fail "$path does not persist the overlay-application boundary"
+ grep -Fq 'writer_recovery_state_phase api-restoring' "$path" \
+ || fail "$path does not persist the control-api restore boundary"
else
[[ -n "$dsn" && -n "$classify" && -n "$ready" && -n "$abort" && -n "$reconcile" && -n "$fallback" && -n "$fresh" ]] \
|| fail "$path upgrade classifier is incomplete"
diff --git a/scripts/tests/test-minikube-fenced-recovery-render.sh b/scripts/tests/test-minikube-fenced-recovery-render.sh
new file mode 100755
index 000000000..4b81515d1
--- /dev/null
+++ b/scripts/tests/test-minikube-fenced-recovery-render.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+RENDERER="$ROOT/scripts/minikube/render-fenced-writer-deployments.rb"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-fenced-render.XXXXXX")"
+trap 'rm -rf -- "$TMP_DIR"' EXIT
+
+cat >"$TMP_DIR/input.yaml" <<'EOF_YAML'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: host-context-controller
+ namespace: control-plane
+spec:
+ replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: workflow-recipes
+ namespace: control-plane
+spec:
+ replicas: 2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: control-api
+ namespace: control-plane
+spec:
+ replicas: 1
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: control-api
+ namespace: control-plane
+EOF_YAML
+
+output="$TMP_DIR/output.yaml"
+ruby "$RENDERER" \
+ --target control-plane/host-context-controller \
+ --target control-plane/workflow-recipes \
+ --target control-plane/control-api <"$TMP_DIR/input.yaml" >"$output"
+
+ruby -ryaml -e '
+docs = YAML.load_stream(File.read(ARGV.fetch(0)))
+targets = %w[host-context-controller workflow-recipes control-api]
+targets.each do |name|
+ deployment = docs.find { |doc| doc.is_a?(Hash) && doc["kind"] == "Deployment" && doc.dig("metadata", "namespace") == "control-plane" && doc.dig("metadata", "name") == name }
+ abort "missing #{name}" unless deployment
+ abort "#{name} was not fenced" unless deployment.dig("spec", "replicas") == 0
+end
+service = docs.find { |doc| doc.is_a?(Hash) && doc["kind"] == "Service" }
+abort "non-target resource was changed" unless service.dig("metadata", "name") == "control-api"
+' "$output"
+
+if ruby "$RENDERER" --target control-plane/missing <"$TMP_DIR/input.yaml" >"$TMP_DIR/missing.yaml" 2>"$TMP_DIR/missing.err"; then
+ echo 'FAIL: renderer accepted a missing writer target' >&2
+ exit 1
+fi
+grep -Fq 'FENCED_RENDER_ERROR' "$TMP_DIR/missing.err"
+
+printf 'PASS: recovery overlay keeps every writer Deployment at zero replicas\n'
+printf 'PASS: recovery overlay fails closed when a writer target is missing\n'
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 537ba7ac4..887ef9d9e 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -39,6 +39,7 @@ for file in "$MINIKUBE_DIR/profile-readiness.sh" "$ROOT/scripts/tests/test-minik
"$ROOT/scripts/tests/test-real-postgres-local-preflight.sh" \
"$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh" \
"$ROOT/scripts/tests/test-minikube-t1-port-forward-owner.sh" \
+ "$ROOT/scripts/tests/test-minikube-fenced-recovery-render.sh" \
"$ROOT/scripts/tests/test-minikube-t2-process-owner.sh" \
"$ROOT/scripts/tests/test-minikube-explicit-context.sh" \
"$ROOT/scripts/tests/test-minikube-t2-evidence.sh" \
@@ -54,6 +55,7 @@ done
"$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh"
"$ROOT/scripts/tests/test-minikube-k8s-api-egress-policy.sh"
"$ROOT/scripts/tests/test-minikube-writer-recovery-state.sh"
+"$ROOT/scripts/tests/test-minikube-fenced-recovery-render.sh"
python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'
import os
import py_compile
@@ -156,6 +158,8 @@ fi
grep -Fq 'kubectl --context="$T2_CONTEXT" -n "$PG_NAMESPACE" port-forward' "$T1"
grep -Fq 'pf_owner_record_process "$PORT_FORWARD_RECORD" "$PORT_FORWARD_PID"' "$T1"
grep -Fq 'pf_owner_cleanup_record "$PORT_FORWARD_RECORD"' "$T1"
+grep -Fq 'T1_PORT_FORWARD_CLEANUP_OK=false' "$T1"
+grep -Fq 'preserving T1 temp directory because port-forward ownership cleanup was not proven' "$T1"
grep -Fq 'set +x' "$T1" "$PREFLIGHT" "$T2"
# T1 must restore the branch-profile GFS credentials on the way out with the
diff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.sh
index fcb3949b4..b2ef1c726 100755
--- a/scripts/tests/test-minikube-writer-recovery-state.sh
+++ b/scripts/tests/test-minikube-writer-recovery-state.sh
@@ -37,6 +37,11 @@ pass 'writer recovery state is private and round-trips atomically'
python3 "$HELPER" write "${identity[@]}" --phase api-fencing --hcc 1 --workflow 2 --control-api 1
python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'api-fencing|1|2|1' ||
fail 'fencing phase did not preserve original replicas'
+for phase in api-restoring overlay-applying; do
+ python3 "$HELPER" write "${identity[@]}" --phase "$phase" --hcc 1 --workflow 2 --control-api 1
+ python3 "$HELPER" read "${identity[@]}" | grep -Fxq "${phase}|1|2|1" ||
+ fail "${phase} phase did not round-trip"
+done
wrong_status=0
python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 >/dev/null 2>&1 || wrong_status=$?
[[ "$wrong_status" -ne 0 ]] || fail 'state was readable under a different HEAD'