Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): make writer recovery resumable and bounded

Alfredo Lopez848ffa0f26ae
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 9ed81a1d2..7df5ee6da 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -749,6 +749,7 @@ ok "Cluster '${PROFILE}' is reachable"
maybe_exit_after_cluster_step
SETUP_LOCK_CLEANUP_DONE=false
+K8S_API_POLICY_TMP_DIR=""
cleanup_setup_lock() {
local status="${1:-$?}" cleanup_status=0
if [ "$SETUP_LOCK_CLEANUP_DONE" = true ]; then
@@ -757,6 +758,9 @@ cleanup_setup_lock() {
SETUP_LOCK_CLEANUP_DONE=true
trap - EXIT
trap '' INT TERM
+ if [ -n "${K8S_API_POLICY_TMP_DIR:-}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ]; then
+ rm -rf -- "${K8S_API_POLICY_TMP_DIR}" || cleanup_status=1
+ fi
t2_lock_release "$status" || cleanup_status=$?
if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
status="$cleanup_status"
@@ -1111,12 +1115,156 @@ ok "Minikube K8s API CIDRs refreshed"
# runtime-role readiness has been proved.
WRITER_RECOVERY=false
PARTIAL_BOOTSTRAP_RECOVERY=false
-PARTIAL_CONTROL_API_REPLICAS=1
-PARTIAL_WORKFLOW_REPLICAS=1
-PARTIAL_HCC_REPLICAS=1
+PARTIAL_CONTROL_API_REPLICAS=""
+PARTIAL_WORKFLOW_REPLICAS=""
+PARTIAL_HCC_REPLICAS=""
PARTIAL_CONTROL_API_FENCED=false
PARTIAL_WORKFLOW_FENCED=false
PARTIAL_HCC_FENCED=false
+WRITER_RECOVERY_STATE_HELPER="${SCRIPT_DIR}/writer-recovery-state.py"
+WRITER_RECOVERY_STATE_ROOT="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery"
+WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"
+WRITER_RECOVERY_STATE_LOADED=false
+WRITER_RECOVERY_STATE_PHASE=""
+WRITER_RECOVERY_FENCE_PENDING=false
+K8S_API_EGRESS_POLICY_FILE=""
+K8S_API_EGRESS_POLICY_DRIFT=false
+
+writer_recovery_state_cli() {
+ local command="$1"
+ shift
+ python3 "$WRITER_RECOVERY_STATE_HELPER" "$command" \
+ --path "$WRITER_RECOVERY_STATE_FILE" \
+ --profile "$PROFILE" \
+ --context "$T2_CONTEXT" \
+ --worktree "$PROJECT_DIR" \
+ --branch "$T2_BRANCH" \
+ --head "$T2_HEAD" "$@"
+}
+
+writer_recovery_state_write() {
+ local phase="$1"
+ writer_recovery_state_cli write \
+ --phase "$phase" \
+ --hcc "$PARTIAL_HCC_REPLICAS" \
+ --workflow "$PARTIAL_WORKFLOW_REPLICAS" \
+ --control-api "$PARTIAL_CONTROL_API_REPLICAS"
+}
+
+writer_recovery_state_phase() {
+ writer_recovery_state_write "$1"
+ WRITER_RECOVERY_STATE_LOADED=true
+ WRITER_RECOVERY_STATE_PHASE="$1"
+}
+
+writer_recovery_state_clear() {
+ writer_recovery_state_cli clear
+ WRITER_RECOVERY_STATE_LOADED=false
+ WRITER_RECOVERY_STATE_PHASE=""
+ WRITER_RECOVERY_FENCE_PENDING=false
+}
+
+writer_recovery_state_load() {
+ local state_output state_status=0
+ state_output="$(writer_recovery_state_cli read)" || state_status=$?
+ if [ "$state_status" -ne 0 ]; then
+ err "Unable to validate the durable writer-recovery state; refusing profile mutation"
+ return 1
+ fi
+ if [ "$state_output" = NONE ]; then
+ return 0
+ fi
+ IFS='|' read -r WRITER_RECOVERY_STATE_PHASE PARTIAL_HCC_REPLICAS \
+ PARTIAL_WORKFLOW_REPLICAS PARTIAL_CONTROL_API_REPLICAS <<<"$state_output"
+ [ -n "$WRITER_RECOVERY_STATE_PHASE" ] &&
+ [[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] &&
+ [[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] &&
+ [[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || {
+ err "Durable writer-recovery state is incomplete; refusing profile mutation"
+ return 1
+ }
+ WRITER_RECOVERY_STATE_LOADED=true
+ case "$WRITER_RECOVERY_STATE_PHASE" in
+ planned)
+ # No scale has been authorized yet. Re-run classification with the
+ # recorded original replica counts and either clear or resume it.
+ ;;
+ hcc-fencing|hcc-fenced)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ WRITER_RECOVERY_FENCE_PENDING=true
+ ;;
+ workflow-fencing|workflow-fenced)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ WRITER_RECOVERY_FENCE_PENDING=true
+ ;;
+ api-fencing)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_CONTROL_API_FENCED=true
+ WRITER_RECOVERY_FENCE_PENDING=true
+ ;;
+ api-fenced)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_CONTROL_API_FENCED=true
+ WRITER_RECOVERY_FENCE_PENDING=true
+ ;;
+ policy-ready|roles-ready)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ PARTIAL_CONTROL_API_FENCED=true
+ ;;
+ api-restored|overlay-applied)
+ WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
+ PARTIAL_HCC_FENCED=true
+ PARTIAL_WORKFLOW_FENCED=true
+ ;;
+ *)
+ err "Durable writer-recovery state has an unsupported phase: $WRITER_RECOVERY_STATE_PHASE"
+ return 1
+ ;;
+ esac
+ log "Resuming durable writer recovery at phase ${WRITER_RECOVERY_STATE_PHASE}"
+}
+
+writer_recovery_state_prepare() {
+ local state
+ if [ "$WRITER_RECOVERY_STATE_LOADED" = true ]; then
+ return 0
+ fi
+ state="$($KC -n control-plane get deployment/host-context-controller \
+ -o 'jsonpath={.spec.replicas}')" || return 1
+ PARTIAL_HCC_REPLICAS="$state"
+ state="$($KC -n control-plane get deployment/workflow-recipes \
+ -o 'jsonpath={.spec.replicas}')" || return 1
+ PARTIAL_WORKFLOW_REPLICAS="$state"
+ state="$($KC -n control-plane get deployment/control-api \
+ -o 'jsonpath={.spec.replicas}')" || return 1
+ PARTIAL_CONTROL_API_REPLICAS="$state"
+ [[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] || return 1
+ [[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] || return 1
+ [[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || return 1
+ writer_recovery_state_phase planned
+}
+
+writer_recovery_policy_ready() {
+ case "$WRITER_RECOVERY_STATE_PHASE" in
+ policy-ready|roles-ready|api-restored|overlay-applied) return 0 ;;
+ *) return 1 ;;
+ esac
+}
control_api_is_ready() {
local state replicas ready
@@ -1142,15 +1290,13 @@ runtime_role_secret_is_missing() {
}
fence_partial_control_api() {
- local replicas pods
- replicas="$($KC -n control-plane get deployment/control-api -o 'jsonpath={.spec.replicas}')" \
- || return 1
- if [[ ! "$replicas" =~ ^[1-9][0-9]*$ ]]; then
+ local pods
+ if [[ ! "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then
err "control-api replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
- PARTIAL_CONTROL_API_REPLICAS="$replicas"
log "Fencing control-api writers at ${PARTIAL_CONTROL_API_REPLICAS} replica(s)"
+ writer_recovery_state_phase api-fencing
$KC -n control-plane scale deployment/control-api --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l 'app=control-api,!clerum.io/component' -o name)" \
|| return 1
@@ -1159,18 +1305,17 @@ fence_partial_control_api() {
-l 'app=control-api,!clerum.io/component' --timeout=180s >/dev/null
fi
PARTIAL_CONTROL_API_FENCED=true
+ writer_recovery_state_phase api-fenced
}
fence_partial_workflow_reconciler() {
- local replicas pods
- replicas="$($KC -n control-plane get deployment/workflow-recipes -o 'jsonpath={.spec.replicas}')" \
- || return 1
- if [[ ! "$replicas" =~ ^[0-9]+$ ]]; then
+ local pods
+ if [[ ! "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]]; then
err "workflow-recipes replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
- PARTIAL_WORKFLOW_REPLICAS="$replicas"
log "Fencing workflow-recipes at ${PARTIAL_WORKFLOW_REPLICAS} replica(s)"
+ writer_recovery_state_phase workflow-fencing
$KC -n control-plane scale deployment/workflow-recipes --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l app=workflow-recipes -o name)" \
|| return 1
@@ -1179,18 +1324,17 @@ fence_partial_workflow_reconciler() {
-l app=workflow-recipes --timeout=180s >/dev/null
fi
PARTIAL_WORKFLOW_FENCED=true
+ writer_recovery_state_phase workflow-fenced
}
fence_partial_hcc() {
- local replicas pods
- replicas="$($KC -n control-plane get deployment/host-context-controller -o 'jsonpath={.spec.replicas}')" \
- || return 1
- if [[ ! "$replicas" =~ ^[0-9]+$ ]]; then
+ local pods
+ if [[ ! "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]]; then
err "host-context-controller replica count is invalid; refusing partial-bootstrap recovery"
return 1
fi
- PARTIAL_HCC_REPLICAS="$replicas"
log "Fencing host-context-controller at ${PARTIAL_HCC_REPLICAS} replica(s)"
+ writer_recovery_state_phase hcc-fencing
$KC -n control-plane scale deployment/host-context-controller --replicas=0 >/dev/null
pods="$($KC -n control-plane get pods -l app=host-context-controller -o name)" \
|| return 1
@@ -1199,34 +1343,126 @@ fence_partial_hcc() {
-l app=host-context-controller --timeout=180s >/dev/null
fi
PARTIAL_HCC_FENCED=true
+ writer_recovery_state_phase hcc-fenced
}
fence_partial_bootstrap_writers() {
# HCC is fenced first so it cannot reconcile the existing GlobalFileSystem
# while the control-plane writer window is being closed. The control-api
# selector excludes migration Jobs, which intentionally share app=control-api.
+ writer_recovery_state_prepare
fence_partial_hcc
fence_partial_workflow_reconciler
fence_partial_control_api
}
apply_refreshed_k8s_api_network_policies() {
- local policy_file="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}/patches/k8s-api-ip.yaml"
- if [ ! -s "$policy_file" ]; then
- err "Refreshed Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"
+ if [ ! -s "$K8S_API_EGRESS_POLICY_FILE" ]; then
+ err "Validated Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"
return 1
fi
- # full-setup refreshes this generated file before classification, but the
- # full overlay is deliberately deferred while a writer recovery is active.
- # Apply only the API-egress policies after every control-plane writer is
- # fenced, so control-api can reach Kubernetes without waking HCC/workflow
- # reconciliation or changing any GFS-owned resource.
- log "Applying refreshed Kubernetes API egress policies while writers are fenced"
- $KC apply -f "$policy_file" >/dev/null
+ if writer_recovery_policy_ready; then
+ log "Validated Kubernetes API egress policy already staged; resuming writer recovery"
+ return 0
+ fi
+ if [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; then
+ # The generated k8s-api-ip.yaml is a strategic-merge patch and omits the
+ # inherited podSelector. Apply only the complete object extracted from the
+ # active kustomize render, while every control-plane writer is fenced.
+ log "Applying the complete refreshed Kubernetes API egress policy while writers are fenced"
+ $KC apply -f "$K8S_API_EGRESS_POLICY_FILE" >/dev/null
+ else
+ log "Current Kubernetes API egress policy already matches the rendered endpoint"
+ fi
+ writer_recovery_state_phase policy-ready
+}
+
+prepare_refreshed_k8s_api_network_policy() {
+ local endpoint_ip rendered_file live_file live_status=0 check_status=0
+ local octet
+ endpoint_ip="$($KC get endpoints kubernetes \
+ -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null)" || {
+ err "Unable to read the Kubernetes API endpoint; refusing writer recovery"
+ return 1
+ }
+ if [[ ! "$endpoint_ip" =~ ^[0-9]+(\.[0-9]+){3}$ ]]; then
+ err "Kubernetes API endpoint is not exactly one IPv4 address; refusing writer recovery"
+ return 1
+ fi
+ IFS=. read -r endpoint_octet_1 endpoint_octet_2 endpoint_octet_3 endpoint_octet_4 <<<"$endpoint_ip"
+ for octet in "$endpoint_octet_1" "$endpoint_octet_2" "$endpoint_octet_3" "$endpoint_octet_4"; do
+ if ! [[ "$octet" =~ ^[0-9]+$ ]] || (( 10#$octet > 255 )); then
+ err "Kubernetes API endpoint contains an invalid IPv4 octet; refusing writer recovery"
+ return 1
+ fi
+ done
+
+ [ -f "$WRITER_RECOVERY_STATE_HELPER" ] || {
+ err "Writer recovery state helper is missing"
+ return 1
+ }
+ [ -f "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" ] || {
+ err "Kubernetes API policy validator is missing; refusing writer recovery"
+ return 1
+ }
+ K8S_API_POLICY_TMP_DIR="$(mktemp -d "${T2_TMP_ROOT:-${TMPDIR:-/tmp}}/evenfire-k8s-api-policy.XXXXXX")"
+ rendered_file="${K8S_API_POLICY_TMP_DIR}/rendered.yaml"
+ K8S_API_EGRESS_POLICY_FILE="${K8S_API_POLICY_TMP_DIR}/control-plane-policy.yaml"
+ live_file="${K8S_API_POLICY_TMP_DIR}/live.json"
+
+ # Render the active overlay first so the object applied during recovery
+ # includes every inherited field from the base NetworkPolicy.
+ if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" >"$rendered_file"; then
+ err "Unable to render the active overlay; refusing writer recovery"
+ return 1
+ fi
+ if ! RUBYOPT=--disable=gems ruby \
+ "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --extract "${endpoint_ip}/32" \
+ <"$rendered_file" >"$K8S_API_EGRESS_POLICY_FILE"; then
+ err "Rendered Kubernetes API NetworkPolicy is incomplete or unsafe; refusing writer recovery"
+ return 1
+ fi
+
+ if $KC -n control-plane get networkpolicy/allow-k8s-api-egress-control-plane \
+ -o json --ignore-not-found >"$live_file" 2>/dev/null; then
+ :
+ else
+ live_status=$?
+ err "Unable to inspect the live Kubernetes API NetworkPolicy (status ${live_status}); refusing writer recovery"
+ return 1
+ fi
+ if [ ! -s "$live_file" ] || [ "$(tr -d '[:space:]' <"$live_file")" = "{}" ]; then
+ K8S_API_EGRESS_POLICY_DRIFT=true
+ log "Kubernetes API egress NetworkPolicy is absent; a complete rendered policy will be staged after fencing"
+ return 0
+ fi
+ if RUBYOPT=--disable=gems ruby \
+ "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --check-live "${endpoint_ip}/32" \
+ <"$live_file" >/dev/null; then
+ K8S_API_EGRESS_POLICY_DRIFT=false
+ log "Kubernetes API egress NetworkPolicy matches the current endpoint"
+ return 0
+ else
+ check_status=$?
+ fi
+ if [ "$check_status" -eq 1 ]; then
+ K8S_API_EGRESS_POLICY_DRIFT=true
+ log "Kubernetes API egress NetworkPolicy has a recognized endpoint/port drift; a complete policy will be staged after fencing"
+ return 0
+ fi
+ err "Live Kubernetes API NetworkPolicy has an unexpected shape; refusing writer recovery"
+ return 1
}
restore_partial_control_api() {
if [ "$PARTIAL_CONTROL_API_FENCED" != true ]; then
+ if [[ "$WRITER_RECOVERY_STATE_PHASE" == api-restored ||
+ "$WRITER_RECOVERY_STATE_PHASE" == overlay-applied ]]; then
+ if control_api_is_ready; then
+ return 0
+ fi
+ return 1
+ fi
return 1
fi
log "Restoring control-api to ${PARTIAL_CONTROL_API_REPLICAS} replica(s) after runtime-role provisioning"
@@ -1234,6 +1470,7 @@ restore_partial_control_api() {
--replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null
$KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null
PARTIAL_CONTROL_API_FENCED=false
+ writer_recovery_state_phase api-restored
}
restore_partial_non_api_writers() {
@@ -1271,13 +1508,17 @@ reconcile_existing_gfs_credentials() {
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
}
+writer_recovery_state_load || exit 1
+prepare_refreshed_k8s_api_network_policy || exit 1
CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
if [ "$RESET_DB" = true ]; then
log "Database reset path — HCC cutover deferred until post-convergence verification"
else
writer_dsn="$($KC -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"
if [ -n "${writer_dsn}" ]; then
- if control_api_is_ready; then
+ if [ "$WRITER_RECOVERY" = true ]; then
+ log "Existing GFS writer recovery is already in progress; preserving the durable fence state"
+ elif control_api_is_ready; then
log "Existing GFS writer detected — reconciling credentials before overlay upgrade"
reconcile_existing_gfs_credentials
else
@@ -1286,27 +1527,45 @@ else
err "Unable to determine control-api readiness; refusing to classify the GFS upgrade path"
exit 1
fi
- fence_partial_bootstrap_writers
- apply_refreshed_k8s_api_network_policies
if runtime_role_secret_is_missing; then
+ runtime_secret_status=0
+ else
+ runtime_secret_status=$?
+ fi
+ if [ "$runtime_secret_status" -eq 0 ]; then
WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
log "Existing GFS writer and empty control-api runtime Secret detected — fencing writers until migrations and runtime roles converge"
- else
- runtime_secret_status=$?
- if [ "$runtime_secret_status" -ne 1 ]; then
- err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"
- exit 1
- fi
+ elif [ "$runtime_secret_status" -eq 1 ] && [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; then
WRITER_RECOVERY=true
+ PARTIAL_BOOTSTRAP_RECOVERY=true
log "Existing GFS writer detected with control-api not Ready — repairing the stale API-egress policy before the fail-closed readiness decision"
+ elif [ "$runtime_secret_status" -eq 1 ]; then
+ err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
+ exit 1
+ else
+ err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"
+ exit 1
fi
+ fence_partial_bootstrap_writers
+ apply_refreshed_k8s_api_network_policies
fi
else
log "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"
fi
fi
+if [ "$WRITER_RECOVERY" = true ] && [ "$WRITER_RECOVERY_FENCE_PENDING" = true ]; then
+ fence_partial_bootstrap_writers
+ apply_refreshed_k8s_api_network_policies
+ WRITER_RECOVERY_FENCE_PENDING=false
+fi
+if [ "$WRITER_RECOVERY_STATE_LOADED" = true ] &&
+ [ "$WRITER_RECOVERY_STATE_PHASE" = planned ] &&
+ [ "$WRITER_RECOVERY" = false ]; then
+ writer_recovery_state_clear
+fi
+
log "Applying kustomize overlay (${ACTIVE_MINIKUBE_RENDER_DIR})..."
if [ "$RESET_DB" = true ]; then
# Keep control-api scaled to zero until migrations and role restoration are
@@ -1356,6 +1615,9 @@ else
CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \
bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"
ok "Control-api database migrations and runtime roles applied"
+ if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then
+ writer_recovery_state_phase roles-ready
+ fi
# The credential probe runs through control-api, so prove that deployment is
# live before normal staging on bootstrap or an idempotent upgrade. On the
@@ -1387,6 +1649,7 @@ else
log "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"
reconcile_existing_gfs_credentials
$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
+ writer_recovery_state_phase overlay-applied
restore_partial_non_api_writers
ok "Deferred full kustomize overlay applied after fenced writer/runtime-role readiness"
fi
@@ -1411,6 +1674,9 @@ else
# The preceding staged sync is allowed to wait for this DSN-producing
# reconcile; the strict pass is the durable consumer attestation.
bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs
+ if [ "$WRITER_RECOVERY" = true ]; then
+ writer_recovery_state_clear
+ fi
ok "GFS credentials reconciled and writer bootstrap verified"
fi
diff --git a/scripts/minikube/port-forward-owner.sh b/scripts/minikube/port-forward-owner.sh
index ddaed49a6..ba51cf92b 100755
--- a/scripts/minikube/port-forward-owner.sh
+++ b/scripts/minikube/port-forward-owner.sh
@@ -364,8 +364,21 @@ pf_owner_cleanup_record() {
return
;;
live)
- actual_start="$(pf_owner_process_start "${pid}")" ||
- pf_owner_error "cannot recheck live PID ${pid} after TERM; leaving ${pidfile}" || return 1
+ if ! actual_start="$(pf_owner_process_start "${pid}")"; then
+ # A port-forward can exit in the small interval between the state
+ # probe above and ps(1) re-reading its start time. It is safe to
+ # reap and remove the record only when a second state probe proves
+ # that the exact process is now dead; a live or ambiguous PID still
+ # fails closed and keeps its ownership evidence.
+ state="$(pf_owner_process_state "${pid}")"
+ if [[ "${state}" == dead ]]; then
+ pf_owner_reap_process "${pid}"
+ pf_owner_remove_dead_record "${pidfile}"
+ return
+ fi
+ pf_owner_error "cannot recheck live PID ${pid} after TERM; leaving ${pidfile}"
+ return 1
+ fi
[[ "${actual_start}" == "${PF_OWNER_RECORD_START}" ]] ||
pf_owner_error "PID ${pid} was reused after TERM; leaving ${pidfile}" || return 1
;;
diff --git a/scripts/minikube/validate-k8s-api-egress-policy.rb b/scripts/minikube/validate-k8s-api-egress-policy.rb
new file mode 100755
index 000000000..baf34839e
--- /dev/null
+++ b/scripts/minikube/validate-k8s-api-egress-policy.rb
@@ -0,0 +1,162 @@
+#!/usr/bin/env ruby
+# Validate and extract the fully rendered control-plane Kubernetes API
+# NetworkPolicy. The generated k8s-api-ip.yaml is a strategic-merge patch and
+# is intentionally not safe to apply as a standalone resource.
+
+require "json"
+require "yaml"
+
+TARGET_API_VERSION = "networking.k8s.io/v1"
+TARGET_KIND = "NetworkPolicy"
+TARGET_NAME = "allow-k8s-api-egress-control-plane"
+TARGET_NAMESPACE = "control-plane"
+EXPECTED_SELECTOR = {
+ "matchExpressions" => [
+ {
+ "key" => "app",
+ "operator" => "In",
+ "values" => [
+ "host-context-controller",
+ "workflow-recipes",
+ "control-api",
+ "trace-maintenance-worker"
+ ]
+ }
+ ]
+}.freeze
+EXPECTED_POLICY_TYPES = ["Egress"].freeze
+EXPECTED_PORTS = [
+ { "port" => 443, "protocol" => "TCP" },
+ { "port" => 8443, "protocol" => "TCP" }
+].freeze
+
+class PolicyValidationError < StandardError; end
+
+def fail_policy(message, status = 2)
+ warn "K8S_API_POLICY_ERROR: #{message}"
+ exit status
+end
+
+def target_identity!(policy)
+ unless policy.is_a?(Hash) &&
+ policy["apiVersion"] == TARGET_API_VERSION &&
+ policy["kind"] == TARGET_KIND
+ raise PolicyValidationError, "rendered object is not the expected NetworkPolicy"
+ end
+
+ metadata = policy["metadata"]
+ unless metadata.is_a?(Hash) && metadata["name"] == TARGET_NAME &&
+ metadata["namespace"] == TARGET_NAMESPACE
+ raise PolicyValidationError, "NetworkPolicy identity is not control-plane/#{TARGET_NAME}"
+ end
+end
+
+def validated_signature!(policy, expected_cidr, exact_ports:)
+ target_identity!(policy)
+ spec = policy["spec"]
+ unless spec.is_a?(Hash) && spec.keys.sort == %w[egress podSelector policyTypes]
+ raise PolicyValidationError, "NetworkPolicy spec is incomplete or contains unexpected fields"
+ end
+ unless spec["podSelector"] == EXPECTED_SELECTOR
+ raise PolicyValidationError, "NetworkPolicy podSelector is not the complete control-plane writer selector"
+ end
+ unless spec["policyTypes"] == EXPECTED_POLICY_TYPES
+ raise PolicyValidationError, "NetworkPolicy policyTypes is not exactly Egress"
+ end
+
+ egress = spec["egress"]
+ unless egress.is_a?(Array) && egress.length == 1 && egress[0].is_a?(Hash)
+ raise PolicyValidationError, "NetworkPolicy must contain exactly one egress rule"
+ end
+ rule = egress[0]
+ unless rule.keys.sort == %w[ports to]
+ raise PolicyValidationError, "NetworkPolicy egress rule is incomplete or contains unexpected fields"
+ end
+
+ peers = rule["to"]
+ unless peers.is_a?(Array) && peers.length == 1 && peers[0].is_a?(Hash) &&
+ peers[0].keys == ["ipBlock"] && peers[0]["ipBlock"].is_a?(Hash)
+ raise PolicyValidationError, "NetworkPolicy must contain exactly one ipBlock peer"
+ end
+ ip_block = peers[0]["ipBlock"]
+ unless ip_block.keys == ["cidr"] && ip_block["cidr"].is_a?(String) &&
+ ip_block["cidr"].match?(/\A(?:\d{1,3}\.){3}\d{1,3}\/32\z/)
+ raise PolicyValidationError, "NetworkPolicy must contain exactly one IPv4 /32 CIDR"
+ end
+ if exact_ports && ip_block["cidr"] != expected_cidr
+ raise PolicyValidationError, "rendered NetworkPolicy CIDR does not match the current Kubernetes API endpoint"
+ end
+
+ ports = rule["ports"]
+ unless ports.is_a?(Array) && !ports.empty? && ports.all? do |port|
+ port.is_a?(Hash) && port.keys.sort == %w[port protocol] &&
+ port["port"].is_a?(Integer) && [443, 8443].include?(port["port"]) &&
+ port["protocol"] == "TCP"
+ end
+ raise PolicyValidationError, "NetworkPolicy ports are not restricted to TCP 443/8443"
+ end
+ normalized_ports = ports.map { |port| { "port" => port["port"], "protocol" => port["protocol"] } }
+ if normalized_ports.uniq.length != normalized_ports.length
+ raise PolicyValidationError, "NetworkPolicy egress ports contain duplicates"
+ end
+ if exact_ports && normalized_ports.sort_by { |port| port["port"] } != EXPECTED_PORTS
+ raise PolicyValidationError, "rendered NetworkPolicy does not contain exactly TCP 443 and 8443"
+ end
+
+ {
+ "selector" => spec["podSelector"],
+ "policyTypes" => spec["policyTypes"],
+ "ports" => normalized_ports.sort_by { |port| port["port"] },
+ "cidr" => ip_block["cidr"]
+ }
+end
+
+mode = ARGV.shift
+cidr = ARGV.shift
+unless ["--extract", "--check-live"].include?(mode) && cidr&.match?(/\A(?:\d{1,3}\.){3}\d{1,3}\/32\z/)
+ fail_policy "usage: #{File.basename($PROGRAM_NAME)} --extract|--check-live <ipv4>/32"
+end
+
+if mode == "--extract"
+ documents = YAML.load_stream($stdin.read).compact
+ candidates = documents.select do |document|
+ begin
+ target_identity!(document)
+ true
+ rescue PolicyValidationError
+ false
+ end
+ end
+ fail_policy "rendered overlay must contain exactly one #{TARGET_NAMESPACE}/#{TARGET_NAME}" unless candidates.length == 1
+ begin
+ validated_signature!(candidates.first, cidr, exact_ports: true)
+ rescue PolicyValidationError => error
+ fail_policy error.message
+ end
+ # Emit only the complete, merged object. Applying this output cannot create
+ # the broad all-pods policy that applying the strategic-merge patch itself
+ # could create.
+ print YAML.dump(candidates.first)
+else
+ raw = $stdin.read
+ if raw.strip.empty?
+ puts "MISSING"
+ exit 1
+ end
+ begin
+ live = JSON.parse(raw)
+ rescue JSON::ParserError => error
+ fail_policy "live NetworkPolicy JSON is invalid: #{error.message}"
+ end
+ begin
+ signature = validated_signature!(live, cidr, exact_ports: false)
+ rescue PolicyValidationError => error
+ fail_policy error.message
+ end
+ if signature["cidr"] == cidr && signature["ports"] == EXPECTED_PORTS
+ puts "MATCH"
+ exit 0
+ end
+ puts "DRIFT"
+ exit 1
+end
diff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.py
new file mode 100755
index 000000000..f059053a6
--- /dev/null
+++ b/scripts/minikube/writer-recovery-state.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Private, atomic, identity-bound state for partial writer recovery.
+
+The state contains only recovery phase and replica counts. It is deliberately
+not shell-sourced: an interrupted local run must not turn an untrusted file
+into executable shell input.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import tempfile
+from pathlib import Path
+
+
+VERSION = 1
+PHASES = {
+ "planned",
+ "hcc-fencing",
+ "hcc-fenced",
+ "workflow-fencing",
+ "workflow-fenced",
+ "api-fencing",
+ "api-fenced",
+ "policy-ready",
+ "roles-ready",
+ "api-restored",
+ "overlay-applied",
+}
+
+
+def fail(message: str, status: int = 2) -> None:
+ raise SystemExit(f"WRITER_RECOVERY_STATE_ERROR: {message}")
+
+
+def validate_identity(args: argparse.Namespace) -> dict[str, str]:
+ values = {
+ "profile": args.profile,
+ "context": args.context,
+ "worktree": args.worktree,
+ "branch": args.branch,
+ "head": args.head,
+ }
+ if any(not value or any(char in value for char in "\r\n\t") for value in values.values()):
+ fail("identity fields must be non-empty and control-character free")
+ return values
+
+
+def validate_replica(value: str, name: str, *, allow_zero: bool = False) -> int:
+ try:
+ number = int(value)
+ except (TypeError, ValueError):
+ fail(f"{name} replica count is not an integer")
+ if number < 0 or (number == 0 and not allow_zero):
+ fail(f"{name} replica count is outside the supported recovery range")
+ return number
+
+
+def state_path(args: argparse.Namespace) -> Path:
+ path = Path(args.path)
+ if not path.is_absolute() or path.name != path.name.replace("/", ""):
+ fail("state path must be an absolute file path")
+ if path.is_symlink():
+ fail(f"refusing a symlinked state path: {path}")
+ return path
+
+
+def write_state(args: argparse.Namespace) -> None:
+ identity = validate_identity(args)
+ if args.phase not in PHASES:
+ fail(f"unsupported recovery phase: {args.phase}")
+ payload = {
+ "version": VERSION,
+ **identity,
+ "phase": args.phase,
+ "hccReplicas": validate_replica(args.hcc, "hcc", allow_zero=True),
+ "workflowReplicas": validate_replica(args.workflow, "workflow", allow_zero=True),
+ "controlApiReplicas": validate_replica(args.control_api, "control-api"),
+ }
+ path = state_path(args)
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ if path.exists() and path.is_symlink():
+ fail(f"refusing a symlinked state path: {path}")
+ fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
+ try:
+ os.fchmod(fd, 0o600)
+ with os.fdopen(fd, "w", encoding="utf-8") as stream:
+ json.dump(payload, stream, sort_keys=True, separators=(",", ":"))
+ stream.write("\n")
+ os.replace(temporary, path)
+ os.chmod(path, 0o600)
+ except Exception:
+ try:
+ os.unlink(temporary)
+ except FileNotFoundError:
+ pass
+ raise
+
+
+def read_state(args: argparse.Namespace) -> None:
+ identity = validate_identity(args)
+ path = state_path(args)
+ if not path.exists():
+ print("NONE")
+ return
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as error:
+ fail(f"cannot read state: {error}")
+ if not isinstance(data, dict) or data.get("version") != VERSION:
+ fail("state version is unsupported")
+ for key, expected in identity.items():
+ if data.get(key) != expected:
+ fail(f"state identity mismatch for {key}; refusing to touch another lane")
+ phase = data.get("phase")
+ if phase not in PHASES:
+ fail("state phase is invalid")
+ hcc = validate_replica(str(data.get("hccReplicas")), "hcc", allow_zero=True)
+ workflow = validate_replica(str(data.get("workflowReplicas")), "workflow", allow_zero=True)
+ control_api = validate_replica(str(data.get("controlApiReplicas")), "control-api")
+ print(f"{phase}|{hcc}|{workflow}|{control_api}")
+
+
+def clear_state(args: argparse.Namespace) -> None:
+ path = state_path(args)
+ if path.exists() or path.is_symlink():
+ if path.is_symlink():
+ fail(f"refusing to unlink a symlinked state path: {path}")
+ path.unlink()
+
+
+def parser() -> argparse.ArgumentParser:
+ result = argparse.ArgumentParser(description=__doc__)
+ subparsers = result.add_subparsers(dest="command", required=True)
+ common = argparse.ArgumentParser(add_help=False)
+ common.add_argument("--path", required=True)
+ common.add_argument("--profile", required=True)
+ common.add_argument("--context", required=True)
+ common.add_argument("--worktree", required=True)
+ common.add_argument("--branch", required=True)
+ common.add_argument("--head", required=True)
+ write = subparsers.add_parser("write", parents=[common])
+ write.add_argument("--phase", required=True)
+ write.add_argument("--hcc", required=True)
+ write.add_argument("--workflow", required=True)
+ write.add_argument("--control-api", required=True)
+ subparsers.add_parser("read", parents=[common])
+ subparsers.add_parser("clear", parents=[common])
+ return result
+
+
+def main() -> None:
+ args = parser().parse_args()
+ if args.command == "write":
+ write_state(args)
+ elif args.command == "read":
+ read_state(args)
+ else:
+ clear_state(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index c6a235f06..fee63107c 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -169,6 +169,7 @@ assert_minikube_upgrade_classifier() {
reconcile="$(grep -n 'reconcile-gfs-deploy-credentials.sh' <<<"$block" | head -1 | cut -d: -f1)"
fallback="$(grep -nE '^[[:space:]]*else' <<<"$block" | tail -1 | cut -d: -f1)"
fresh="$(grep -ni 'fresh bootstrap' <<<"$block" | tail -1 | cut -d: -f1)"
+ classifier_start="$dsn"
for line_name in dsn classify ready abort reconcile fallback fresh; do
line_value="${!line_name}"
if [[ -n "$line_value" ]]; then
@@ -181,16 +182,25 @@ assert_minikube_upgrade_classifier() {
if [[ -n "$readiness_probe" ]]; then
readiness_probe=$((block_start + readiness_probe - 1))
fi
- writer_flag="$(grep -n 'WRITER_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"
- partial_flag="$(grep -n 'PARTIAL_BOOTSTRAP_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"
- fence_call="$(grep -nE '^[[:space:]]+fence_partial_bootstrap_writers$' "$path" | tail -1 | cut -d: -f1)"
- policy_apply="$(grep -nE '^[[:space:]]+apply_refreshed_k8s_api_network_policies$' "$path" | tail -1 | cut -d: -f1)"
+ # The durable resume classifier is defined before the executable upgrade
+ # branch. Select the first recovery assignment/fence after the writer
+ # Secret read so this contract checks mutation order, not helper details.
+ writer_flag="$(grep -n 'WRITER_RECOVERY=true' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"
+ partial_flag="$(grep -n 'PARTIAL_BOOTSTRAP_RECOVERY=true' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"
+ fence_call="$(grep -nE '^[[:space:]]+fence_partial_bootstrap_writers$' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"
+ policy_apply="$(grep -nE '^[[:space:]]+apply_refreshed_k8s_api_network_policies$' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"
migration="$(grep -n 'run-control-api-db-migration.sh' "$path" | head -1 | cut -d: -f1)"
roles="$(grep -n 'provision-control-api-runtime-roles.sh' "$path" | head -1 | cut -d: -f1)"
restore_api="$(grep -nE '^[[:space:]]+restore_partial_control_api$' "$path" | tail -1 | cut -d: -f1)"
deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"
post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"
post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"
+ for line_name in writer_flag partial_flag fence_call policy_apply; do
+ line_value="${!line_name}"
+ if [[ -n "$line_value" ]]; then
+ printf -v "$line_name" '%d' "$((block_start + line_value - 1))"
+ fi
+ done
[[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$writer_flag" && \
-n "$partial_flag" && -n "$fence_call" && -n "$policy_apply" && \
-n "$fallback" && -n "$fresh" && -n "$migration" && \
@@ -198,8 +208,8 @@ assert_minikube_upgrade_classifier() {
-n "$post_ready" && -n "$post_reconcile" ]] \
|| fail "$path recovery classifier is incomplete"
[[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \
- "$readiness_probe" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \
- "$policy_apply" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \
+ "$readiness_probe" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \
+ "$partial_flag" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \
"$partial_flag" -lt "$fallback" && "$fallback" -lt "$fresh" && \
"$migration" -lt "$roles" && "$roles" -lt "$restore_api" && \
"$restore_api" -lt "$post_reconcile" && "$post_reconcile" -lt "$deferred_full_overlay" && \
diff --git a/scripts/tests/test-minikube-k8s-api-egress-policy.sh b/scripts/tests/test-minikube-k8s-api-egress-policy.sh
new file mode 100755
index 000000000..686cdc253
--- /dev/null
+++ b/scripts/tests/test-minikube-k8s-api-egress-policy.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# The recovery path must apply a complete rendered object, never the
+# strategic-merge patch that omits the control-plane podSelector.
+set -euo pipefail
+set +x
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+HELPER="$ROOT/scripts/minikube/validate-k8s-api-egress-policy.rb"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-k8s-api-policy.XXXXXX")"
+trap 'rm -rf -- "$TMP_DIR"' EXIT
+
+fail() {
+ printf 'FAIL: %s\n' "$*" >&2
+ exit 1
+}
+pass() {
+ printf 'PASS: %s\n' "$1"
+}
+
+cat >"$TMP_DIR/rendered.yaml" <<'YAML'
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: unrelated-policy
+ namespace: control-plane
+spec:
+ podSelector: {}
+ policyTypes: [Egress]
+ egress: []
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-k8s-api-egress-control-plane
+ namespace: control-plane
+ labels:
+ clerum.io/policy-type: infrastructure
+spec:
+ podSelector:
+ matchExpressions:
+ - key: app
+ operator: In
+ values: [host-context-controller, workflow-recipes, control-api, trace-maintenance-worker]
+ policyTypes: [Egress]
+ egress:
+ - ports:
+ - port: 443
+ protocol: TCP
+ - port: 8443
+ protocol: TCP
+ to:
+ - ipBlock:
+ cidr: 172.17.0.3/32
+YAML
+
+RUBYOPT=--disable=gems ruby "$HELPER" --extract 172.17.0.3/32 <"$TMP_DIR/rendered.yaml" >"$TMP_DIR/complete.yaml"
+grep -Fq 'name: allow-k8s-api-egress-control-plane' "$TMP_DIR/complete.yaml" ||
+ fail 'complete extraction omitted the target identity'
+grep -Fq 'matchExpressions:' "$TMP_DIR/complete.yaml" ||
+ fail 'complete extraction omitted the inherited podSelector'
+! grep -Fq 'name: unrelated-policy' "$TMP_DIR/complete.yaml" ||
+ fail 'complete extraction emitted an unrelated object'
+pass 'rendered extraction emits one complete control-plane policy'
+
+RUBYOPT=--disable=gems ruby -rjson -ryaml -e 'puts JSON.generate(YAML.load_stream(File.read(ARGV.fetch(0))).first)' \
+ "$TMP_DIR/complete.yaml" >"$TMP_DIR/complete.json"
+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/complete.json" | grep -Fxq MATCH ||
+ fail 'complete live policy was not recognized as matching'
+pass 'complete live policy matches the current endpoint and ports'
+
+RUBYOPT=--disable=gems ruby -rjson -ryaml -e '
+ value = YAML.load_stream(File.read(ARGV.fetch(0))).first
+ value.fetch("spec").fetch("egress").first.fetch("to").first.fetch("ipBlock")["cidr"] = "172.17.0.5/32"
+ puts JSON.generate(value)
+' "$TMP_DIR/complete.yaml" >"$TMP_DIR/stale.json"
+stale_status=0
+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/stale.json" >"$TMP_DIR/stale.out" || stale_status=$?
+[[ "$stale_status" -eq 1 ]] || fail 'stale endpoint did not return the drift status'
+grep -Fxq DRIFT "$TMP_DIR/stale.out" || fail 'stale endpoint was not classified as drift'
+pass 'stale endpoint policy is classified as a repairable drift'
+
+RUBYOPT=--disable=gems ruby -rjson -ryaml -e '
+ value = YAML.load_stream(File.read(ARGV.fetch(0))).first
+ value.fetch("spec")["podSelector"] = {}
+ puts JSON.generate(value)
+' "$TMP_DIR/complete.yaml" >"$TMP_DIR/broad.json"
+broad_status=0
+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/broad.json" >"$TMP_DIR/broad.out" 2>"$TMP_DIR/broad.err" || broad_status=$?
+[[ "$broad_status" -eq 2 ]] || fail 'broad all-pods selector was accepted for recovery'
+grep -Fq 'complete control-plane writer selector' "$TMP_DIR/broad.err" ||
+ fail 'broad selector failure was not explicit'
+pass 'broad selectors fail closed instead of being overwritten'
+
+cat >"$TMP_DIR/strategic-patch.yaml" <<'YAML'
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: allow-k8s-api-egress-control-plane
+ namespace: control-plane
+spec:
+ egress:
+ - to:
+ - ipBlock:
+ cidr: 172.17.0.3/32
+ ports:
+ - port: 443
+ protocol: TCP
+ - port: 8443
+ protocol: TCP
+YAML
+patch_status=0
+RUBYOPT=--disable=gems ruby "$HELPER" --extract 172.17.0.3/32 <"$TMP_DIR/strategic-patch.yaml" >"$TMP_DIR/unsafe.out" 2>"$TMP_DIR/unsafe.err" || patch_status=$?
+[[ "$patch_status" -eq 2 ]] || fail 'strategic-merge patch without inherited selector was accepted'
+pass 'strategic-merge patch is rejected unless the rendered object is complete'
+
+printf 'PASS: Minikube Kubernetes API egress policy contract\n'
diff --git a/scripts/tests/test-minikube-port-forward-owner.sh b/scripts/tests/test-minikube-port-forward-owner.sh
index 11ff9bab2..9bf0ea8d5 100755
--- a/scripts/tests/test-minikube-port-forward-owner.sh
+++ b/scripts/tests/test-minikube-port-forward-owner.sh
@@ -198,6 +198,43 @@ grep -Fxq "${PID} TERM" "${KILL_LOG}" || fail 'bounded TERM timeout did not sign
[[ ! -s "${REAP_LOG}" ]] || fail 'bounded TERM timeout called blocking reap on a live process'
pass 'TERM-resistant cleanup is bounded and never waits on a live process'
+# A fast child may disappear after the post-TERM state probe but before ps(1)
+# can return its start time. The cleanup contract must accept that only after a
+# second state probe proves the PID is dead; it must never remove a live or
+# ambiguous record on a missing start-time read.
+reset_process_fixture
+write_record
+RACE_AFTER_TERM=false
+RACE_START_FAILED_FILE="${TMP_ROOT}/race-start-failed"
+rm -f -- "${RACE_START_FAILED_FILE}"
+pf_owner_signal_process() {
+ printf '%s %s\n' "$1" "$2" >>"${KILL_LOG}"
+ RACE_AFTER_TERM=true
+}
+pf_owner_process_state() {
+ if [[ "${RACE_AFTER_TERM}" == true ]]; then
+ if [[ -e "${RACE_START_FAILED_FILE}" ]]; then
+ printf 'dead\n'
+ else
+ printf 'live\n'
+ fi
+ else
+ printf 'live\n'
+ fi
+}
+pf_owner_process_start() {
+ if [[ "${RACE_AFTER_TERM}" == true ]]; then
+ : >"${RACE_START_FAILED_FILE}"
+ return 1
+ fi
+ printf '%s\n' "${ACTUAL_START}"
+}
+PF_OWNER_TERMINATE_ATTEMPTS=1 PF_OWNER_TERMINATE_DELAY=0 cleanup_record ||
+ fail 'cleanup rejected a process that exited during the post-TERM start-time race'
+[[ ! -e "${RECORD}" ]] || fail 'post-TERM race cleanup kept the dead ownership record'
+[[ -s "${REAP_LOG}" ]] || fail 'post-TERM race cleanup did not reap the dead process'
+pass 'post-TERM start-time race cleans up only after a dead-state recheck'
+
pf_owner_signal_process() {
printf '%s %s\n' "$1" "$2" >>"${KILL_LOG}"
printf 'dead\n' >"${STATE_FILE}"
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index d5890e912..537ba7ac4 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -52,6 +52,8 @@ done
"$ROOT/scripts/tests/test-minikube-targeted-health.sh"
"$ROOT/scripts/tests/test-minikube-filter-gfs-resources.sh"
"$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh"
+"$ROOT/scripts/tests/test-minikube-k8s-api-egress-policy.sh"
+"$ROOT/scripts/tests/test-minikube-writer-recovery-state.sh"
python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'
import os
import py_compile
diff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.sh
new file mode 100755
index 000000000..fcb3949b4
--- /dev/null
+++ b/scripts/tests/test-minikube-writer-recovery-state.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Durable writer recovery state must be atomic, private, identity-bound, and
+# safe to resume without sourcing shell-controlled content.
+set -euo pipefail
+set +x
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+HELPER="$ROOT/scripts/minikube/writer-recovery-state.py"
+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-writer-state.XXXXXX")"
+trap 'rm -rf -- "$TMP_DIR"' EXIT
+STATE="$TMP_DIR/recovery.json"
+
+fail() {
+ printf 'FAIL: %s\n' "$*" >&2
+ exit 1
+}
+pass() {
+ printf 'PASS: %s\n' "$1"
+}
+
+identity=(
+ --path "$STATE"
+ --profile profile-state
+ --context profile-state
+ --worktree "$TMP_DIR"
+ --branch fix/state
+ --head 0123456789abcdef
+)
+
+python3 "$HELPER" write "${identity[@]}" --phase planned --hcc 1 --workflow 1 --control-api 1
+mode="$(stat -f '%Lp' "$STATE" 2>/dev/null || stat -c '%a' "$STATE")"
+[[ "$mode" == 600 ]] || fail "state mode is $mode, expected 600"
+python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'planned|1|1|1' ||
+ fail 'planned state did not round-trip with exact replicas'
+pass 'writer recovery state is private and round-trips atomically'
+
+python3 "$HELPER" write "${identity[@]}" --phase api-fencing --hcc 1 --workflow 2 --control-api 1
+python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'api-fencing|1|2|1' ||
+ fail 'fencing phase did not preserve original replicas'
+wrong_status=0
+python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 >/dev/null 2>&1 || wrong_status=$?
+[[ "$wrong_status" -ne 0 ]] || fail 'state was readable under a different HEAD'
+[[ -s "$STATE" ]] || fail 'identity mismatch destroyed durable state'
+pass 'writer recovery state fails closed on a different HEAD'
+
+python3 "$HELPER" clear "${identity[@]}"
+[[ ! -e "$STATE" ]] || fail 'clear left durable writer recovery state behind'
+pass 'writer recovery state clears only after the recovery boundary completes'
+
+printf 'PASS: Minikube writer recovery state contract\n'