Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(minikube): make writer recovery resumable and bounded
Alfredo Lopez848ffa0f26ae
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 9ed81a1d2..7df5ee6da 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -749,6 +749,7 @@ ok "Cluster '${PROFILE}' is reachable"maybe_exit_after_cluster_stepSETUP_LOCK_CLEANUP_DONE=false+K8S_API_POLICY_TMP_DIR=""cleanup_setup_lock() {local status="${1:-$?}" cleanup_status=0if [ "$SETUP_LOCK_CLEANUP_DONE" = true ]; then@@ -757,6 +758,9 @@ cleanup_setup_lock() {SETUP_LOCK_CLEANUP_DONE=truetrap - EXITtrap '' INT TERM+ if [ -n "${K8S_API_POLICY_TMP_DIR:-}" ] && [ -d "${K8S_API_POLICY_TMP_DIR}" ]; then+ rm -rf -- "${K8S_API_POLICY_TMP_DIR}" || cleanup_status=1+ fit2_lock_release "$status" || cleanup_status=$?if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; thenstatus="$cleanup_status"@@ -1111,12 +1115,156 @@ ok "Minikube K8s API CIDRs refreshed"# runtime-role readiness has been proved.WRITER_RECOVERY=falsePARTIAL_BOOTSTRAP_RECOVERY=false-PARTIAL_CONTROL_API_REPLICAS=1-PARTIAL_WORKFLOW_REPLICAS=1-PARTIAL_HCC_REPLICAS=1+PARTIAL_CONTROL_API_REPLICAS=""+PARTIAL_WORKFLOW_REPLICAS=""+PARTIAL_HCC_REPLICAS=""PARTIAL_CONTROL_API_FENCED=falsePARTIAL_WORKFLOW_FENCED=falsePARTIAL_HCC_FENCED=false+WRITER_RECOVERY_STATE_HELPER="${SCRIPT_DIR}/writer-recovery-state.py"+WRITER_RECOVERY_STATE_ROOT="${PROJECT_DIR}/.local-notes/infra/runs/writer-recovery"+WRITER_RECOVERY_STATE_FILE="${WRITER_RECOVERY_STATE_ROOT}/${PROFILE}.json"+WRITER_RECOVERY_STATE_LOADED=false+WRITER_RECOVERY_STATE_PHASE=""+WRITER_RECOVERY_FENCE_PENDING=false+K8S_API_EGRESS_POLICY_FILE=""+K8S_API_EGRESS_POLICY_DRIFT=false++writer_recovery_state_cli() {+ local command="$1"+ shift+ python3 "$WRITER_RECOVERY_STATE_HELPER" "$command" \+ --path "$WRITER_RECOVERY_STATE_FILE" \+ --profile "$PROFILE" \+ --context "$T2_CONTEXT" \+ --worktree "$PROJECT_DIR" \+ --branch "$T2_BRANCH" \+ --head "$T2_HEAD" "$@"+}++writer_recovery_state_write() {+ local phase="$1"+ writer_recovery_state_cli write \+ --phase "$phase" \+ --hcc "$PARTIAL_HCC_REPLICAS" \+ --workflow "$PARTIAL_WORKFLOW_REPLICAS" \+ --control-api "$PARTIAL_CONTROL_API_REPLICAS"+}++writer_recovery_state_phase() {+ writer_recovery_state_write "$1"+ WRITER_RECOVERY_STATE_LOADED=true+ WRITER_RECOVERY_STATE_PHASE="$1"+}++writer_recovery_state_clear() {+ writer_recovery_state_cli clear+ WRITER_RECOVERY_STATE_LOADED=false+ WRITER_RECOVERY_STATE_PHASE=""+ WRITER_RECOVERY_FENCE_PENDING=false+}++writer_recovery_state_load() {+ local state_output state_status=0+ state_output="$(writer_recovery_state_cli read)" || state_status=$?+ if [ "$state_status" -ne 0 ]; then+ err "Unable to validate the durable writer-recovery state; refusing profile mutation"+ return 1+ fi+ if [ "$state_output" = NONE ]; then+ return 0+ fi+ IFS='|' read -r WRITER_RECOVERY_STATE_PHASE PARTIAL_HCC_REPLICAS \+ PARTIAL_WORKFLOW_REPLICAS PARTIAL_CONTROL_API_REPLICAS <<<"$state_output"+ [ -n "$WRITER_RECOVERY_STATE_PHASE" ] &&+ [[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] &&+ [[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] &&+ [[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || {+ err "Durable writer-recovery state is incomplete; refusing profile mutation"+ return 1+ }+ WRITER_RECOVERY_STATE_LOADED=true+ case "$WRITER_RECOVERY_STATE_PHASE" in+ planned)+ # No scale has been authorized yet. Re-run classification with the+ # recorded original replica counts and either clear or resume it.+ ;;+ hcc-fencing|hcc-fenced)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ WRITER_RECOVERY_FENCE_PENDING=true+ ;;+ workflow-fencing|workflow-fenced)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ WRITER_RECOVERY_FENCE_PENDING=true+ ;;+ api-fencing)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ PARTIAL_CONTROL_API_FENCED=true+ WRITER_RECOVERY_FENCE_PENDING=true+ ;;+ api-fenced)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ PARTIAL_CONTROL_API_FENCED=true+ WRITER_RECOVERY_FENCE_PENDING=true+ ;;+ policy-ready|roles-ready)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ PARTIAL_CONTROL_API_FENCED=true+ ;;+ api-restored|overlay-applied)+ WRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=true+ PARTIAL_HCC_FENCED=true+ PARTIAL_WORKFLOW_FENCED=true+ ;;+ *)+ err "Durable writer-recovery state has an unsupported phase: $WRITER_RECOVERY_STATE_PHASE"+ return 1+ ;;+ esac+ log "Resuming durable writer recovery at phase ${WRITER_RECOVERY_STATE_PHASE}"+}++writer_recovery_state_prepare() {+ local state+ if [ "$WRITER_RECOVERY_STATE_LOADED" = true ]; then+ return 0+ fi+ state="$($KC -n control-plane get deployment/host-context-controller \+ -o 'jsonpath={.spec.replicas}')" || return 1+ PARTIAL_HCC_REPLICAS="$state"+ state="$($KC -n control-plane get deployment/workflow-recipes \+ -o 'jsonpath={.spec.replicas}')" || return 1+ PARTIAL_WORKFLOW_REPLICAS="$state"+ state="$($KC -n control-plane get deployment/control-api \+ -o 'jsonpath={.spec.replicas}')" || return 1+ PARTIAL_CONTROL_API_REPLICAS="$state"+ [[ "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]] || return 1+ [[ "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]] || return 1+ [[ "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]] || return 1+ writer_recovery_state_phase planned+}++writer_recovery_policy_ready() {+ case "$WRITER_RECOVERY_STATE_PHASE" in+ policy-ready|roles-ready|api-restored|overlay-applied) return 0 ;;+ *) return 1 ;;+ esac+}control_api_is_ready() {local state replicas ready@@ -1142,15 +1290,13 @@ runtime_role_secret_is_missing() {}fence_partial_control_api() {- local replicas pods- replicas="$($KC -n control-plane get deployment/control-api -o 'jsonpath={.spec.replicas}')" \- || return 1- if [[ ! "$replicas" =~ ^[1-9][0-9]*$ ]]; then+ local pods+ if [[ ! "$PARTIAL_CONTROL_API_REPLICAS" =~ ^[1-9][0-9]*$ ]]; thenerr "control-api replica count is invalid; refusing partial-bootstrap recovery"return 1fi- PARTIAL_CONTROL_API_REPLICAS="$replicas"log "Fencing control-api writers at ${PARTIAL_CONTROL_API_REPLICAS} replica(s)"+ writer_recovery_state_phase api-fencing$KC -n control-plane scale deployment/control-api --replicas=0 >/dev/nullpods="$($KC -n control-plane get pods -l 'app=control-api,!clerum.io/component' -o name)" \|| return 1@@ -1159,18 +1305,17 @@ fence_partial_control_api() {-l 'app=control-api,!clerum.io/component' --timeout=180s >/dev/nullfiPARTIAL_CONTROL_API_FENCED=true+ writer_recovery_state_phase api-fenced}fence_partial_workflow_reconciler() {- local replicas pods- replicas="$($KC -n control-plane get deployment/workflow-recipes -o 'jsonpath={.spec.replicas}')" \- || return 1- if [[ ! "$replicas" =~ ^[0-9]+$ ]]; then+ local pods+ if [[ ! "$PARTIAL_WORKFLOW_REPLICAS" =~ ^[0-9]+$ ]]; thenerr "workflow-recipes replica count is invalid; refusing partial-bootstrap recovery"return 1fi- PARTIAL_WORKFLOW_REPLICAS="$replicas"log "Fencing workflow-recipes at ${PARTIAL_WORKFLOW_REPLICAS} replica(s)"+ writer_recovery_state_phase workflow-fencing$KC -n control-plane scale deployment/workflow-recipes --replicas=0 >/dev/nullpods="$($KC -n control-plane get pods -l app=workflow-recipes -o name)" \|| return 1@@ -1179,18 +1324,17 @@ fence_partial_workflow_reconciler() {-l app=workflow-recipes --timeout=180s >/dev/nullfiPARTIAL_WORKFLOW_FENCED=true+ writer_recovery_state_phase workflow-fenced}fence_partial_hcc() {- local replicas pods- replicas="$($KC -n control-plane get deployment/host-context-controller -o 'jsonpath={.spec.replicas}')" \- || return 1- if [[ ! "$replicas" =~ ^[0-9]+$ ]]; then+ local pods+ if [[ ! "$PARTIAL_HCC_REPLICAS" =~ ^[0-9]+$ ]]; thenerr "host-context-controller replica count is invalid; refusing partial-bootstrap recovery"return 1fi- PARTIAL_HCC_REPLICAS="$replicas"log "Fencing host-context-controller at ${PARTIAL_HCC_REPLICAS} replica(s)"+ writer_recovery_state_phase hcc-fencing$KC -n control-plane scale deployment/host-context-controller --replicas=0 >/dev/nullpods="$($KC -n control-plane get pods -l app=host-context-controller -o name)" \|| return 1@@ -1199,34 +1343,126 @@ fence_partial_hcc() {-l app=host-context-controller --timeout=180s >/dev/nullfiPARTIAL_HCC_FENCED=true+ writer_recovery_state_phase hcc-fenced}fence_partial_bootstrap_writers() {# HCC is fenced first so it cannot reconcile the existing GlobalFileSystem# while the control-plane writer window is being closed. The control-api# selector excludes migration Jobs, which intentionally share app=control-api.+ writer_recovery_state_preparefence_partial_hccfence_partial_workflow_reconcilerfence_partial_control_api}apply_refreshed_k8s_api_network_policies() {- local policy_file="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}/patches/k8s-api-ip.yaml"- if [ ! -s "$policy_file" ]; then- err "Refreshed Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"+ if [ ! -s "$K8S_API_EGRESS_POLICY_FILE" ]; then+ err "Validated Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"return 1fi- # full-setup refreshes this generated file before classification, but the- # full overlay is deliberately deferred while a writer recovery is active.- # Apply only the API-egress policies after every control-plane writer is- # fenced, so control-api can reach Kubernetes without waking HCC/workflow- # reconciliation or changing any GFS-owned resource.- log "Applying refreshed Kubernetes API egress policies while writers are fenced"- $KC apply -f "$policy_file" >/dev/null+ if writer_recovery_policy_ready; then+ log "Validated Kubernetes API egress policy already staged; resuming writer recovery"+ return 0+ fi+ if [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; then+ # The generated k8s-api-ip.yaml is a strategic-merge patch and omits the+ # inherited podSelector. Apply only the complete object extracted from the+ # active kustomize render, while every control-plane writer is fenced.+ log "Applying the complete refreshed Kubernetes API egress policy while writers are fenced"+ $KC apply -f "$K8S_API_EGRESS_POLICY_FILE" >/dev/null+ else+ log "Current Kubernetes API egress policy already matches the rendered endpoint"+ fi+ writer_recovery_state_phase policy-ready+}++prepare_refreshed_k8s_api_network_policy() {+ local endpoint_ip rendered_file live_file live_status=0 check_status=0+ local octet+ endpoint_ip="$($KC get endpoints kubernetes \+ -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null)" || {+ err "Unable to read the Kubernetes API endpoint; refusing writer recovery"+ return 1+ }+ if [[ ! "$endpoint_ip" =~ ^[0-9]+(\.[0-9]+){3}$ ]]; then+ err "Kubernetes API endpoint is not exactly one IPv4 address; refusing writer recovery"+ return 1+ fi+ IFS=. read -r endpoint_octet_1 endpoint_octet_2 endpoint_octet_3 endpoint_octet_4 <<<"$endpoint_ip"+ for octet in "$endpoint_octet_1" "$endpoint_octet_2" "$endpoint_octet_3" "$endpoint_octet_4"; do+ if ! [[ "$octet" =~ ^[0-9]+$ ]] || (( 10#$octet > 255 )); then+ err "Kubernetes API endpoint contains an invalid IPv4 octet; refusing writer recovery"+ return 1+ fi+ done++ [ -f "$WRITER_RECOVERY_STATE_HELPER" ] || {+ err "Writer recovery state helper is missing"+ return 1+ }+ [ -f "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" ] || {+ err "Kubernetes API policy validator is missing; refusing writer recovery"+ return 1+ }+ K8S_API_POLICY_TMP_DIR="$(mktemp -d "${T2_TMP_ROOT:-${TMPDIR:-/tmp}}/evenfire-k8s-api-policy.XXXXXX")"+ rendered_file="${K8S_API_POLICY_TMP_DIR}/rendered.yaml"+ K8S_API_EGRESS_POLICY_FILE="${K8S_API_POLICY_TMP_DIR}/control-plane-policy.yaml"+ live_file="${K8S_API_POLICY_TMP_DIR}/live.json"++ # Render the active overlay first so the object applied during recovery+ # includes every inherited field from the base NetworkPolicy.+ if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" >"$rendered_file"; then+ err "Unable to render the active overlay; refusing writer recovery"+ return 1+ fi+ if ! RUBYOPT=--disable=gems ruby \+ "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --extract "${endpoint_ip}/32" \+ <"$rendered_file" >"$K8S_API_EGRESS_POLICY_FILE"; then+ err "Rendered Kubernetes API NetworkPolicy is incomplete or unsafe; refusing writer recovery"+ return 1+ fi++ if $KC -n control-plane get networkpolicy/allow-k8s-api-egress-control-plane \+ -o json --ignore-not-found >"$live_file" 2>/dev/null; then+ :+ else+ live_status=$?+ err "Unable to inspect the live Kubernetes API NetworkPolicy (status ${live_status}); refusing writer recovery"+ return 1+ fi+ if [ ! -s "$live_file" ] || [ "$(tr -d '[:space:]' <"$live_file")" = "{}" ]; then+ K8S_API_EGRESS_POLICY_DRIFT=true+ log "Kubernetes API egress NetworkPolicy is absent; a complete rendered policy will be staged after fencing"+ return 0+ fi+ if RUBYOPT=--disable=gems ruby \+ "${SCRIPT_DIR}/validate-k8s-api-egress-policy.rb" --check-live "${endpoint_ip}/32" \+ <"$live_file" >/dev/null; then+ K8S_API_EGRESS_POLICY_DRIFT=false+ log "Kubernetes API egress NetworkPolicy matches the current endpoint"+ return 0+ else+ check_status=$?+ fi+ if [ "$check_status" -eq 1 ]; then+ K8S_API_EGRESS_POLICY_DRIFT=true+ log "Kubernetes API egress NetworkPolicy has a recognized endpoint/port drift; a complete policy will be staged after fencing"+ return 0+ fi+ err "Live Kubernetes API NetworkPolicy has an unexpected shape; refusing writer recovery"+ return 1}restore_partial_control_api() {if [ "$PARTIAL_CONTROL_API_FENCED" != true ]; then+ if [[ "$WRITER_RECOVERY_STATE_PHASE" == api-restored ||+ "$WRITER_RECOVERY_STATE_PHASE" == overlay-applied ]]; then+ if control_api_is_ready; then+ return 0+ fi+ return 1+ fireturn 1filog "Restoring control-api to ${PARTIAL_CONTROL_API_REPLICAS} replica(s) after runtime-role provisioning"@@ -1234,6 +1470,7 @@ restore_partial_control_api() {--replicas="$PARTIAL_CONTROL_API_REPLICAS" >/dev/null$KC -n control-plane rollout status deployment/control-api --timeout=180s >/dev/nullPARTIAL_CONTROL_API_FENCED=false+ writer_recovery_state_phase api-restored}restore_partial_non_api_writers() {@@ -1271,13 +1508,17 @@ reconcile_existing_gfs_credentials() {bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs}+writer_recovery_state_load || exit 1+prepare_refreshed_k8s_api_network_policy || exit 1CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"if [ "$RESET_DB" = true ]; thenlog "Database reset path — HCC cutover deferred until post-convergence verification"elsewriter_dsn="$($KC -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"if [ -n "${writer_dsn}" ]; then- if control_api_is_ready; then+ if [ "$WRITER_RECOVERY" = true ]; then+ log "Existing GFS writer recovery is already in progress; preserving the durable fence state"+ elif control_api_is_ready; thenlog "Existing GFS writer detected — reconciling credentials before overlay upgrade"reconcile_existing_gfs_credentialselse@@ -1286,27 +1527,45 @@ elseerr "Unable to determine control-api readiness; refusing to classify the GFS upgrade path"exit 1fi- fence_partial_bootstrap_writers- apply_refreshed_k8s_api_network_policiesif runtime_role_secret_is_missing; then+ runtime_secret_status=0+ else+ runtime_secret_status=$?+ fi+ if [ "$runtime_secret_status" -eq 0 ]; thenWRITER_RECOVERY=truePARTIAL_BOOTSTRAP_RECOVERY=truelog "Existing GFS writer and empty control-api runtime Secret detected — fencing writers until migrations and runtime roles converge"- else- runtime_secret_status=$?- if [ "$runtime_secret_status" -ne 1 ]; then- err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"- exit 1- fi+ elif [ "$runtime_secret_status" -eq 1 ] && [ "$K8S_API_EGRESS_POLICY_DRIFT" = true ]; thenWRITER_RECOVERY=true+ PARTIAL_BOOTSTRAP_RECOVERY=truelog "Existing GFS writer detected with control-api not Ready — repairing the stale API-egress policy before the fail-closed readiness decision"+ elif [ "$runtime_secret_status" -eq 1 ]; then+ err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"+ exit 1+ else+ err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"+ exit 1fi+ fence_partial_bootstrap_writers+ apply_refreshed_k8s_api_network_policiesfielselog "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"fifi+if [ "$WRITER_RECOVERY" = true ] && [ "$WRITER_RECOVERY_FENCE_PENDING" = true ]; then+ fence_partial_bootstrap_writers+ apply_refreshed_k8s_api_network_policies+ WRITER_RECOVERY_FENCE_PENDING=false+fi+if [ "$WRITER_RECOVERY_STATE_LOADED" = true ] &&+ [ "$WRITER_RECOVERY_STATE_PHASE" = planned ] &&+ [ "$WRITER_RECOVERY" = false ]; then+ writer_recovery_state_clear+fi+log "Applying kustomize overlay (${ACTIVE_MINIKUBE_RENDER_DIR})..."if [ "$RESET_DB" = true ]; then# Keep control-api scaled to zero until migrations and role restoration are@@ -1356,6 +1615,9 @@ elseCONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"ok "Control-api database migrations and runtime roles applied"+ if [ "$WRITER_RECOVERY" = true ] && [ "$PARTIAL_CONTROL_API_FENCED" = true ]; then+ writer_recovery_state_phase roles-ready+ fi# The credential probe runs through control-api, so prove that deployment is# live before normal staging on bootstrap or an idempotent upgrade. On the@@ -1387,6 +1649,7 @@ elselog "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"reconcile_existing_gfs_credentials$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -+ writer_recovery_state_phase overlay-appliedrestore_partial_non_api_writersok "Deferred full kustomize overlay applied after fenced writer/runtime-role readiness"fi@@ -1411,6 +1674,9 @@ else# The preceding staged sync is allowed to wait for this DSN-producing# reconcile; the strict pass is the durable consumer attestation.bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+ if [ "$WRITER_RECOVERY" = true ]; then+ writer_recovery_state_clear+ fiok "GFS credentials reconciled and writer bootstrap verified"fidiff --git a/scripts/minikube/port-forward-owner.sh b/scripts/minikube/port-forward-owner.shindex ddaed49a6..ba51cf92b 100755--- a/scripts/minikube/port-forward-owner.sh+++ b/scripts/minikube/port-forward-owner.sh@@ -364,8 +364,21 @@ pf_owner_cleanup_record() {return;;live)- actual_start="$(pf_owner_process_start "${pid}")" ||- pf_owner_error "cannot recheck live PID ${pid} after TERM; leaving ${pidfile}" || return 1+ if ! actual_start="$(pf_owner_process_start "${pid}")"; then+ # A port-forward can exit in the small interval between the state+ # probe above and ps(1) re-reading its start time. It is safe to+ # reap and remove the record only when a second state probe proves+ # that the exact process is now dead; a live or ambiguous PID still+ # fails closed and keeps its ownership evidence.+ state="$(pf_owner_process_state "${pid}")"+ if [[ "${state}" == dead ]]; then+ pf_owner_reap_process "${pid}"+ pf_owner_remove_dead_record "${pidfile}"+ return+ fi+ pf_owner_error "cannot recheck live PID ${pid} after TERM; leaving ${pidfile}"+ return 1+ fi[[ "${actual_start}" == "${PF_OWNER_RECORD_START}" ]] ||pf_owner_error "PID ${pid} was reused after TERM; leaving ${pidfile}" || return 1;;diff --git a/scripts/minikube/validate-k8s-api-egress-policy.rb b/scripts/minikube/validate-k8s-api-egress-policy.rbnew file mode 100755index 000000000..baf34839e--- /dev/null+++ b/scripts/minikube/validate-k8s-api-egress-policy.rb@@ -0,0 +1,162 @@+#!/usr/bin/env ruby+# Validate and extract the fully rendered control-plane Kubernetes API+# NetworkPolicy. The generated k8s-api-ip.yaml is a strategic-merge patch and+# is intentionally not safe to apply as a standalone resource.++require "json"+require "yaml"++TARGET_API_VERSION = "networking.k8s.io/v1"+TARGET_KIND = "NetworkPolicy"+TARGET_NAME = "allow-k8s-api-egress-control-plane"+TARGET_NAMESPACE = "control-plane"+EXPECTED_SELECTOR = {+ "matchExpressions" => [+ {+ "key" => "app",+ "operator" => "In",+ "values" => [+ "host-context-controller",+ "workflow-recipes",+ "control-api",+ "trace-maintenance-worker"+ ]+ }+ ]+}.freeze+EXPECTED_POLICY_TYPES = ["Egress"].freeze+EXPECTED_PORTS = [+ { "port" => 443, "protocol" => "TCP" },+ { "port" => 8443, "protocol" => "TCP" }+].freeze++class PolicyValidationError < StandardError; end++def fail_policy(message, status = 2)+ warn "K8S_API_POLICY_ERROR: #{message}"+ exit status+end++def target_identity!(policy)+ unless policy.is_a?(Hash) &&+ policy["apiVersion"] == TARGET_API_VERSION &&+ policy["kind"] == TARGET_KIND+ raise PolicyValidationError, "rendered object is not the expected NetworkPolicy"+ end++ metadata = policy["metadata"]+ unless metadata.is_a?(Hash) && metadata["name"] == TARGET_NAME &&+ metadata["namespace"] == TARGET_NAMESPACE+ raise PolicyValidationError, "NetworkPolicy identity is not control-plane/#{TARGET_NAME}"+ end+end++def validated_signature!(policy, expected_cidr, exact_ports:)+ target_identity!(policy)+ spec = policy["spec"]+ unless spec.is_a?(Hash) && spec.keys.sort == %w[egress podSelector policyTypes]+ raise PolicyValidationError, "NetworkPolicy spec is incomplete or contains unexpected fields"+ end+ unless spec["podSelector"] == EXPECTED_SELECTOR+ raise PolicyValidationError, "NetworkPolicy podSelector is not the complete control-plane writer selector"+ end+ unless spec["policyTypes"] == EXPECTED_POLICY_TYPES+ raise PolicyValidationError, "NetworkPolicy policyTypes is not exactly Egress"+ end++ egress = spec["egress"]+ unless egress.is_a?(Array) && egress.length == 1 && egress[0].is_a?(Hash)+ raise PolicyValidationError, "NetworkPolicy must contain exactly one egress rule"+ end+ rule = egress[0]+ unless rule.keys.sort == %w[ports to]+ raise PolicyValidationError, "NetworkPolicy egress rule is incomplete or contains unexpected fields"+ end++ peers = rule["to"]+ unless peers.is_a?(Array) && peers.length == 1 && peers[0].is_a?(Hash) &&+ peers[0].keys == ["ipBlock"] && peers[0]["ipBlock"].is_a?(Hash)+ raise PolicyValidationError, "NetworkPolicy must contain exactly one ipBlock peer"+ end+ ip_block = peers[0]["ipBlock"]+ unless ip_block.keys == ["cidr"] && ip_block["cidr"].is_a?(String) &&+ ip_block["cidr"].match?(/\A(?:\d{1,3}\.){3}\d{1,3}\/32\z/)+ raise PolicyValidationError, "NetworkPolicy must contain exactly one IPv4 /32 CIDR"+ end+ if exact_ports && ip_block["cidr"] != expected_cidr+ raise PolicyValidationError, "rendered NetworkPolicy CIDR does not match the current Kubernetes API endpoint"+ end++ ports = rule["ports"]+ unless ports.is_a?(Array) && !ports.empty? && ports.all? do |port|+ port.is_a?(Hash) && port.keys.sort == %w[port protocol] &&+ port["port"].is_a?(Integer) && [443, 8443].include?(port["port"]) &&+ port["protocol"] == "TCP"+ end+ raise PolicyValidationError, "NetworkPolicy ports are not restricted to TCP 443/8443"+ end+ normalized_ports = ports.map { |port| { "port" => port["port"], "protocol" => port["protocol"] } }+ if normalized_ports.uniq.length != normalized_ports.length+ raise PolicyValidationError, "NetworkPolicy egress ports contain duplicates"+ end+ if exact_ports && normalized_ports.sort_by { |port| port["port"] } != EXPECTED_PORTS+ raise PolicyValidationError, "rendered NetworkPolicy does not contain exactly TCP 443 and 8443"+ end++ {+ "selector" => spec["podSelector"],+ "policyTypes" => spec["policyTypes"],+ "ports" => normalized_ports.sort_by { |port| port["port"] },+ "cidr" => ip_block["cidr"]+ }+end++mode = ARGV.shift+cidr = ARGV.shift+unless ["--extract", "--check-live"].include?(mode) && cidr&.match?(/\A(?:\d{1,3}\.){3}\d{1,3}\/32\z/)+ fail_policy "usage: #{File.basename($PROGRAM_NAME)} --extract|--check-live <ipv4>/32"+end++if mode == "--extract"+ documents = YAML.load_stream($stdin.read).compact+ candidates = documents.select do |document|+ begin+ target_identity!(document)+ true+ rescue PolicyValidationError+ false+ end+ end+ fail_policy "rendered overlay must contain exactly one #{TARGET_NAMESPACE}/#{TARGET_NAME}" unless candidates.length == 1+ begin+ validated_signature!(candidates.first, cidr, exact_ports: true)+ rescue PolicyValidationError => error+ fail_policy error.message+ end+ # Emit only the complete, merged object. Applying this output cannot create+ # the broad all-pods policy that applying the strategic-merge patch itself+ # could create.+ print YAML.dump(candidates.first)+else+ raw = $stdin.read+ if raw.strip.empty?+ puts "MISSING"+ exit 1+ end+ begin+ live = JSON.parse(raw)+ rescue JSON::ParserError => error+ fail_policy "live NetworkPolicy JSON is invalid: #{error.message}"+ end+ begin+ signature = validated_signature!(live, cidr, exact_ports: false)+ rescue PolicyValidationError => error+ fail_policy error.message+ end+ if signature["cidr"] == cidr && signature["ports"] == EXPECTED_PORTS+ puts "MATCH"+ exit 0+ end+ puts "DRIFT"+ exit 1+enddiff --git a/scripts/minikube/writer-recovery-state.py b/scripts/minikube/writer-recovery-state.pynew file mode 100755index 000000000..f059053a6--- /dev/null+++ b/scripts/minikube/writer-recovery-state.py@@ -0,0 +1,165 @@+#!/usr/bin/env python3+"""Private, atomic, identity-bound state for partial writer recovery.++The state contains only recovery phase and replica counts. It is deliberately+not shell-sourced: an interrupted local run must not turn an untrusted file+into executable shell input.+"""++from __future__ import annotations++import argparse+import json+import os+import tempfile+from pathlib import Path+++VERSION = 1+PHASES = {+ "planned",+ "hcc-fencing",+ "hcc-fenced",+ "workflow-fencing",+ "workflow-fenced",+ "api-fencing",+ "api-fenced",+ "policy-ready",+ "roles-ready",+ "api-restored",+ "overlay-applied",+}+++def fail(message: str, status: int = 2) -> None:+ raise SystemExit(f"WRITER_RECOVERY_STATE_ERROR: {message}")+++def validate_identity(args: argparse.Namespace) -> dict[str, str]:+ values = {+ "profile": args.profile,+ "context": args.context,+ "worktree": args.worktree,+ "branch": args.branch,+ "head": args.head,+ }+ if any(not value or any(char in value for char in "\r\n\t") for value in values.values()):+ fail("identity fields must be non-empty and control-character free")+ return values+++def validate_replica(value: str, name: str, *, allow_zero: bool = False) -> int:+ try:+ number = int(value)+ except (TypeError, ValueError):+ fail(f"{name} replica count is not an integer")+ if number < 0 or (number == 0 and not allow_zero):+ fail(f"{name} replica count is outside the supported recovery range")+ return number+++def state_path(args: argparse.Namespace) -> Path:+ path = Path(args.path)+ if not path.is_absolute() or path.name != path.name.replace("/", ""):+ fail("state path must be an absolute file path")+ if path.is_symlink():+ fail(f"refusing a symlinked state path: {path}")+ return path+++def write_state(args: argparse.Namespace) -> None:+ identity = validate_identity(args)+ if args.phase not in PHASES:+ fail(f"unsupported recovery phase: {args.phase}")+ payload = {+ "version": VERSION,+ **identity,+ "phase": args.phase,+ "hccReplicas": validate_replica(args.hcc, "hcc", allow_zero=True),+ "workflowReplicas": validate_replica(args.workflow, "workflow", allow_zero=True),+ "controlApiReplicas": validate_replica(args.control_api, "control-api"),+ }+ path = state_path(args)+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)+ if path.exists() and path.is_symlink():+ fail(f"refusing a symlinked state path: {path}")+ fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))+ try:+ os.fchmod(fd, 0o600)+ with os.fdopen(fd, "w", encoding="utf-8") as stream:+ json.dump(payload, stream, sort_keys=True, separators=(",", ":"))+ stream.write("\n")+ os.replace(temporary, path)+ os.chmod(path, 0o600)+ except Exception:+ try:+ os.unlink(temporary)+ except FileNotFoundError:+ pass+ raise+++def read_state(args: argparse.Namespace) -> None:+ identity = validate_identity(args)+ path = state_path(args)+ if not path.exists():+ print("NONE")+ return+ try:+ data = json.loads(path.read_text(encoding="utf-8"))+ except (OSError, json.JSONDecodeError) as error:+ fail(f"cannot read state: {error}")+ if not isinstance(data, dict) or data.get("version") != VERSION:+ fail("state version is unsupported")+ for key, expected in identity.items():+ if data.get(key) != expected:+ fail(f"state identity mismatch for {key}; refusing to touch another lane")+ phase = data.get("phase")+ if phase not in PHASES:+ fail("state phase is invalid")+ hcc = validate_replica(str(data.get("hccReplicas")), "hcc", allow_zero=True)+ workflow = validate_replica(str(data.get("workflowReplicas")), "workflow", allow_zero=True)+ control_api = validate_replica(str(data.get("controlApiReplicas")), "control-api")+ print(f"{phase}|{hcc}|{workflow}|{control_api}")+++def clear_state(args: argparse.Namespace) -> None:+ path = state_path(args)+ if path.exists() or path.is_symlink():+ if path.is_symlink():+ fail(f"refusing to unlink a symlinked state path: {path}")+ path.unlink()+++def parser() -> argparse.ArgumentParser:+ result = argparse.ArgumentParser(description=__doc__)+ subparsers = result.add_subparsers(dest="command", required=True)+ common = argparse.ArgumentParser(add_help=False)+ common.add_argument("--path", required=True)+ common.add_argument("--profile", required=True)+ common.add_argument("--context", required=True)+ common.add_argument("--worktree", required=True)+ common.add_argument("--branch", required=True)+ common.add_argument("--head", required=True)+ write = subparsers.add_parser("write", parents=[common])+ write.add_argument("--phase", required=True)+ write.add_argument("--hcc", required=True)+ write.add_argument("--workflow", required=True)+ write.add_argument("--control-api", required=True)+ subparsers.add_parser("read", parents=[common])+ subparsers.add_parser("clear", parents=[common])+ return result+++def main() -> None:+ args = parser().parse_args()+ if args.command == "write":+ write_state(args)+ elif args.command == "read":+ read_state(args)+ else:+ clear_state(args)+++if __name__ == "__main__":+ main()diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex c6a235f06..fee63107c 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -169,6 +169,7 @@ assert_minikube_upgrade_classifier() {reconcile="$(grep -n 'reconcile-gfs-deploy-credentials.sh' <<<"$block" | head -1 | cut -d: -f1)"fallback="$(grep -nE '^[[:space:]]*else' <<<"$block" | tail -1 | cut -d: -f1)"fresh="$(grep -ni 'fresh bootstrap' <<<"$block" | tail -1 | cut -d: -f1)"+ classifier_start="$dsn"for line_name in dsn classify ready abort reconcile fallback fresh; doline_value="${!line_name}"if [[ -n "$line_value" ]]; then@@ -181,16 +182,25 @@ assert_minikube_upgrade_classifier() {if [[ -n "$readiness_probe" ]]; thenreadiness_probe=$((block_start + readiness_probe - 1))fi- writer_flag="$(grep -n 'WRITER_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"- partial_flag="$(grep -n 'PARTIAL_BOOTSTRAP_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"- fence_call="$(grep -nE '^[[:space:]]+fence_partial_bootstrap_writers$' "$path" | tail -1 | cut -d: -f1)"- policy_apply="$(grep -nE '^[[:space:]]+apply_refreshed_k8s_api_network_policies$' "$path" | tail -1 | cut -d: -f1)"+ # The durable resume classifier is defined before the executable upgrade+ # branch. Select the first recovery assignment/fence after the writer+ # Secret read so this contract checks mutation order, not helper details.+ writer_flag="$(grep -n 'WRITER_RECOVERY=true' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"+ partial_flag="$(grep -n 'PARTIAL_BOOTSTRAP_RECOVERY=true' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"+ fence_call="$(grep -nE '^[[:space:]]+fence_partial_bootstrap_writers$' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"+ policy_apply="$(grep -nE '^[[:space:]]+apply_refreshed_k8s_api_network_policies$' <<<"${block}" | awk -F: -v start="${classifier_start}" '$1 >= start { print $1; exit }')"migration="$(grep -n 'run-control-api-db-migration.sh' "$path" | head -1 | cut -d: -f1)"roles="$(grep -n 'provision-control-api-runtime-roles.sh' "$path" | head -1 | cut -d: -f1)"restore_api="$(grep -nE '^[[:space:]]+restore_partial_control_api$' "$path" | tail -1 | cut -d: -f1)"deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"+ for line_name in writer_flag partial_flag fence_call policy_apply; do+ line_value="${!line_name}"+ if [[ -n "$line_value" ]]; then+ printf -v "$line_name" '%d' "$((block_start + line_value - 1))"+ fi+ done[[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$writer_flag" && \-n "$partial_flag" && -n "$fence_call" && -n "$policy_apply" && \-n "$fallback" && -n "$fresh" && -n "$migration" && \@@ -198,8 +208,8 @@ assert_minikube_upgrade_classifier() {-n "$post_ready" && -n "$post_reconcile" ]] \|| fail "$path recovery classifier is incomplete"[[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \- "$readiness_probe" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \- "$policy_apply" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \+ "$readiness_probe" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \+ "$partial_flag" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \"$partial_flag" -lt "$fallback" && "$fallback" -lt "$fresh" && \"$migration" -lt "$roles" && "$roles" -lt "$restore_api" && \"$restore_api" -lt "$post_reconcile" && "$post_reconcile" -lt "$deferred_full_overlay" && \diff --git a/scripts/tests/test-minikube-k8s-api-egress-policy.sh b/scripts/tests/test-minikube-k8s-api-egress-policy.shnew file mode 100755index 000000000..686cdc253--- /dev/null+++ b/scripts/tests/test-minikube-k8s-api-egress-policy.sh@@ -0,0 +1,116 @@+#!/usr/bin/env bash+# The recovery path must apply a complete rendered object, never the+# strategic-merge patch that omits the control-plane podSelector.+set -euo pipefail+set +x++ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+HELPER="$ROOT/scripts/minikube/validate-k8s-api-egress-policy.rb"+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-k8s-api-policy.XXXXXX")"+trap 'rm -rf -- "$TMP_DIR"' EXIT++fail() {+ printf 'FAIL: %s\n' "$*" >&2+ exit 1+}+pass() {+ printf 'PASS: %s\n' "$1"+}++cat >"$TMP_DIR/rendered.yaml" <<'YAML'+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: unrelated-policy+ namespace: control-plane+spec:+ podSelector: {}+ policyTypes: [Egress]+ egress: []+---+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: allow-k8s-api-egress-control-plane+ namespace: control-plane+ labels:+ clerum.io/policy-type: infrastructure+spec:+ podSelector:+ matchExpressions:+ - key: app+ operator: In+ values: [host-context-controller, workflow-recipes, control-api, trace-maintenance-worker]+ policyTypes: [Egress]+ egress:+ - ports:+ - port: 443+ protocol: TCP+ - port: 8443+ protocol: TCP+ to:+ - ipBlock:+ cidr: 172.17.0.3/32+YAML++RUBYOPT=--disable=gems ruby "$HELPER" --extract 172.17.0.3/32 <"$TMP_DIR/rendered.yaml" >"$TMP_DIR/complete.yaml"+grep -Fq 'name: allow-k8s-api-egress-control-plane' "$TMP_DIR/complete.yaml" ||+ fail 'complete extraction omitted the target identity'+grep -Fq 'matchExpressions:' "$TMP_DIR/complete.yaml" ||+ fail 'complete extraction omitted the inherited podSelector'+! grep -Fq 'name: unrelated-policy' "$TMP_DIR/complete.yaml" ||+ fail 'complete extraction emitted an unrelated object'+pass 'rendered extraction emits one complete control-plane policy'++RUBYOPT=--disable=gems ruby -rjson -ryaml -e 'puts JSON.generate(YAML.load_stream(File.read(ARGV.fetch(0))).first)' \+ "$TMP_DIR/complete.yaml" >"$TMP_DIR/complete.json"+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/complete.json" | grep -Fxq MATCH ||+ fail 'complete live policy was not recognized as matching'+pass 'complete live policy matches the current endpoint and ports'++RUBYOPT=--disable=gems ruby -rjson -ryaml -e '+ value = YAML.load_stream(File.read(ARGV.fetch(0))).first+ value.fetch("spec").fetch("egress").first.fetch("to").first.fetch("ipBlock")["cidr"] = "172.17.0.5/32"+ puts JSON.generate(value)+' "$TMP_DIR/complete.yaml" >"$TMP_DIR/stale.json"+stale_status=0+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/stale.json" >"$TMP_DIR/stale.out" || stale_status=$?+[[ "$stale_status" -eq 1 ]] || fail 'stale endpoint did not return the drift status'+grep -Fxq DRIFT "$TMP_DIR/stale.out" || fail 'stale endpoint was not classified as drift'+pass 'stale endpoint policy is classified as a repairable drift'++RUBYOPT=--disable=gems ruby -rjson -ryaml -e '+ value = YAML.load_stream(File.read(ARGV.fetch(0))).first+ value.fetch("spec")["podSelector"] = {}+ puts JSON.generate(value)+' "$TMP_DIR/complete.yaml" >"$TMP_DIR/broad.json"+broad_status=0+RUBYOPT=--disable=gems ruby "$HELPER" --check-live 172.17.0.3/32 <"$TMP_DIR/broad.json" >"$TMP_DIR/broad.out" 2>"$TMP_DIR/broad.err" || broad_status=$?+[[ "$broad_status" -eq 2 ]] || fail 'broad all-pods selector was accepted for recovery'+grep -Fq 'complete control-plane writer selector' "$TMP_DIR/broad.err" ||+ fail 'broad selector failure was not explicit'+pass 'broad selectors fail closed instead of being overwritten'++cat >"$TMP_DIR/strategic-patch.yaml" <<'YAML'+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: allow-k8s-api-egress-control-plane+ namespace: control-plane+spec:+ egress:+ - to:+ - ipBlock:+ cidr: 172.17.0.3/32+ ports:+ - port: 443+ protocol: TCP+ - port: 8443+ protocol: TCP+YAML+patch_status=0+RUBYOPT=--disable=gems ruby "$HELPER" --extract 172.17.0.3/32 <"$TMP_DIR/strategic-patch.yaml" >"$TMP_DIR/unsafe.out" 2>"$TMP_DIR/unsafe.err" || patch_status=$?+[[ "$patch_status" -eq 2 ]] || fail 'strategic-merge patch without inherited selector was accepted'+pass 'strategic-merge patch is rejected unless the rendered object is complete'++printf 'PASS: Minikube Kubernetes API egress policy contract\n'diff --git a/scripts/tests/test-minikube-port-forward-owner.sh b/scripts/tests/test-minikube-port-forward-owner.shindex 11ff9bab2..9bf0ea8d5 100755--- a/scripts/tests/test-minikube-port-forward-owner.sh+++ b/scripts/tests/test-minikube-port-forward-owner.sh@@ -198,6 +198,43 @@ grep -Fxq "${PID} TERM" "${KILL_LOG}" || fail 'bounded TERM timeout did not sign[[ ! -s "${REAP_LOG}" ]] || fail 'bounded TERM timeout called blocking reap on a live process'pass 'TERM-resistant cleanup is bounded and never waits on a live process'+# A fast child may disappear after the post-TERM state probe but before ps(1)+# can return its start time. The cleanup contract must accept that only after a+# second state probe proves the PID is dead; it must never remove a live or+# ambiguous record on a missing start-time read.+reset_process_fixture+write_record+RACE_AFTER_TERM=false+RACE_START_FAILED_FILE="${TMP_ROOT}/race-start-failed"+rm -f -- "${RACE_START_FAILED_FILE}"+pf_owner_signal_process() {+ printf '%s %s\n' "$1" "$2" >>"${KILL_LOG}"+ RACE_AFTER_TERM=true+}+pf_owner_process_state() {+ if [[ "${RACE_AFTER_TERM}" == true ]]; then+ if [[ -e "${RACE_START_FAILED_FILE}" ]]; then+ printf 'dead\n'+ else+ printf 'live\n'+ fi+ else+ printf 'live\n'+ fi+}+pf_owner_process_start() {+ if [[ "${RACE_AFTER_TERM}" == true ]]; then+ : >"${RACE_START_FAILED_FILE}"+ return 1+ fi+ printf '%s\n' "${ACTUAL_START}"+}+PF_OWNER_TERMINATE_ATTEMPTS=1 PF_OWNER_TERMINATE_DELAY=0 cleanup_record ||+ fail 'cleanup rejected a process that exited during the post-TERM start-time race'+[[ ! -e "${RECORD}" ]] || fail 'post-TERM race cleanup kept the dead ownership record'+[[ -s "${REAP_LOG}" ]] || fail 'post-TERM race cleanup did not reap the dead process'+pass 'post-TERM start-time race cleans up only after a dead-state recheck'+pf_owner_signal_process() {printf '%s %s\n' "$1" "$2" >>"${KILL_LOG}"printf 'dead\n' >"${STATE_FILE}"diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex d5890e912..537ba7ac4 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -52,6 +52,8 @@ done"$ROOT/scripts/tests/test-minikube-targeted-health.sh""$ROOT/scripts/tests/test-minikube-filter-gfs-resources.sh""$ROOT/scripts/tests/test-minikube-t1-docker-boundary.sh"+"$ROOT/scripts/tests/test-minikube-k8s-api-egress-policy.sh"+"$ROOT/scripts/tests/test-minikube-writer-recovery-state.sh"python3 - "$GFS_FILTER" "$TMP_ROOT" <<'PY'import osimport py_compilediff --git a/scripts/tests/test-minikube-writer-recovery-state.sh b/scripts/tests/test-minikube-writer-recovery-state.shnew file mode 100755index 000000000..fcb3949b4--- /dev/null+++ b/scripts/tests/test-minikube-writer-recovery-state.sh@@ -0,0 +1,50 @@+#!/usr/bin/env bash+# Durable writer recovery state must be atomic, private, identity-bound, and+# safe to resume without sourcing shell-controlled content.+set -euo pipefail+set +x++ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+HELPER="$ROOT/scripts/minikube/writer-recovery-state.py"+TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-writer-state.XXXXXX")"+trap 'rm -rf -- "$TMP_DIR"' EXIT+STATE="$TMP_DIR/recovery.json"++fail() {+ printf 'FAIL: %s\n' "$*" >&2+ exit 1+}+pass() {+ printf 'PASS: %s\n' "$1"+}++identity=(+ --path "$STATE"+ --profile profile-state+ --context profile-state+ --worktree "$TMP_DIR"+ --branch fix/state+ --head 0123456789abcdef+)++python3 "$HELPER" write "${identity[@]}" --phase planned --hcc 1 --workflow 1 --control-api 1+mode="$(stat -f '%Lp' "$STATE" 2>/dev/null || stat -c '%a' "$STATE")"+[[ "$mode" == 600 ]] || fail "state mode is $mode, expected 600"+python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'planned|1|1|1' ||+ fail 'planned state did not round-trip with exact replicas'+pass 'writer recovery state is private and round-trips atomically'++python3 "$HELPER" write "${identity[@]}" --phase api-fencing --hcc 1 --workflow 2 --control-api 1+python3 "$HELPER" read "${identity[@]}" | grep -Fxq 'api-fencing|1|2|1' ||+ fail 'fencing phase did not preserve original replicas'+wrong_status=0+python3 "$HELPER" read "${identity[@]}" --head fedcba9876543210 >/dev/null 2>&1 || wrong_status=$?+[[ "$wrong_status" -ne 0 ]] || fail 'state was readable under a different HEAD'+[[ -s "$STATE" ]] || fail 'identity mismatch destroyed durable state'+pass 'writer recovery state fails closed on a different HEAD'++python3 "$HELPER" clear "${identity[@]}"+[[ ! -e "$STATE" ]] || fail 'clear left durable writer recovery state behind'+pass 'writer recovery state clears only after the recovery boundary completes'++printf 'PASS: Minikube writer recovery state contract\n'