Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(minikube): repair API egress before writer recovery

Alfredo Lopez42e0bb4b9af0
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 17597cef7..9ed81a1d2 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -1109,6 +1109,7 @@ ok "Minikube K8s API CIDRs refreshed"
# condition either. Fence the control-plane writers before migrations/roles,
# then complete the full overlay and GFS credential reconciliation only after
# runtime-role readiness has been proved.
+WRITER_RECOVERY=false
PARTIAL_BOOTSTRAP_RECOVERY=false
PARTIAL_CONTROL_API_REPLICAS=1
PARTIAL_WORKFLOW_REPLICAS=1
@@ -1209,6 +1210,21 @@ fence_partial_bootstrap_writers() {
fence_partial_control_api
}
+apply_refreshed_k8s_api_network_policies() {
+ local policy_file="${ACTIVE_MINIKUBE_KUSTOMIZE_DIR}/patches/k8s-api-ip.yaml"
+ if [ ! -s "$policy_file" ]; then
+ err "Refreshed Kubernetes API NetworkPolicy manifest is missing; refusing writer recovery"
+ return 1
+ fi
+ # full-setup refreshes this generated file before classification, but the
+ # full overlay is deliberately deferred while a writer recovery is active.
+ # Apply only the API-egress policies after every control-plane writer is
+ # fenced, so control-api can reach Kubernetes without waking HCC/workflow
+ # reconciliation or changing any GFS-owned resource.
+ log "Applying refreshed Kubernetes API egress policies while writers are fenced"
+ $KC apply -f "$policy_file" >/dev/null
+}
+
restore_partial_control_api() {
if [ "$PARTIAL_CONTROL_API_FENCED" != true ]; then
return 1
@@ -1270,18 +1286,20 @@ else
err "Unable to determine control-api readiness; refusing to classify the GFS upgrade path"
exit 1
fi
+ fence_partial_bootstrap_writers
+ apply_refreshed_k8s_api_network_policies
if runtime_role_secret_is_missing; then
+ WRITER_RECOVERY=true
PARTIAL_BOOTSTRAP_RECOVERY=true
log "Existing GFS writer and empty control-api runtime Secret detected — fencing writers until migrations and runtime roles converge"
- fence_partial_bootstrap_writers
else
runtime_secret_status=$?
if [ "$runtime_secret_status" -ne 1 ]; then
err "Unable to inspect control-api-postgres-runtime; refusing to classify the GFS upgrade path"
exit 1
fi
- err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
- exit 1
+ WRITER_RECOVERY=true
+ log "Existing GFS writer detected with control-api not Ready — repairing the stale API-egress policy before the fail-closed readiness decision"
fi
fi
else
@@ -1294,7 +1312,7 @@ if [ "$RESET_DB" = true ]; then
# Keep control-api scaled to zero until migrations and role restoration are
# complete; applying the full overlay here would race it against a fresh DB.
$KC apply -k "$ACTIVE_MINIKUBE_RENDER_DIR" -l app=control-postgres
-elif [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
+elif [ "$WRITER_RECOVERY" = true ]; then
log "Deferring the full kustomize overlay until control-api/runtime roles converge"
else
$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
@@ -1343,10 +1361,21 @@ else
# live before normal staging on bootstrap or an idempotent upgrade. On the
# deferred recovery path this is also the point where the missing runtime
# Secret has been provisioned, so the previously blocked deployment can start.
- if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
- restore_partial_control_api
+ if [ "$WRITER_RECOVERY" = true ]; then
+ if ! restore_partial_control_api; then
+ if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
+ err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
+ else
+ err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
+ fi
+ exit 1
+ fi
if ! control_api_is_ready; then
- err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
+ if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
+ err "Restored control-api is not Ready; refusing GFS cutover after runtime-role provisioning"
+ else
+ err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"
+ fi
exit 1
fi
else
@@ -1354,7 +1383,7 @@ else
$KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null
fi
- if [ "$PARTIAL_BOOTSTRAP_RECOVERY" = true ]; then
+ if [ "$WRITER_RECOVERY" = true ]; then
log "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"
reconcile_existing_gfs_credentials
$KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index 7aa136832..c6a235f06 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -176,33 +176,39 @@ assert_minikube_upgrade_classifier() {
fi
done
if [[ "$path" == *"scripts/minikube/full-setup.sh" ]]; then
- local readiness_probe partial_flag fence_call migration roles restore_api deferred_full_overlay post_ready post_reconcile
+ local readiness_probe writer_flag partial_flag fence_call policy_apply migration roles restore_api deferred_full_overlay post_ready post_reconcile
readiness_probe="$(grep -n 'if control_api_is_ready' <<<"$block" | head -1 | cut -d: -f1)"
if [[ -n "$readiness_probe" ]]; then
readiness_probe=$((block_start + readiness_probe - 1))
fi
+ writer_flag="$(grep -n 'WRITER_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"
partial_flag="$(grep -n 'PARTIAL_BOOTSTRAP_RECOVERY=true' "$path" | head -1 | cut -d: -f1)"
fence_call="$(grep -nE '^[[:space:]]+fence_partial_bootstrap_writers$' "$path" | tail -1 | cut -d: -f1)"
+ policy_apply="$(grep -nE '^[[:space:]]+apply_refreshed_k8s_api_network_policies$' "$path" | tail -1 | cut -d: -f1)"
migration="$(grep -n 'run-control-api-db-migration.sh' "$path" | head -1 | cut -d: -f1)"
roles="$(grep -n 'provision-control-api-runtime-roles.sh' "$path" | head -1 | cut -d: -f1)"
restore_api="$(grep -nE '^[[:space:]]+restore_partial_control_api$' "$path" | tail -1 | cut -d: -f1)"
deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"
post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"
post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"
- [[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$partial_flag" && \
- -n "$fence_call" && -n "$fallback" && -n "$fresh" && -n "$migration" && \
+ [[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$writer_flag" && \
+ -n "$partial_flag" && -n "$fence_call" && -n "$policy_apply" && \
+ -n "$fallback" && -n "$fresh" && -n "$migration" && \
-n "$roles" && -n "$restore_api" && -n "$deferred_full_overlay" && \
-n "$post_ready" && -n "$post_reconcile" ]] \
|| fail "$path recovery classifier is incomplete"
[[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \
- "$readiness_probe" -lt "$partial_flag" && "$partial_flag" -lt "$fence_call" && \
- "$fence_call" -lt "$fallback" && "$fallback" -lt "$fresh" && \
+ "$readiness_probe" -lt "$fence_call" && "$fence_call" -lt "$policy_apply" && \
+ "$policy_apply" -lt "$writer_flag" && "$writer_flag" -lt "$partial_flag" && \
+ "$partial_flag" -lt "$fallback" && "$fallback" -lt "$fresh" && \
"$migration" -lt "$roles" && "$roles" -lt "$restore_api" && \
"$restore_api" -lt "$post_reconcile" && "$post_reconcile" -lt "$deferred_full_overlay" && \
"$post_ready" -lt "$deferred_full_overlay" ]] \
|| fail "$path does not fence writers and defer the full overlay until runtime-role readiness"
grep -Fq 'Existing GFS writer detected but control-api is not Ready; refusing HCC cutover' "$path" \
|| fail "$path no longer retains the fail-closed guard for an unrecognized unready upgrade"
+ grep -Fq 'app=control-api,!clerum.io/component' "$path" \
+ || fail "$path does not exclude migration Jobs from the control-api writer fence"
else
[[ -n "$dsn" && -n "$classify" && -n "$ready" && -n "$abort" && -n "$reconcile" && -n "$fallback" && -n "$fresh" ]] \
|| fail "$path upgrade classifier is incomplete"