Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(minikube): recover partial GFS bootstrap before cutover
Alfredo Lopez70f819a5fcdb
diff --git a/scripts/minikube/filter-gfs-resources.py b/scripts/minikube/filter-gfs-resources.pyindex d56323d3c..f895ca34f 100755--- a/scripts/minikube/filter-gfs-resources.py+++ b/scripts/minikube/filter-gfs-resources.py@@ -1,9 +1,11 @@#!/usr/bin/env python3"""Remove GFS-owned Kubernetes documents from an intentionally non-GFS apply.-This is used only by non-T2 pre-gate deployment syncs. The filter is explicit-about the small set of cluster-scoped and control-plane GFS boundary objects;-unknown documents are retained rather than guessed away.+This is used by non-GFS deployment syncs and by the T2 full-setup recovery path+when an existing GFS writer must remain untouched until control-api/runtime+roles converge. The filter is explicit about the small set of cluster-scoped+and control-plane GFS boundary objects; unknown documents are retained rather+than guessed away.The input is preserved byte-for-byte apart from removed documents. A smallstructural scanner is used instead of a regular expression so documentdiff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex acf0d7022..49a85fa95 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -1100,53 +1100,92 @@ ok "Minikube K8s API CIDRs refreshed"# Upgrade path: stage the additive reader credential before the full overlay# changes HCC. Fresh bootstrap has no ready control-api yet and remains# fail-closed until the post-migration branch immediately below.+#+# A partially bootstrapped REUSE_DB profile is a distinct state from both a+# fresh bootstrap and a healthy upgrade: an existing GFS writer proves that the+# profile already owns a serving plane, while an unready control-api can mean+# that the runtime-role Secret is still waiting for the migration below. Do not+# cut HCC over in that state, but do not deadlock recovery on the same readiness+# condition either. Keep the GFS-owned documents untouched, apply only the+# explicit non-GFS portion of the overlay, then complete migrations/roles before+# the full overlay and GFS credential reconciliation.+GFS_OVERLAY_DEFERRED=false++control_api_is_ready() {+ local state replicas ready+ if ! state="$($KC -n control-plane get deployment/control-api \+ -o 'jsonpath={.spec.replicas}{"|"}{.status.readyReplicas}' 2>/dev/null)"; then+ return 2+ fi+ IFS='|' read -r replicas ready <<<"$state"+ ready="${ready:-0}"+ if [[ ! "$replicas" =~ ^[1-9][0-9]*$ || ! "$ready" =~ ^[0-9]+$ ]]; then+ return 2+ fi+ [[ "$ready" == "$replicas" ]]+}++apply_non_gfs_overlay() {+ local filtered_manifest+ filtered_manifest="$(mktemp "${TMPDIR:-/tmp}/evenfire-gfs-recovery.XXXXXX")"+ if ! $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" \+ | python3 "${PROJECT_DIR}/scripts/minikube/filter-gfs-resources.py" >"$filtered_manifest"; then+ rm -f -- "$filtered_manifest"+ err "Failed to render or filter the non-GFS overlay during recovery"+ return 1+ fi+ if [ -s "$filtered_manifest" ]; then+ if ! $KC apply -f "$filtered_manifest"; then+ rm -f -- "$filtered_manifest"+ err "Failed to apply the non-GFS overlay during recovery"+ return 1+ fi+ else+ log "Filtered recovery overlay contains no non-GFS resources; skipping apply"+ fi+ rm -f -- "$filtered_manifest"+}++reconcile_existing_gfs_credentials() {+ # A staged sync may have deferred active-pod proof until the reconciler+ # created the reader DSN. Complete that proof before the overlay changes HCC.+ GFS_AUTH_SYNC_ALLOW_STAGED=true \+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+ # If gfsc-reader is already Ready, settle the leftover claim first so+ # reconcile does not rollout restart and race HCC's gfsReconciler. The+ # gfs-rollout-shim makes the wait judge readiness rather than the template+ # generation HCC rewrites.+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \+ T2_SKIP_LOCK=true \+ bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"+ PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \+ GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \+ T2_SKIP_LOCK=true \+ CONTEXT="${PROFILE}" \+ bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"+ # The staged sync may have deferred active-pod proof until this reconcile+ # produced the reader DSN; the strict pass is the durable consumer attestation.+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+}+CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"if [ "$RESET_DB" = true ]; thenlog "Database reset path — HCC cutover deferred until post-convergence verification"elsewriter_dsn="$($KC -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"if [ -n "${writer_dsn}" ]; then- # apply-inter-service-tokens.sh intentionally restarts control-api before- # this cutover check. A five-second probe races that rollout and makes an- # otherwise healthy REUSE_DB rerun fail closed before the new pod can be- # Ready. Keep the refusal, but wait for the same bounded rollout window- # used by the main readiness gate.- if ! $KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null 2>&1; then- err "Existing GFS writer detected but control-api is not Ready; refusing HCC cutover"- exit 1+ if control_api_is_ready; then+ log "Existing GFS writer detected — reconciling credentials before overlay upgrade"+ reconcile_existing_gfs_credentials+ else+ control_api_ready_status=$?+ if [ "$control_api_ready_status" -ne 1 ]; then+ err "Unable to determine control-api readiness; refusing to classify the GFS upgrade path"+ exit 1+ fi+ GFS_OVERLAY_DEFERRED=true+ log "Existing GFS writer detected but control-api is not Ready — deferring GFS-owned overlay and credential cutover until runtime roles converge"fi- log "Existing GFS writer detected — reconciling credentials before overlay upgrade"- # REUSE_DB / T2 full-reconcile reaches this before pre-gate-sync. A prior T1- # can leave gfs_controller_reader NOLOGIN; restore from the committed- # Secret DSN (same opt-in as the GFS T1 gate). Still fail-loud on auth.- # GFS_RECOVER_ABANDONED_STATE: a timed-out prior setup can leave the- # reader Secret in rollout-running. This path holds the T2 profile lock,- # so the prior process is dead; resume that claim instead of asking for- # a manual retry flag between runs.- # A reader pod resolves gfs-config.jwt-public-key at container start and- # fails closed when it is empty (a prior interrupted run can leave it- # wiped by the overlay re-apply). Re-sync it from the live platform- # Secret before reconcile so a reader pod can actually start; this is a- # no-op when the key already matches.- GFS_AUTH_SYNC_ALLOW_STAGED=true \- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs- # If gfsc-reader is already Ready, settle the leftover claim first so- # reconcile does not rollout restart and race HCC's gfsReconciler.- # The gfs-rollout-shim PATH prefix makes any reader rollout wait inside- # reconcile judge readiness instead of the template generation HCC keeps- # rewriting (it strips the restartedAt annotation, so a generation wait- # loops until timeout).- GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \- T2_SKIP_LOCK=true \- bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"- PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \- GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \- T2_SKIP_LOCK=true \- CONTEXT="${PROFILE}" \- bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"- # A staged sync may have deferred active-pod proof until the reconciler- # created the reader DSN. Complete that proof before the overlay is applied.- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfselselog "Fresh bootstrap detected — reader staging deferred until migrations; GFSC remains fail-closed"fi@@ -1157,10 +1196,13 @@ if [ "$RESET_DB" = true ]; then# Keep control-api scaled to zero until migrations and role restoration are# complete; applying the full overlay here would race it against a fresh DB.$KC apply -k "$ACTIVE_MINIKUBE_RENDER_DIR" -l app=control-postgres+elif [ "$GFS_OVERLAY_DEFERRED" = true ]; then+ log "Applying the non-GFS recovery overlay before database migration..."+ apply_non_gfs_overlayelse- kubectl kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -+ $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -fi-ok "Kustomize overlay applied"+ok "Kustomize overlay applied (full or non-GFS recovery phase)"if [ "$RESET_DB" = true ]; thenlog "Rebuilding database contracts and restoring GFS roles after reset..."@@ -1168,7 +1210,7 @@ if [ "$RESET_DB" = true ]; thenCONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/converge-control-db-after-reset.sh" \--overlay "$ACTIVE_MINIKUBE_RENDER_DIR" \--job-name control-api-db-migrate-reset- kubectl kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -+ $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -# The overlay declares ordinary one-replica defaults. Reassert the exact# pre-reset operating counts after applying it; credentials are verified and# HCC is restored by convergence before this safe rollout boundary.@@ -1198,33 +1240,43 @@ else--overlay "$ACTIVE_MINIKUBE_RENDER_DIR"CONTEXT="${PROFILE}" ALLOWED_CONTEXTS="${PROFILE}" \bash "${PROJECT_DIR}/deploy/scripts/provision-control-api-runtime-roles.sh"- ok "Control-api database migrations and runtime roles applied"-- # The credential probe runs through control-api, so prove that deployment is- # live before normal staging on bootstrap or an idempotent upgrade.- $KC scale deployment/control-api -n control-plane --replicas=1 >/dev/null- $KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null- # On the upgrade path the overlay above may still be cutting HCC over to- # the split writer/reader templates; reconciling before that lands leaves- # the staged reader credential rollout-pending and fails the final verify.- CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/wait-gfsc-secret-references.sh"- # The overlay apply above re-declared the base gfs-config with an empty- # jwt-public-key; re-sync it before any reader pod may need to start,- # otherwise the reconcile readiness wait can only time out.- GFS_AUTH_SYNC_ALLOW_STAGED=true \- bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs- GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \- T2_SKIP_LOCK=true \- bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"- PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \- GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \- T2_SKIP_LOCK=true \- CONTEXT="${PROFILE}" \- bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"- # The preceding staged sync is allowed to wait for this DSN-producing- # reconcile; the strict pass is the durable consumer attestation.+ ok "Control-api database migrations and runtime roles applied"++ # The credential probe runs through control-api, so prove that deployment is+ # live before normal staging on bootstrap or an idempotent upgrade. On the+ # deferred recovery path this is also the point where the missing runtime+ # Secret has been provisioned, so the previously blocked deployment can start.+ $KC scale deployment/control-api -n control-plane --replicas=1 >/dev/null+ $KC rollout status deployment/control-api -n control-plane --timeout=180s >/dev/null++ if [ "$GFS_OVERLAY_DEFERRED" = true ]; then+ log "Control-api/runtime roles are Ready — reconciling GFS before the deferred full overlay"+ reconcile_existing_gfs_credentials+ $KC kustomize "$ACTIVE_MINIKUBE_RENDER_DIR" | $KC apply -f -+ ok "Deferred full kustomize overlay applied after control-api/runtime role readiness"+ fi++ # On the upgrade path the full overlay may still cut HCC over to the+ # split writer/reader templates; reconciling after that lands leaves the+ # staged reader credential rollout-pending and fails the final verify.+ CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/wait-gfsc-secret-references.sh"+ # The overlay apply re-declares the base gfs-config with an empty jwt-public-key;+ # re-sync it before any reader pod may need to start, otherwise the reconcile+ # readiness wait can only time out.+ GFS_AUTH_SYNC_ALLOW_STAGED=true \bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs- ok "GFS credentials reconciled and writer bootstrap verified"+ GFS_READER_ROLLOUT_AUTHORIZED=true ALLOWED_CONTEXTS="${PROFILE}" CONTEXT="${PROFILE}" \+ T2_SKIP_LOCK=true \+ bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"+ PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \+ GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \+ T2_SKIP_LOCK=true \+ CONTEXT="${PROFILE}" \+ bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"+ # The preceding staged sync is allowed to wait for this DSN-producing+ # reconcile; the strict pass is the durable consumer attestation.+ bash "${SCRIPT_DIR}/sync-auth-key.sh" --context="${PROFILE}" --require-gfs+ ok "GFS credentials reconciled and writer bootstrap verified"fi# 6c. Re-apply generated service tokens after kustomize.diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex e065837e6..b126c8c38 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -167,11 +167,32 @@ assert_minikube_upgrade_classifier() {reconcile="$(grep -n 'reconcile-gfs-deploy-credentials.sh' <<<"$block" | head -1 | cut -d: -f1)"fallback="$(grep -nE '^[[:space:]]*else' <<<"$block" | tail -1 | cut -d: -f1)"fresh="$(grep -ni 'fresh bootstrap' <<<"$block" | tail -1 | cut -d: -f1)"- [[ -n "$dsn" && -n "$classify" && -n "$ready" && -n "$abort" && -n "$reconcile" && -n "$fallback" && -n "$fresh" ]] \- || fail "$path upgrade classifier is incomplete"- [[ "$dsn" -lt "$classify" && "$classify" -lt "$ready" && "$ready" -lt "$abort" && \- "$abort" -lt "$reconcile" && "$reconcile" -lt "$fallback" && "$fallback" -lt "$fresh" ]] \- || fail "$path does not reconcile ready upgrades, abort unready upgrades, and defer only empty bootstraps"+ if [[ "$path" == *"scripts/minikube/full-setup.sh" ]]; then+ local readiness_probe deferred non_gfs_filter deferred_full_overlay post_ready post_reconcile+ readiness_probe="$(grep -n 'if control_api_is_ready' <<<"$block" | head -1 | cut -d: -f1)"+ deferred="$(grep -n 'GFS_OVERLAY_DEFERRED=true' <<<"$block" | head -1 | cut -d: -f1)"+ non_gfs_filter="$(grep -n 'filter-gfs-resources.py' "$path" | head -1 | cut -d: -f1)"+ deferred_full_overlay="$(grep -n 'Deferred full kustomize overlay applied' "$path" | head -1 | cut -d: -f1)"+ post_ready="$(grep -n 'rollout status deployment/control-api.*timeout=180s' "$path" | tail -1 | cut -d: -f1)"+ post_reconcile="$(grep -nE '^[[:space:]]+reconcile_existing_gfs_credentials$' "$path" | tail -1 | cut -d: -f1)"+ [[ -n "$dsn" && -n "$classify" && -n "$readiness_probe" && -n "$deferred" && \+ -n "$fallback" && -n "$fresh" && -n "$non_gfs_filter" && \+ -n "$deferred_full_overlay" && -n "$post_ready" && -n "$post_reconcile" ]] \+ || fail "$path recovery classifier is incomplete"+ [[ "$dsn" -lt "$classify" && "$classify" -lt "$readiness_probe" && \+ "$readiness_probe" -lt "$deferred" && "$deferred" -lt "$fallback" && \+ "$fallback" -lt "$fresh" && "$non_gfs_filter" -lt "$deferred_full_overlay" && \+ "$post_ready" -lt "$deferred_full_overlay" ]] \+ || fail "$path does not defer only the GFS-owned overlay until runtime-role readiness"+ ! grep -Fq 'Existing GFS writer detected but control-api is not Ready; refusing HCC cutover' "$path" \+ || fail "$path still deadlocks an unready partial bootstrap before runtime-role provisioning"+ else+ [[ -n "$dsn" && -n "$classify" && -n "$ready" && -n "$abort" && -n "$reconcile" && -n "$fallback" && -n "$fresh" ]] \+ || fail "$path upgrade classifier is incomplete"+ [[ "$dsn" -lt "$classify" && "$classify" -lt "$ready" && "$ready" -lt "$abort" && \+ "$abort" -lt "$reconcile" && "$reconcile" -lt "$fallback" && "$fallback" -lt "$fresh" ]] \+ || fail "$path does not reconcile ready upgrades, abort unready upgrades, and defer only empty bootstraps"+ fi}assert_pre_gate_defers_gfs_reconcile() {local block@@ -201,7 +222,7 @@ assert_pre_gate_defers_gfs_reconcile 'apply-gfs-writer-secret.sh' 'incremental_bfull_setup_classifier="$(sed -n '/# Upgrade path: stage the additive reader/,/Applying kustomize overlay/p' scripts/minikube/full-setup.sh)"reset_classifier="$(grep -n 'if \[ "$RESET_DB" = true \]; then' <<<"$full_setup_classifier" | head -1 | cut -d: -f1)"reset_classifier_defer="$(grep -n 'HCC cutover deferred until post-convergence verification' <<<"$full_setup_classifier" | head -1 | cut -d: -f1)"-reset_classifier_else="$(grep -n '^else$' <<<"$full_setup_classifier" | head -1 | cut -d: -f1)"+reset_classifier_else="$(awk -v start="$reset_classifier" 'NR > start && /^else$/ { print NR; exit }' <<<"$full_setup_classifier")"reset_classifier_dsn="$(grep -n 'writer_dsn=.*get secret gfs-controller-db' <<<"$full_setup_classifier" | head -1 | cut -d: -f1)"[[ -n "$reset_classifier" && -n "$reset_classifier_defer" && -n "$reset_classifier_else" && -n "$reset_classifier_dsn" ]] && \[[ "$reset_classifier" -lt "$reset_classifier_defer" && "$reset_classifier_defer" -lt "$reset_classifier_else" && \@@ -211,7 +232,7 @@ full_setup_reset="$(sed -n '/# 6a. Optional DB reset/,/# 6c. Re-apply generatedreset_defer="$(grep -n 'HCC cutover deferred until post-convergence verification' <<<"$full_setup_reset" | head -1 | cut -d: -f1)"reset_postgres_only="$(grep -n 'apply -k .* -l app=control-postgres' <<<"$full_setup_reset" | head -1 | cut -d: -f1)"reset_converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$full_setup_reset" | head -1 | cut -d: -f1)"-reset_full_overlay="$(grep -n 'kubectl kustomize .* | .* apply -f -' <<<"$full_setup_reset" | tail -1 | cut -d: -f1)"+reset_full_overlay="$(grep -nE '(kubectl|\$KC) kustomize .* \| .* apply -f -' <<<"$full_setup_reset" | tail -1 | cut -d: -f1)"[[ -n "$reset_defer" && -n "$reset_postgres_only" && -n "$reset_converge" && -n "$reset_full_overlay" ]] && \[[ "$reset_defer" -lt "$reset_postgres_only" && "$reset_postgres_only" -lt "$reset_converge" && "$reset_converge" -lt "$reset_full_overlay" ]] \|| fail 'full-setup reset can require ready control-api or cut over HCC before convergence'