Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): require auth key before recovery
Alfredo Lopez33c7b4f6e95a
diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.shindex 51524f8f7..99f50d8bf 100755--- a/deploy/scripts/converge-control-db-after-reset.sh+++ b/deploy/scripts/converge-control-db-after-reset.sh@@ -234,6 +234,12 @@ env CONTEXT="$CONTEXT" ALLOWED_CONTEXTS="$CONTEXT" \kc -n control-plane scale deployment/control-api --replicas="$CONTROL_API_REPLICAS" >/dev/nullkc -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null+# The base gfs-config intentionally carries no real key. Reset recovery is a+# complete serving restoration boundary, so materialize the canonical platform+# public key while GFSC remains at zero; missing authority inputs fail closed.+bash "$ROOT/scripts/minikube/sync-auth-key.sh" \+ --context "$CONTEXT" --require-gfs+GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="$CONTEXT" \bash "$ROOT/deploy/scripts/reconcile-gfs-deploy-credentials.sh"diff --git a/docs/deploy/gfs-permission-store.md b/docs/deploy/gfs-permission-store.mdindex e7c6309ed..8594d0dfa 100644--- a/docs/deploy/gfs-permission-store.md+++ b/docs/deploy/gfs-permission-store.md@@ -85,6 +85,12 @@ the saved replica count is then restored, creating Pods with the current DSN.Migration Job Pods intentionally share `app=control-api` for NetworkPolicyaccess, so the fence excludes Pods carrying `clerum.io/component`; a completedmigration Pod awaiting TTL cleanup cannot delay runtime writer quiescence.+After Control API is Ready and while GFSC remains at zero, convergence invokes+the canonical `sync-auth-key.sh --require-gfs` path. It must copy the non-empty+`rpc-proxy-secrets.RPC_PROXY_JWT_PUBLIC_KEY` authority into+`gfs-config.jwt-public-key` before GFS credentials are restored or either GFSC+deployment is scaled up. Missing source or target resources abort recovery and+retain the fail-closed replica state; no reset path patches the key ad hoc.Any later convergence failure reasserts the Control API fence together with theother database-dependent controllers and waits for the Control API Pods toterminate before returning.diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.shindex 5d991dfd6..d37836469 100755--- a/scripts/minikube/sync-auth-key.sh+++ b/scripts/minikube/sync-auth-key.sh@@ -2,6 +2,7 @@set -euo pipefailPROFILE="clerum-test"+REQUIRE_GFS=falsewhile [[ $# -gt 0 ]]; docase "$1" in@@ -13,8 +14,15 @@ while [[ $# -gt 0 ]]; doPROFILE="${1#--context=}"shift;;+ --require-gfs)+ # Reset recovery cannot restore GFSC until the canonical platform key is+ # materialized in gfs-config. Generic bootstrap callers retain the+ # existing skip-if-absent behavior.+ REQUIRE_GFS=true+ shift+ ;;-h|--help)- echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>]" >&2+ echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-gfs]" >&2exit 0;;*)@@ -42,6 +50,10 @@ GFS_TARGET_KEY="jwt-public-key"GFS_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"log() { printf '[sync-auth-key] %s\n' "$*"; }+die() {+ log "ERROR: $*" >&2+ exit 1+}rollout_restart_with_retry() {local deployment="$1"@@ -78,10 +90,20 @@ PY# The public key is the SAME for every consumer (one platform keypair), so it is# fetched once and synced into each consumer's ConfigMap below.if ! "${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" >/dev/null 2>&1; then+ if [[ "${REQUIRE_GFS}" == "true" ]]; then+ die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing"+ filog "Skipping auth key sync (${SOURCE_SECRET} not found)"exit 0fisource_key="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}" | base64 -d)"+if [[ "${REQUIRE_GFS}" == "true" && -z "${source_key}" ]]; then+ die "required GFS auth source key ${SOURCE_KEY} is empty"+fi+if [[ "${REQUIRE_GFS}" == "true" ]] && \+ ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing"+fi# ── Target 1: mcp-host-config (restart named chatllm/mcp-host with retry) ──────sync_mcp_host() {@@ -114,6 +136,9 @@ sync_mcp_host() {# ── Target 2: gfs-config (gfsc fails closed without the verification key) ──────sync_gfs() {if ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then+ if [[ "${REQUIRE_GFS}" == "true" ]]; then+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared during sync"+ filog "Skipping ${GFS_CONFIGMAP} sync (not found)"return 0fidiff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.shindex 552e95982..71f036828 100755--- a/scripts/tests/test-converge-control-db-after-reset.sh+++ b/scripts/tests/test-converge-control-db-after-reset.sh@@ -70,6 +70,7 @@ cat >"$TMP/bin/bash" <<'EOF'#!/bin/shprintf 'bash %s\n' "$*" >>"$FAKE_LOG"case "$*" in+ *sync-auth-key.sh*) [ "${FAIL_AUTH_SYNC:-false}" != true ] ;;*reconcile-gfs-deploy-credentials.sh*) [ "${FAIL_RECONCILE:-false}" != true ] ;;*scripts/minikube/verify-gfs.sh*) [ "${FAIL_VERIFY:-false}" != true ] ;;*) exit 0 ;;@@ -102,6 +103,8 @@ grep -q 'wait --for=delete pod -l app=control-api,!clerum.io/component' "$LOG" \|| { printf 'FAIL: stale control-api pod was not removed before credential reconciliation\n' >&2; exit 1; }! grep -q 'get pods -l app=control-api -o name' "$LOG" \|| { printf 'FAIL: completed migration pods were included in the control-api fence\n' >&2; exit 1; }+grep -q 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" \+ || { printf 'FAIL: required GFS auth key was not synced before GFSC restoration\n' >&2; exit 1; }writer_zero="$(grep -n 'scale deployment/gfsc-writer deployment/gfsc-reader --replicas=0' "$LOG" | head -1 | cut -d: -f1)"hcc_zero="$(grep -n 'scale deployment/host-context-controller --replicas=0' "$LOG" | head -1 | cut -d: -f1)"hcc_wait="$(grep -n 'wait --for=delete pod -l app=host-context-controller' "$LOG" | cut -d: -f1)"@@ -111,6 +114,7 @@ control_zero="$(grep -n 'scale deployment/control-api --replicas=0' "$LOG" | heacontrol_wait="$(grep -n 'wait --for=delete pod -l app=control-api,!clerum.io/component' "$LOG" | head -1 | cut -d: -f1)"control_restore="$(grep -n 'scale deployment/control-api --replicas=2' "$LOG" | head -1 | cut -d: -f1)"control_ready="$(grep -n 'rollout status deployment/control-api --timeout=180s' "$LOG" | head -1 | cut -d: -f1)"+auth_sync="$(grep -n 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" | head -1 | cut -d: -f1)"restore_writer="$(grep -n 'scale deployment/gfsc-writer --replicas=3' "$LOG" | cut -d: -f1)"restore_reader="$(grep -n 'scale deployment/gfsc-reader --replicas=2' "$LOG" | cut -d: -f1)"reconcile="$(grep -n 'reconcile-gfs-deploy-credentials.sh' "$LOG" | cut -d: -f1)"@@ -122,7 +126,8 @@ state_delete="$(grep -n 'delete --raw /api/v1/namespaces/control-plane/configmap[[ "$hcc_zero" -lt "$hcc_wait" && "$hcc_wait" -lt "$writer_zero" && "$writer_zero" -lt "$migration" && \"$control_zero" -lt "$control_wait" && "$control_wait" -lt "$migration" && \"$migration" -lt "$runtime_roles" && "$runtime_roles" -lt "$control_restore" && \- "$control_restore" -lt "$control_ready" && \+ "$control_restore" -lt "$control_ready" && "$control_ready" -lt "$auth_sync" && \+ "$auth_sync" -lt "$reconcile" && \"$reconcile" -lt "$restore_writer" && "$reconcile" -lt "$restore_reader" && \"$gfs_verify" -lt "$workflow_restore" && "$gfs_verify" -lt "$trace_restore" && \"$workflow_restore" -lt "$hcc_restore" && "$hcc_restore" -lt "$state_delete" ]] \@@ -147,6 +152,21 @@ retry_control_restore="$(grep -n 'scale deployment/control-api --replicas=2' "$L"$retry_runtime_roles" -lt "$retry_control_restore" ]] \|| { printf 'FAIL: replacement-bound retry reused a stale control-api pod\n' >&2; exit 1; }+: >"$LOG"+init_state replacement-bound replacement-uid+if run_converge env FAIL_AUTH_SYNC=true >/dev/null 2>&1; then+ printf 'FAIL: missing required GFS auth source was reported as success\n' >&2+ exit 1+fi+grep -q 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" \+ || { printf 'FAIL: reset convergence did not invoke strict GFS auth sync\n' >&2; exit 1; }+! grep -q 'reconcile-gfs-deploy-credentials.sh' "$LOG" \+ || { printf 'FAIL: GFS credentials reconciled after required auth sync failed\n' >&2; exit 1; }+! grep -q 'scale deployment/gfsc-writer --replicas=3' "$LOG" \+ || { printf 'FAIL: GFSC writer was restored without its required auth key\n' >&2; exit 1; }+[[ "$(grep -c 'scale deployment/control-api --replicas=0' "$LOG")" -eq 2 ]] \+ || { printf 'FAIL: auth sync failure did not preserve the fail-closed writer fence\n' >&2; exit 1; }+: >"$LOG"init_stateif run_converge env FAIL_RECONCILE=true >/dev/null 2>&1; thendiff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex 43fbaa02f..7a3525759 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -56,7 +56,7 @@ grep -q 'claim.get("namespace") == "control-plane"' deploy/scripts/reset-controlgrep -q 'claim.get("uid") == expected_uid' deploy/scripts/reset-control-db-storage.sh || fail 'released PV cleanup is not claim-UID-scoped'! grep -q 'ensure_pvc "control-postgres-data"' scripts/minikube/ensure-pvcs.sh || fail 'generic PVC reconciler can still delete the control database'reset_convergence="$(cat deploy/scripts/converge-control-db-after-reset.sh)"-for required in run-control-api-db-migration.sh provision-control-api-runtime-roles.sh 'scale deployment/control-api --replicas=0' "CONTROL_API_POD_SELECTOR='app=control-api,!clerum.io/component'" 'wait --for=delete pod' 'rollout status deployment/control-api' 'GFS_RESTORE_ACTIVE_NOLOGIN=true' reconcile-gfs-deploy-credentials.sh verify-gfs.sh; do+for required in run-control-api-db-migration.sh provision-control-api-runtime-roles.sh 'scale deployment/control-api --replicas=0' "CONTROL_API_POD_SELECTOR='app=control-api,!clerum.io/component'" 'wait --for=delete pod' 'rollout status deployment/control-api' 'sync-auth-key.sh' '--require-gfs' 'GFS_RESTORE_ACTIVE_NOLOGIN=true' reconcile-gfs-deploy-credentials.sh verify-gfs.sh; do[[ "$reset_convergence" == *"$required"* ]] || fail "reset convergence omits $required"donemigration="$(grep -n 'run-control-api-db-migration.sh' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"@@ -65,12 +65,14 @@ control_zero="$(grep -n 'scale deployment/control-api --replicas=0' deploy/scripcontrol_wait="$(grep -n -- '-l "$CONTROL_API_POD_SELECTOR" --timeout=180s' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"control_restore="$(grep -n 'scale deployment/control-api --replicas="\$CONTROL_API_REPLICAS"' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"ready="$(grep -n 'rollout status deployment/control-api' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"+auth_sync="$(grep -n 'sync-auth-key.sh' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"restore="$(grep -n 'GFS_RESTORE_ACTIVE_NOLOGIN=true' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"verify="$(grep -n 'verify-gfs.sh' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"success="$(grep -n 'migrations, runtime roles, GFS restore, and verification complete' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"[[ "$control_zero" -lt "$control_wait" && "$control_wait" -lt "$migration" && \"$migration" -lt "$roles" && "$roles" -lt "$control_restore" && \- "$control_restore" -lt "$ready" && "$ready" -lt "$restore" && \+ "$control_restore" -lt "$ready" && "$ready" -lt "$auth_sync" && \+ "$auth_sync" -lt "$restore" && \"$restore" -lt "$verify" && "$verify" -lt "$success" ]] \|| fail 'reset convergence ordering is incomplete'! grep -q '|| true' deploy/scripts/converge-control-db-after-reset.sh || fail 'reset convergence suppresses a critical failure'diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.shindex ac40a67be..abeca521e 100755--- a/scripts/tests/test-minikube-sync-auth-key.sh+++ b/scripts/tests/test-minikube-sync-auth-key.sh@@ -48,8 +48,9 @@ restart_count() {}if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "rpc-proxy-secrets" ]]; then+ [[ "${TEST_SOURCE_SECRET_PRESENT:-1}" == "1" ]] || exit 1if has_output_flag "$@"; then- printf 'cHVibGljLWtleQ=='+ [[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] || printf 'cHVibGljLWtleQ=='fiexit 0fi@@ -159,4 +160,64 @@ if grep -q 'rollout restart deployment -l' "${LOG_FILE}"; thenexit 1fi+: >"${LOG_FILE}"+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_SOURCE_SECRET_PRESENT=0 \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ echo "FAIL: strict GFS auth sync accepted a missing source Secret" >&2+ exit 1+fi+grep -q 'required GFS auth source rpc-proxy/rpc-proxy-secrets is missing' "${OUT_FILE}" || {+ cat "${OUT_FILE}" >&2+ echo "FAIL: strict GFS auth sync did not diagnose its missing source Secret" >&2+ exit 1+}+if grep -Eq '^(patch|rollout restart)' "${LOG_FILE}"; then+ echo "FAIL: missing required GFS source mutated consumers" >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi++: >"${LOG_FILE}"+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_SOURCE_KEY_EMPTY=1 \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ echo "FAIL: strict GFS auth sync accepted an empty source key" >&2+ exit 1+fi+grep -q 'required GFS auth source key RPC_PROXY_JWT_PUBLIC_KEY is empty' "${OUT_FILE}" || {+ cat "${OUT_FILE}" >&2+ echo "FAIL: strict GFS auth sync did not diagnose its empty source key" >&2+ exit 1+}++: >"${LOG_FILE}"+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ echo "FAIL: strict GFS auth sync accepted a missing gfs-config target" >&2+ exit 1+fi+grep -q 'required GFS auth target gfs/gfs-config is missing' "${OUT_FILE}" || {+ cat "${OUT_FILE}" >&2+ echo "FAIL: strict GFS auth sync did not diagnose its missing target" >&2+ exit 1+}+if grep -Eq '^(patch|rollout restart)' "${LOG_FILE}"; then+ echo "FAIL: missing required GFS target mutated another auth consumer" >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi++: >"${LOG_FILE}"+if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFIG_PRESENT=1 \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo "FAIL: strict GFS auth sync rejected complete authority inputs" >&2+ exit 1+fi+grep -q 'patch configmap gfs-config' "${LOG_FILE}"+echo "PASS: auth-key sync retries transient rollout restart race and skips gfsc restart until DSN provisioning"