Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): require auth key before recovery

Alfredo Lopez33c7b4f6e95a
diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.sh
index 51524f8f7..99f50d8bf 100755
--- a/deploy/scripts/converge-control-db-after-reset.sh
+++ b/deploy/scripts/converge-control-db-after-reset.sh
@@ -234,6 +234,12 @@ env CONTEXT="$CONTEXT" ALLOWED_CONTEXTS="$CONTEXT" \
kc -n control-plane scale deployment/control-api --replicas="$CONTROL_API_REPLICAS" >/dev/null
kc -n control-plane rollout status deployment/control-api --timeout=180s >/dev/null
+# The base gfs-config intentionally carries no real key. Reset recovery is a
+# complete serving restoration boundary, so materialize the canonical platform
+# public key while GFSC remains at zero; missing authority inputs fail closed.
+bash "$ROOT/scripts/minikube/sync-auth-key.sh" \
+ --context "$CONTEXT" --require-gfs
+
GFS_RESTORE_ACTIVE_NOLOGIN=true CONTEXT="$CONTEXT" \
bash "$ROOT/deploy/scripts/reconcile-gfs-deploy-credentials.sh"
diff --git a/docs/deploy/gfs-permission-store.md b/docs/deploy/gfs-permission-store.md
index e7c6309ed..8594d0dfa 100644
--- a/docs/deploy/gfs-permission-store.md
+++ b/docs/deploy/gfs-permission-store.md
@@ -85,6 +85,12 @@ the saved replica count is then restored, creating Pods with the current DSN.
Migration Job Pods intentionally share `app=control-api` for NetworkPolicy
access, so the fence excludes Pods carrying `clerum.io/component`; a completed
migration Pod awaiting TTL cleanup cannot delay runtime writer quiescence.
+After Control API is Ready and while GFSC remains at zero, convergence invokes
+the canonical `sync-auth-key.sh --require-gfs` path. It must copy the non-empty
+`rpc-proxy-secrets.RPC_PROXY_JWT_PUBLIC_KEY` authority into
+`gfs-config.jwt-public-key` before GFS credentials are restored or either GFSC
+deployment is scaled up. Missing source or target resources abort recovery and
+retain the fail-closed replica state; no reset path patches the key ad hoc.
Any later convergence failure reasserts the Control API fence together with the
other database-dependent controllers and waits for the Control API Pods to
terminate before returning.
diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.sh
index 5d991dfd6..d37836469 100755
--- a/scripts/minikube/sync-auth-key.sh
+++ b/scripts/minikube/sync-auth-key.sh
@@ -2,6 +2,7 @@
set -euo pipefail
PROFILE="clerum-test"
+REQUIRE_GFS=false
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -13,8 +14,15 @@ while [[ $# -gt 0 ]]; do
PROFILE="${1#--context=}"
shift
;;
+ --require-gfs)
+ # Reset recovery cannot restore GFSC until the canonical platform key is
+ # materialized in gfs-config. Generic bootstrap callers retain the
+ # existing skip-if-absent behavior.
+ REQUIRE_GFS=true
+ shift
+ ;;
-h|--help)
- echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>]" >&2
+ echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-gfs]" >&2
exit 0
;;
*)
@@ -42,6 +50,10 @@ GFS_TARGET_KEY="jwt-public-key"
GFS_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"
log() { printf '[sync-auth-key] %s\n' "$*"; }
+die() {
+ log "ERROR: $*" >&2
+ exit 1
+}
rollout_restart_with_retry() {
local deployment="$1"
@@ -78,10 +90,20 @@ PY
# The public key is the SAME for every consumer (one platform keypair), so it is
# fetched once and synced into each consumer's ConfigMap below.
if ! "${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" >/dev/null 2>&1; then
+ if [[ "${REQUIRE_GFS}" == "true" ]]; then
+ die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing"
+ fi
log "Skipping auth key sync (${SOURCE_SECRET} not found)"
exit 0
fi
source_key="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}" | base64 -d)"
+if [[ "${REQUIRE_GFS}" == "true" && -z "${source_key}" ]]; then
+ die "required GFS auth source key ${SOURCE_KEY} is empty"
+fi
+if [[ "${REQUIRE_GFS}" == "true" ]] && \
+ ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then
+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing"
+fi
# ── Target 1: mcp-host-config (restart named chatllm/mcp-host with retry) ──────
sync_mcp_host() {
@@ -114,6 +136,9 @@ sync_mcp_host() {
# ── Target 2: gfs-config (gfsc fails closed without the verification key) ──────
sync_gfs() {
if ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then
+ if [[ "${REQUIRE_GFS}" == "true" ]]; then
+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared during sync"
+ fi
log "Skipping ${GFS_CONFIGMAP} sync (not found)"
return 0
fi
diff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.sh
index 552e95982..71f036828 100755
--- a/scripts/tests/test-converge-control-db-after-reset.sh
+++ b/scripts/tests/test-converge-control-db-after-reset.sh
@@ -70,6 +70,7 @@ cat >"$TMP/bin/bash" <<'EOF'
#!/bin/sh
printf 'bash %s\n' "$*" >>"$FAKE_LOG"
case "$*" in
+ *sync-auth-key.sh*) [ "${FAIL_AUTH_SYNC:-false}" != true ] ;;
*reconcile-gfs-deploy-credentials.sh*) [ "${FAIL_RECONCILE:-false}" != true ] ;;
*scripts/minikube/verify-gfs.sh*) [ "${FAIL_VERIFY:-false}" != true ] ;;
*) exit 0 ;;
@@ -102,6 +103,8 @@ grep -q 'wait --for=delete pod -l app=control-api,!clerum.io/component' "$LOG" \
|| { printf 'FAIL: stale control-api pod was not removed before credential reconciliation\n' >&2; exit 1; }
! grep -q 'get pods -l app=control-api -o name' "$LOG" \
|| { printf 'FAIL: completed migration pods were included in the control-api fence\n' >&2; exit 1; }
+grep -q 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" \
+ || { printf 'FAIL: required GFS auth key was not synced before GFSC restoration\n' >&2; exit 1; }
writer_zero="$(grep -n 'scale deployment/gfsc-writer deployment/gfsc-reader --replicas=0' "$LOG" | head -1 | cut -d: -f1)"
hcc_zero="$(grep -n 'scale deployment/host-context-controller --replicas=0' "$LOG" | head -1 | cut -d: -f1)"
hcc_wait="$(grep -n 'wait --for=delete pod -l app=host-context-controller' "$LOG" | cut -d: -f1)"
@@ -111,6 +114,7 @@ control_zero="$(grep -n 'scale deployment/control-api --replicas=0' "$LOG" | hea
control_wait="$(grep -n 'wait --for=delete pod -l app=control-api,!clerum.io/component' "$LOG" | head -1 | cut -d: -f1)"
control_restore="$(grep -n 'scale deployment/control-api --replicas=2' "$LOG" | head -1 | cut -d: -f1)"
control_ready="$(grep -n 'rollout status deployment/control-api --timeout=180s' "$LOG" | head -1 | cut -d: -f1)"
+auth_sync="$(grep -n 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" | head -1 | cut -d: -f1)"
restore_writer="$(grep -n 'scale deployment/gfsc-writer --replicas=3' "$LOG" | cut -d: -f1)"
restore_reader="$(grep -n 'scale deployment/gfsc-reader --replicas=2' "$LOG" | cut -d: -f1)"
reconcile="$(grep -n 'reconcile-gfs-deploy-credentials.sh' "$LOG" | cut -d: -f1)"
@@ -122,7 +126,8 @@ state_delete="$(grep -n 'delete --raw /api/v1/namespaces/control-plane/configmap
[[ "$hcc_zero" -lt "$hcc_wait" && "$hcc_wait" -lt "$writer_zero" && "$writer_zero" -lt "$migration" && \
"$control_zero" -lt "$control_wait" && "$control_wait" -lt "$migration" && \
"$migration" -lt "$runtime_roles" && "$runtime_roles" -lt "$control_restore" && \
- "$control_restore" -lt "$control_ready" && \
+ "$control_restore" -lt "$control_ready" && "$control_ready" -lt "$auth_sync" && \
+ "$auth_sync" -lt "$reconcile" && \
"$reconcile" -lt "$restore_writer" && "$reconcile" -lt "$restore_reader" && \
"$gfs_verify" -lt "$workflow_restore" && "$gfs_verify" -lt "$trace_restore" && \
"$workflow_restore" -lt "$hcc_restore" && "$hcc_restore" -lt "$state_delete" ]] \
@@ -147,6 +152,21 @@ retry_control_restore="$(grep -n 'scale deployment/control-api --replicas=2' "$L
"$retry_runtime_roles" -lt "$retry_control_restore" ]] \
|| { printf 'FAIL: replacement-bound retry reused a stale control-api pod\n' >&2; exit 1; }
+: >"$LOG"
+init_state replacement-bound replacement-uid
+if run_converge env FAIL_AUTH_SYNC=true >/dev/null 2>&1; then
+ printf 'FAIL: missing required GFS auth source was reported as success\n' >&2
+ exit 1
+fi
+grep -q 'sync-auth-key.sh --context test-context --require-gfs' "$LOG" \
+ || { printf 'FAIL: reset convergence did not invoke strict GFS auth sync\n' >&2; exit 1; }
+! grep -q 'reconcile-gfs-deploy-credentials.sh' "$LOG" \
+ || { printf 'FAIL: GFS credentials reconciled after required auth sync failed\n' >&2; exit 1; }
+! grep -q 'scale deployment/gfsc-writer --replicas=3' "$LOG" \
+ || { printf 'FAIL: GFSC writer was restored without its required auth key\n' >&2; exit 1; }
+[[ "$(grep -c 'scale deployment/control-api --replicas=0' "$LOG")" -eq 2 ]] \
+ || { printf 'FAIL: auth sync failure did not preserve the fail-closed writer fence\n' >&2; exit 1; }
+
: >"$LOG"
init_state
if run_converge env FAIL_RECONCILE=true >/dev/null 2>&1; then
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index 43fbaa02f..7a3525759 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -56,7 +56,7 @@ grep -q 'claim.get("namespace") == "control-plane"' deploy/scripts/reset-control
grep -q 'claim.get("uid") == expected_uid' deploy/scripts/reset-control-db-storage.sh || fail 'released PV cleanup is not claim-UID-scoped'
! grep -q 'ensure_pvc "control-postgres-data"' scripts/minikube/ensure-pvcs.sh || fail 'generic PVC reconciler can still delete the control database'
reset_convergence="$(cat deploy/scripts/converge-control-db-after-reset.sh)"
-for required in run-control-api-db-migration.sh provision-control-api-runtime-roles.sh 'scale deployment/control-api --replicas=0' "CONTROL_API_POD_SELECTOR='app=control-api,!clerum.io/component'" 'wait --for=delete pod' 'rollout status deployment/control-api' 'GFS_RESTORE_ACTIVE_NOLOGIN=true' reconcile-gfs-deploy-credentials.sh verify-gfs.sh; do
+for required in run-control-api-db-migration.sh provision-control-api-runtime-roles.sh 'scale deployment/control-api --replicas=0' "CONTROL_API_POD_SELECTOR='app=control-api,!clerum.io/component'" 'wait --for=delete pod' 'rollout status deployment/control-api' 'sync-auth-key.sh' '--require-gfs' 'GFS_RESTORE_ACTIVE_NOLOGIN=true' reconcile-gfs-deploy-credentials.sh verify-gfs.sh; do
[[ "$reset_convergence" == *"$required"* ]] || fail "reset convergence omits $required"
done
migration="$(grep -n 'run-control-api-db-migration.sh' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"
@@ -65,12 +65,14 @@ control_zero="$(grep -n 'scale deployment/control-api --replicas=0' deploy/scrip
control_wait="$(grep -n -- '-l "$CONTROL_API_POD_SELECTOR" --timeout=180s' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"
control_restore="$(grep -n 'scale deployment/control-api --replicas="\$CONTROL_API_REPLICAS"' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"
ready="$(grep -n 'rollout status deployment/control-api' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"
+auth_sync="$(grep -n 'sync-auth-key.sh' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"
restore="$(grep -n 'GFS_RESTORE_ACTIVE_NOLOGIN=true' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"
verify="$(grep -n 'verify-gfs.sh' deploy/scripts/converge-control-db-after-reset.sh | tail -1 | cut -d: -f1)"
success="$(grep -n 'migrations, runtime roles, GFS restore, and verification complete' deploy/scripts/converge-control-db-after-reset.sh | cut -d: -f1)"
[[ "$control_zero" -lt "$control_wait" && "$control_wait" -lt "$migration" && \
"$migration" -lt "$roles" && "$roles" -lt "$control_restore" && \
- "$control_restore" -lt "$ready" && "$ready" -lt "$restore" && \
+ "$control_restore" -lt "$ready" && "$ready" -lt "$auth_sync" && \
+ "$auth_sync" -lt "$restore" && \
"$restore" -lt "$verify" && "$verify" -lt "$success" ]] \
|| fail 'reset convergence ordering is incomplete'
! grep -q '|| true' deploy/scripts/converge-control-db-after-reset.sh || fail 'reset convergence suppresses a critical failure'
diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.sh
index ac40a67be..abeca521e 100755
--- a/scripts/tests/test-minikube-sync-auth-key.sh
+++ b/scripts/tests/test-minikube-sync-auth-key.sh
@@ -48,8 +48,9 @@ restart_count() {
}
if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "rpc-proxy-secrets" ]]; then
+ [[ "${TEST_SOURCE_SECRET_PRESENT:-1}" == "1" ]] || exit 1
if has_output_flag "$@"; then
- printf 'cHVibGljLWtleQ=='
+ [[ "${TEST_SOURCE_KEY_EMPTY:-0}" == "1" ]] || printf 'cHVibGljLWtleQ=='
fi
exit 0
fi
@@ -159,4 +160,64 @@ if grep -q 'rollout restart deployment -l' "${LOG_FILE}"; then
exit 1
fi
+: >"${LOG_FILE}"
+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_SOURCE_SECRET_PRESENT=0 \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: strict GFS auth sync accepted a missing source Secret" >&2
+ exit 1
+fi
+grep -q 'required GFS auth source rpc-proxy/rpc-proxy-secrets is missing' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo "FAIL: strict GFS auth sync did not diagnose its missing source Secret" >&2
+ exit 1
+}
+if grep -Eq '^(patch|rollout restart)' "${LOG_FILE}"; then
+ echo "FAIL: missing required GFS source mutated consumers" >&2
+ cat "${LOG_FILE}" >&2
+ exit 1
+fi
+
+: >"${LOG_FILE}"
+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_SOURCE_KEY_EMPTY=1 \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: strict GFS auth sync accepted an empty source key" >&2
+ exit 1
+fi
+grep -q 'required GFS auth source key RPC_PROXY_JWT_PUBLIC_KEY is empty' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo "FAIL: strict GFS auth sync did not diagnose its empty source key" >&2
+ exit 1
+}
+
+: >"${LOG_FILE}"
+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: strict GFS auth sync accepted a missing gfs-config target" >&2
+ exit 1
+fi
+grep -q 'required GFS auth target gfs/gfs-config is missing' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo "FAIL: strict GFS auth sync did not diagnose its missing target" >&2
+ exit 1
+}
+if grep -Eq '^(patch|rollout restart)' "${LOG_FILE}"; then
+ echo "FAIL: missing required GFS target mutated another auth consumer" >&2
+ cat "${LOG_FILE}" >&2
+ exit 1
+fi
+
+: >"${LOG_FILE}"
+if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFIG_PRESENT=1 \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ cat "${OUT_FILE}" >&2
+ cat "${LOG_FILE}" >&2
+ echo "FAIL: strict GFS auth sync rejected complete authority inputs" >&2
+ exit 1
+fi
+grep -q 'patch configmap gfs-config' "${LOG_FILE}"
+
echo "PASS: auth-key sync retries transient rollout restart race and skips gfsc restart until DSN provisioning"