Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(auth): reconcile JWT consumers by runtime contract
Alfredo Lopeze13168f10a89
diff --git a/scripts/minikube/discover-configmap-key-consumers.py b/scripts/minikube/discover-configmap-key-consumers.pynew file mode 100755index 000000000..458e5f136--- /dev/null+++ b/scripts/minikube/discover-configmap-key-consumers.py@@ -0,0 +1,213 @@+#!/usr/bin/env python3+"""Discover effective Deployment env bindings to one ConfigMap key.++The DeploymentList is read from stdin. Each binding is emitted as one record+whose fields are separated by ASCII unit separator (0x1f): namespace,+Deployment, desired replicas, exact label selector, container, and effective+environment-variable name. Kubernetes names and label selectors cannot+contain that delimiter, so the shell caller can preserve empty fields without+inventing an escaping convention.+"""++from __future__ import annotations++import argparse+import json+import sys+from typing import Any+++FIELD_SEPARATOR = "\x1f"+++class ConsumerContractError(ValueError):+ """Raised when an active consumer cannot be described safely."""+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser()+ parser.add_argument("--config-map", required=True)+ parser.add_argument("--key", required=True)+ parser.add_argument("--namespace", required=True)+ return parser.parse_args()+++def mapping(value: Any, field: str) -> dict[str, Any]:+ if value is None:+ return {}+ if not isinstance(value, dict):+ raise ConsumerContractError(f"{field} is not an object")+ return value+++def sequence(value: Any, field: str) -> list[Any]:+ if value is None:+ return []+ if not isinstance(value, list):+ raise ConsumerContractError(f"{field} is not an array")+ return value+++def text(value: Any, field: str, *, allow_empty: bool = False) -> str:+ if not isinstance(value, str) or (not allow_empty and not value):+ raise ConsumerContractError(f"{field} is not a valid string")+ if any(character in value for character in (FIELD_SEPARATOR, "\n", "\r")):+ raise ConsumerContractError(f"{field} contains an unsupported control character")+ return value+++def references_target(env: dict[str, Any], config_map: str, key: str) -> bool:+ value_from = mapping(env.get("valueFrom"), "env.valueFrom")+ key_ref = mapping(value_from.get("configMapKeyRef"), "env.valueFrom.configMapKeyRef")+ return key_ref.get("name") == config_map and key_ref.get("key") == key+++def effective_bindings(+ container: dict[str, Any], config_map: str, key: str+) -> list[tuple[str, str]]:+ container_name = text(container.get("name"), "container.name")+ env = sequence(container.get("env"), f"container {container_name}.env")+ env_from = sequence(container.get("envFrom"), f"container {container_name}.envFrom")++ # Explicit env entries override values imported through envFrom. Keep the+ # last definition for defensive parity with the runtime environment build;+ # valid Kubernetes templates normally de-duplicate this merge-keyed list.+ explicit_by_name: dict[str, dict[str, Any]] = {}+ for index, raw_env in enumerate(env):+ explicit = mapping(raw_env, f"container {container_name}.env[{index}]")+ env_name = text(explicit.get("name"), f"container {container_name}.env[{index}].name")+ explicit_by_name[env_name] = explicit++ names: set[str] = set()+ for index, raw_source in enumerate(env_from):+ source = mapping(raw_source, f"container {container_name}.envFrom[{index}]")+ config_ref = mapping(+ source.get("configMapRef"),+ f"container {container_name}.envFrom[{index}].configMapRef",+ )+ if config_ref.get("name") != config_map:+ continue+ prefix = text(+ source.get("prefix", ""),+ f"container {container_name}.envFrom[{index}].prefix",+ allow_empty=True,+ )+ effective_name = prefix + key+ override = explicit_by_name.get(effective_name)+ if override is None or references_target(override, config_map, key):+ names.add(effective_name)++ for env_name, explicit in explicit_by_name.items():+ if references_target(explicit, config_map, key):+ names.add(env_name)++ return [(container_name, env_name) for env_name in sorted(names)]+++def render_selector(deployment: dict[str, Any]) -> str:+ spec = mapping(deployment.get("spec"), "deployment.spec")+ selector = mapping(spec.get("selector"), "deployment.spec.selector")+ match_labels = mapping(selector.get("matchLabels"), "deployment.spec.selector.matchLabels")+ match_expressions = sequence(+ selector.get("matchExpressions"), "deployment.spec.selector.matchExpressions"+ )+ terms: list[str] = []+ for raw_key in sorted(match_labels):+ key = text(raw_key, "deployment selector label key")+ value = text(match_labels[raw_key], f"deployment selector label {key}", allow_empty=True)+ terms.append(f"{key}={value}")++ for index, raw_expression in enumerate(match_expressions):+ expression = mapping(raw_expression, f"deployment selector expression[{index}]")+ key = text(expression.get("key"), f"deployment selector expression[{index}].key")+ operator = expression.get("operator")+ values = [+ text(value, f"deployment selector expression[{index}].values")+ for value in sequence(+ expression.get("values"), f"deployment selector expression[{index}].values"+ )+ ]+ if operator == "In" and values:+ terms.append(f"{key} in ({','.join(values)})")+ elif operator == "NotIn" and values:+ terms.append(f"{key} notin ({','.join(values)})")+ elif operator == "Exists" and not values:+ terms.append(key)+ elif operator == "DoesNotExist" and not values:+ terms.append(f"!{key}")+ else:+ raise ConsumerContractError(+ f"deployment selector expression[{index}] has unsupported operator/value shape"+ )++ if not terms:+ raise ConsumerContractError("active consumer Deployment has no usable pod selector")+ return ",".join(terms)+++def desired_replicas(deployment: dict[str, Any]) -> int:+ spec = mapping(deployment.get("spec"), "deployment.spec")+ replicas = spec.get("replicas", 1)+ if isinstance(replicas, bool) or not isinstance(replicas, int) or replicas < 0:+ raise ConsumerContractError("deployment.spec.replicas is not a non-negative integer")+ return replicas+++def records(payload: dict[str, Any], args: argparse.Namespace) -> list[tuple[str, ...]]:+ items = sequence(payload.get("items"), "DeploymentList.items")+ discovered: list[tuple[str, ...]] = []+ for index, raw_deployment in enumerate(items):+ deployment = mapping(raw_deployment, f"DeploymentList.items[{index}]")+ metadata = mapping(deployment.get("metadata"), "deployment.metadata")+ name = text(metadata.get("name"), "deployment.metadata.name")+ namespace = text(+ metadata.get("namespace", args.namespace), "deployment.metadata.namespace"+ )+ if namespace != args.namespace:+ raise ConsumerContractError(+ f"deployment {namespace}/{name} escaped namespace-scoped discovery"+ )+ spec = mapping(deployment.get("spec"), f"deployment {namespace}/{name}.spec")+ template = mapping(spec.get("template"), f"deployment {namespace}/{name}.spec.template")+ pod_spec = mapping(+ template.get("spec"), f"deployment {namespace}/{name}.spec.template.spec"+ )+ containers = sequence(+ pod_spec.get("containers"),+ f"deployment {namespace}/{name}.spec.template.spec.containers",+ )+ bindings: list[tuple[str, str]] = []+ for raw_container in containers:+ container = mapping(raw_container, f"deployment {namespace}/{name} container")+ bindings.extend(effective_bindings(container, args.config_map, args.key))+ bindings = sorted(set(bindings))+ if not bindings:+ continue++ replicas = desired_replicas(deployment)+ # A scaled-to-zero consumer has no active process to select or prove.+ # Do not let a selector shape block ConfigMap convergence for its next pod.+ selector = "" if replicas == 0 else render_selector(deployment)+ for container_name, env_name in bindings:+ discovered.append(+ (namespace, name, str(replicas), selector, container_name, env_name)+ )+ return sorted(discovered)+++def main() -> int:+ args = parse_args()+ try:+ payload = json.load(sys.stdin)+ if not isinstance(payload, dict):+ raise ConsumerContractError("DeploymentList is not an object")+ for record in records(payload, args):+ print(FIELD_SEPARATOR.join(record))+ except (ConsumerContractError, json.JSONDecodeError) as error:+ print(f"cannot discover ConfigMap key consumers: {error}", file=sys.stderr)+ return 1+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.shindex 221f97f5b..22f1c282c 100755--- a/scripts/minikube/sync-auth-key.sh+++ b/scripts/minikube/sync-auth-key.sh@@ -131,12 +131,12 @@ CONFIGMAP="mcp-host-config"CONFIG_NAMESPACE="mcp-host"SOURCE_KEY="RPC_PROXY_JWT_PUBLIC_KEY"TARGET_KEY="CLERUM_AUTH_JWT_PUBLIC_KEY"-MCP_HOST_ROLLOUT_TIMEOUT="180s"+MCP_CONSUMER_ROLLOUT_TIMEOUT="180s"# This ConfigMap annotation is the durable commit record for a completed# rollout. It is written only after every active consumer proves its in-process# environment contains the source key; matching data without this hash resumes.APPLIED_HASH_ANNOTATION="clerum.io/auth-key-applied-sha256"-MCP_DEPLOY_SELECTOR="clerum.io/managed-by=host-context-controller"+MCP_BINDING_DISCOVERY="${ROOT}/scripts/minikube/discover-configmap-key-consumers.py"# gfsc verifies gfs access tokens with the SAME platform public key (open core# reuses one keypair). gfsc fails closed without it (empty key ⇒ no read serving),@@ -270,6 +270,20 @@ deployment_desired_replicas() {printf '%s' "${desired}"}+discover_mcp_consumer_bindings() {+ local deployments_json bindings+ if ! deployments_json="$("${KCTL[@]}" get deployments -n "${CONFIG_NAMESPACE}" -o json 2>&1)"; then+ log "cannot enumerate Deployment pod templates in ${CONFIG_NAMESPACE}: ${deployments_json}" >&2+ return 1+ fi+ if ! bindings="$(printf '%s' "${deployments_json}" | python3 "${MCP_BINDING_DISCOVERY}" \+ --namespace "${CONFIG_NAMESPACE}" --config-map "${CONFIGMAP}" --key "${TARGET_KEY}" 2>&1)"; then+ log "cannot derive ${CONFIGMAP}.${TARGET_KEY} consumer bindings: ${bindings}" >&2+ return 1+ fi+ printf '%s' "${bindings}"+}+write_source_key() {SOURCE_KEY_B64="${source_key_b64}" python3 - <<'PY'import base64@@ -323,9 +337,14 @@ print("%s\t%s\t%s" % (PY}-consumer_pods_use_key() {- local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"- local rows pod ready deleting pod_count=0+deployment_pods_use_key_bindings() {+ local namespace="$1" selector="$2" bindings="$3" consumer="$4"+ local rows pod ready deleting container env_name pod_count=0 binding_count=0++ [[ -n "${selector}" && -n "${bindings}" ]] || {+ log "${consumer} has no provable selector or environment binding" >&2+ return 1+ }if ! rows="$("${KCTL[@]}" get pods -n "${namespace}" -l "${selector}" -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' 2>&1)"; then@@ -341,18 +360,35 @@ consumer_pods_use_key() {return 1fipod_count=$((pod_count + 1))- if ! write_source_key | "${KCTL[@]}" exec -i "${pod}" -n "${namespace}" -c "${container}" -- \- node -e 'const fs=require("fs");const expected=fs.readFileSync(0,"utf8");process.exit(process.env[process.argv[1]]===expected?0:42)' \- "${env_name}" >/dev/null; then- log "${consumer} pod ${namespace}/${pod} has not consumed the target auth key" >&2- return 1- fi+ while IFS=$'\x1f' read -r container env_name; do+ [[ -n "${container}" && -n "${env_name}" ]] || {+ log "${consumer} contains an incomplete environment binding" >&2+ return 1+ }+ binding_count=$((binding_count + 1))+ if ! write_source_key | "${KCTL[@]}" exec -i "${pod}" -n "${namespace}" -c "${container}" -- \+ node -e 'const fs=require("fs");const expected=fs.readFileSync(0,"utf8");process.exit(process.env[process.argv[1]]===expected?0:42)' \+ "${env_name}" >/dev/null; then+ log "${consumer} pod ${namespace}/${pod} binding ${container}/${env_name} has not consumed the target auth key" >&2+ return 1+ fi+ done <<<"${bindings}"done <<<"${rows}"if [[ "${pod_count}" -eq 0 ]]; thenlog "${consumer} has a deployment but no active pod proved consumption of the target auth key" >&2return 1fi+ if [[ "${binding_count}" -eq 0 ]]; then+ log "${consumer} has no environment binding that proved consumption of the target auth key" >&2+ return 1+ fi+}++consumer_pods_use_key() {+ local namespace="$1" selector="$2" container="$3" env_name="$4" consumer="$5"+ local binding="${container}"$'\x1f'"${env_name}"+ deployment_pods_use_key_bindings "${namespace}" "${selector}" "${binding}" "${consumer}"}verify_consumer_pods_use_key() {@@ -362,6 +398,35 @@ verify_consumer_pods_use_key() {fi}+reconcile_mcp_consumer_deployment() {+ local namespace="$1" deployment="$2" desired_replicas="$3" selector="$4" bindings="$5"+ local consumer="auth-key consumer deployment/${deployment}" rollout_output++ if [[ "${desired_replicas}" == "0" ]]; then+ log "Skipping suspended ${namespace}/${deployment}; desired replicas=0"+ return 0+ fi+ [[ "${desired_replicas}" =~ ^[0-9]+$ ]] ||+ die "cannot determine desired replicas for ${namespace}/deployment/${deployment}"++ if deployment_pods_use_key_bindings "${namespace}" "${selector}" "${bindings}" "${consumer}"; then+ log "${namespace}/${deployment} already consumes the target auth key"+ return 0+ fi++ log "Restarting ${namespace}/${deployment} after auth key drift"+ if ! rollout_restart_with_retry "${namespace}" "${deployment}" >/dev/null; then+ die "cannot restart ${namespace}/deployment/${deployment} after auth key drift"+ fi+ if ! rollout_output="$("${KCTL[@]}" rollout status "deployment/${deployment}" -n "${namespace}" \+ --timeout="${MCP_CONSUMER_ROLLOUT_TIMEOUT}" 2>&1)"; then+ die "${namespace}/deployment/${deployment} did not become Ready after auth key drift: ${rollout_output}"+ fi+ if ! deployment_pods_use_key_bindings "${namespace}" "${selector}" "${bindings}" "${consumer}"; then+ die "${consumer} did not prove all target auth-key bindings after rollout"+ fi+}+# The public key is the SAME for every consumer (one platform keypair), so it is# fetched once and synced into each consumer's ConfigMap below.source_probe=""@@ -397,8 +462,8 @@ if [[ "${REQUIRE_MCP}" == "true" ]] && \die "required MCP auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} is missing or unreadable: ${mcp_target_probe}"fi-# ── Target 1: mcp-host-config (restart named chatllm/mcp-host with retry) ──────-sync_mcp_host() {+# ── Target 1: mcp-host-config (discover and prove effective pod bindings) ─────+sync_mcp_consumers() {local target_probe=""if ! target_probe="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" 2>&1)"; thenif [[ "${REQUIRE_MCP}" == "true" ]]; then@@ -417,11 +482,9 @@ sync_mcp_host() {if ! applied_hash="$(read_applied_hash "${CONFIG_NAMESPACE}" "${CONFIGMAP}" 2>&1)"; thendie "cannot read auth convergence marker from ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${applied_hash}"fi- local force_rollout=falseif [[ "${source_hash}" != "${current_hash}" ]]; thenlog "Patching ${CONFIGMAP}.${TARGET_KEY}""${KCTL[@]}" patch configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" --type=merge -p "$(make_patch "${source_key_b64}" "${TARGET_KEY}")" >/dev/null- force_rollout=trueelseif [[ "${applied_hash}" == "${source_hash}" ]]; thenlog "${CONFIGMAP}.${TARGET_KEY} matches source; checking active consumers before declaring convergence"@@ -429,48 +492,60 @@ sync_mcp_host() {log "${CONFIGMAP}.${TARGET_KEY} matches source but consumer attestation is pending; resuming convergence"fifi- local deployment_resources deployment resource_name desired_replicas- local deployments=()- if ! deployment_resources="$("${KCTL[@]}" get deployment -l "${MCP_DEPLOY_SELECTOR}" \- -n "${CONFIG_NAMESPACE}" -o name 2>&1)"; then- die "cannot enumerate mcp-host consumers before auth key rollout: ${deployment_resources}"- fi- while IFS= read -r resource_name; do- [[ -n "${resource_name}" ]] || continue- case "${resource_name}" in- deployment/*|deployment.apps/*) deployment="${resource_name#*/}" ;;- *) die "unexpected mcp-host consumer resource from Kubernetes: ${resource_name}" ;;- esac- [[ -n "${deployment}" ]] || die "Kubernetes returned an empty mcp-host deployment name"- deployments+=("${deployment}")- done <<<"${deployment_resources}"- if (( ${#deployments[@]} == 0 )); then- log "No mcp-host deployments exist yet; auth key is synced for future pods"+ local binding_records+ if ! binding_records="$(discover_mcp_consumer_bindings)"; then+ die "cannot enumerate effective ${CONFIGMAP}.${TARGET_KEY} consumers before auth key rollout"+ fi+ if [[ -z "${binding_records}" ]]; then+ log "No Deployment pod templates reference ${CONFIGMAP}.${TARGET_KEY}; auth key is synced for future consumers"else- for deployment in "${deployments[@]}"; do- desired_replicas="$(deployment_desired_replicas "${CONFIG_NAMESPACE}" "${deployment}")" ||- die "cannot determine desired replicas for mcp-host deployment/${deployment}"- if [[ "${desired_replicas}" == "0" ]]; then- # HCC intentionally suspends stateless Hosts by scaling their managed- # Deployment to zero. There is no consumer to restart or prove; the- # patched ConfigMap is the authoritative input for the next pod.- log "Skipping suspended ${CONFIG_NAMESPACE}/${deployment}; desired replicas=0"- continue+ local namespace deployment desired_replicas selector container env_name+ local current_namespace="" current_deployment="" current_replicas="" current_selector=""+ local current_bindings=""+ while IFS=$'\x1f' read -r namespace deployment desired_replicas selector container env_name; do+ [[ -n "${namespace}" && -n "${deployment}" && -n "${desired_replicas}" && \+ -n "${container}" && -n "${env_name}" ]] ||+ die "consumer discovery returned an incomplete binding record"+ if [[ -n "${current_deployment}" && \+ ( "${namespace}" != "${current_namespace}" || "${deployment}" != "${current_deployment}" ) ]]; then+ reconcile_mcp_consumer_deployment "${current_namespace}" "${current_deployment}" \+ "${current_replicas}" "${current_selector}" "${current_bindings}"+ current_bindings=""fi- if [[ "${force_rollout}" != true ]] && \- consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \- "${TARGET_KEY}" "mcp-host deployment/${deployment}"; then- log "${CONFIG_NAMESPACE}/${deployment} already consumes the target auth key"- continue+ if [[ -z "${current_bindings}" ]]; then+ current_namespace="${namespace}"+ current_deployment="${deployment}"+ current_replicas="${desired_replicas}"+ current_selector="${selector}"+ elif [[ "${namespace}" != "${current_namespace}" || \+ "${deployment}" != "${current_deployment}" || \+ "${desired_replicas}" != "${current_replicas}" || \+ "${selector}" != "${current_selector}" ]]; then+ die "consumer discovery returned inconsistent Deployment metadata for ${namespace}/${deployment}"fi- log "Restarting ${CONFIG_NAMESPACE}/${deployment} after auth key drift"- rollout_restart_with_retry "${CONFIG_NAMESPACE}" "${deployment}" >/dev/null- "${KCTL[@]}" rollout status "deployment/${deployment}" -n "${CONFIG_NAMESPACE}" --timeout="${MCP_HOST_ROLLOUT_TIMEOUT}"- verify_consumer_pods_use_key "${CONFIG_NAMESPACE}" "app=${deployment}" mcp-host \- "${TARGET_KEY}" "mcp-host deployment/${deployment}"- done+ if [[ -n "${current_bindings}" ]]; then+ current_bindings+=$'\n'+ fi+ current_bindings+="${container}"$'\x1f'"${env_name}"+ done <<<"${binding_records}"+ if [[ -n "${current_deployment}" ]]; then+ reconcile_mcp_consumer_deployment "${current_namespace}" "${current_deployment}" \+ "${current_replicas}" "${current_selector}" "${current_bindings}"+ fi+ fi++ local final_binding_records final_hash+ if ! final_binding_records="$(discover_mcp_consumer_bindings)"; then+ die "cannot re-enumerate effective ${CONFIGMAP}.${TARGET_KEY} consumers before attestation"+ fi+ [[ "${final_binding_records}" == "${binding_records}" ]] ||+ die "${CONFIGMAP}.${TARGET_KEY} consumer contracts changed during convergence"+ if ! final_hash="$(configmap_key_hash "${CONFIG_NAMESPACE}" "${CONFIGMAP}" "${TARGET_KEY}" 2>&1)"; then+ die "cannot re-read auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} before attestation: ${final_hash}"fi+ [[ "${final_hash}" == "${source_hash}" ]] ||+ die "auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}.${TARGET_KEY} changed before attestation"if [[ "${applied_hash}" != "${source_hash}" ]]; thenmark_key_applied "${CONFIG_NAMESPACE}" "${CONFIGMAP}"fi@@ -592,7 +667,7 @@ sync_gfs() {fi}-sync_mcp_host+sync_mcp_consumersif [[ "${SYNC_GFS}" == "true" ]]; thensync_gfsfidiff --git a/scripts/tests/test-minikube-sync-auth-key-durable.sh b/scripts/tests/test-minikube-sync-auth-key-durable.shindex f7e61cdef..736c47529 100755--- a/scripts/tests/test-minikube-sync-auth-key-durable.sh+++ b/scripts/tests/test-minikube-sync-auth-key-durable.sh@@ -125,10 +125,147 @@ PYexit 0fi+# Runtime-contract discovery reads the real Deployment pod templates in one+# namespace-scoped snapshot. Keep these fixtures faithful to Kubernetes JSON:+# selectors are independent from Deployment names, WFC uses an explicit+# configMapKeyRef, and an HCC-owned unrelated Deployment has no target binding.+if [[ "${1:-}" == get && ( "${2:-}" == deployment || "${2:-}" == deployments ) && "$*" == *'-o json'* ]]; then+ python3 - <<'PY'+import json+import os++target_ref = {"configMapRef": {"name": "mcp-host-config"}}+direct_ref = {+ "valueFrom": {+ "configMapKeyRef": {+ "name": "mcp-host-config",+ "key": "CLERUM_AUTH_JWT_PUBLIC_KEY",+ }+ }+}+++def deployment(name, labels, containers, replicas=1, managed_by="host-context-controller"):+ return {+ "apiVersion": "apps/v1",+ "kind": "Deployment",+ "metadata": {+ "name": name,+ "namespace": "mcp-host",+ "labels": {"clerum.io/managed-by": managed_by},+ },+ "spec": {+ "replicas": replicas,+ "selector": {"matchLabels": labels},+ "template": {+ "metadata": {"labels": labels},+ "spec": {"containers": containers},+ },+ },+ }+++def mcp(name="chatllm", replicas=1, selector=None):+ labels = selector or {"app": name}+ return deployment(+ name,+ labels,+ [{"name": "mcp-host", "envFrom": [target_ref], "env": []}],+ replicas,+ )+++def wfc(replicas=1):+ return deployment(+ "wfc-21352205ce",+ {+ "app": "workspace-files-controller",+ "clerum.io/sharedfilesystem": "workspace-alpha",+ "clerum.io/sharedfilesystem-namespace": "mcp-host",+ },+ [{"name": "workspace-files-controller", "env": [{"name": "WSF_JWT_PUBLIC_KEY", **direct_ref}]}],+ replicas,+ )+++mode = os.environ.get("TEST_MCP_DEPLOYMENTS", "chatllm")+replicas = int(os.environ.get("TEST_MCP_REPLICAS", "1"))+if mode == "none":+ items = []+elif mode == "custom":+ items = [mcp("custom-host", replicas)]+elif mode == "runtime-contract":+ items = [+ mcp("chatllm", replicas, {"app": "mcp-runtime", "clerum.io/host": "alpha"}),+ wfc(replicas),+ deployment(+ "hcc-unrelated",+ {"app": "unrelated-runtime"},+ [{"name": "unrelated", "envFrom": [{"configMapRef": {"name": "unrelated-config"}}]}],+ replicas,+ ),+ deployment(+ "wrc-recipe-host",+ {"app": "workflow-recipe-runtime"},+ [+ {+ "name": "mcp-host",+ "env": [+ {+ "name": "CLERUM_AUTH_JWT_PUBLIC_KEY",+ "valueFrom": {+ "configMapKeyRef": {+ "name": "clerum-wrc-public-key",+ "key": "CLERUM_WRC_SIGNING_PUBLIC_KEY",+ }+ },+ }+ ],+ }+ ],+ replicas,+ managed_by="wrc",+ ),+ ]+elif mode == "multiple-bindings":+ items = [+ deployment(+ "multi-consumer",+ {"app": "multi-runtime", "clerum.io/host": "multi"},+ [+ {+ "name": "prefixed",+ "envFrom": [{"prefix": "AUTH_", **target_ref}],+ "env": [],+ },+ {+ "name": "direct",+ "env": [{"name": "DIRECT_JWT_PUBLIC_KEY", **direct_ref}],+ },+ {+ "name": "overridden",+ "envFrom": [target_ref],+ "env": [{"name": "CLERUM_AUTH_JWT_PUBLIC_KEY", "value": "literal-override"}],+ },+ ],+ replicas,+ )+ ]+else:+ items = [mcp("chatllm", replicas)]++print(json.dumps({"apiVersion": "v1", "kind": "List", "items": items}))+PY+ exit 0+fi+if [[ "${1:-}" == get && "${2:-}" == deployment && "${3:-}" == -l ]]; thencase "${TEST_MCP_DEPLOYMENTS:-chatllm}" innone) ;;custom) printf 'deployment.apps/custom-host\n' ;;+ runtime-contract)+ printf 'deployment.apps/chatllm\ndeployment.apps/wfc-21352205ce\ndeployment.apps/hcc-unrelated\n'+ ;;*) printf 'deployment.apps/chatllm\n' ;;esacexit 0@@ -153,7 +290,7 @@ fiif [[ "${1:-}" == get && "${2:-}" == deployment/* ]]; thencase "${2}" in- deployment/chatllm|deployment/custom-host)+ deployment/chatllm|deployment/custom-host|deployment/wfc-21352205ce|deployment/hcc-unrelated)[[ "$*" == *'.spec.replicas'* ]] && printf '%s' "${TEST_MCP_REPLICAS:-1}"exit 0 ;;deployment/mcp-host) echo 'Error from server (NotFound): deployments.apps "mcp-host" not found' >&2; exit 1 ;;@@ -197,6 +334,16 @@ if [[ "${1:-}" == get && "${2:-}" == pods ]]; thencase "$*" in*app=chatllm*) printf 'chatllm-pod|True|\n' ;;*app=custom-host*) printf 'custom-host-pod|True|\n' ;;+ *'app=mcp-runtime,clerum.io/host=alpha'*) printf 'chatllm-pod|True|\n' ;;+ *'app=workspace-files-controller,clerum.io/sharedfilesystem=workspace-alpha,clerum.io/sharedfilesystem-namespace=mcp-host'*)+ if [[ "${TEST_NO_READY_AFTER_ROLLOUT:-}" == wfc-21352205ce && \+ -f "${TEST_STATE_DIR}/wfc-21352205ce-restart.count" ]]; then+ :+ else+ printf 'wfc-21352205ce-pod|True|\n'+ fi+ ;;+ *'app=multi-runtime,clerum.io/host=multi'*) printf 'multi-consumer-pod|True|\n' ;;*gfsc-role=writer*) printf 'gfsc-writer-pod|True|\n' ;;*gfsc-role=reader*) printf 'gfsc-reader-pod|True|\n' ;;*) echo "unexpected pod selector: $*" >&2; exit 92 ;;@@ -211,6 +358,37 @@ if [[ "${1:-}" == exec ]]; thenexpected_key="${TEST_RUNTIME_KEY}"[[ "$*" == *GFS_JWT_PUBLIC_KEY* ]] && expected_key="${TEST_RUNTIME_KEY_GFS:-${expected_key}}"[[ "$*" == *CLERUM_AUTH_JWT_PUBLIC_KEY* ]] && expected_key="${TEST_RUNTIME_KEY_MCP:-${expected_key}}"+ [[ "$*" == *WSF_JWT_PUBLIC_KEY* ]] && expected_key="${TEST_RUNTIME_KEY_WFC:-${expected_key}}"++ pod="${3:-}"+ container=""+ previous=""+ for argument in "$@"; do+ if [[ "${previous}" == -c ]]; then+ container="${argument}"+ break+ fi+ previous="${argument}"+ done+ env_name="${!#}"+ case "${pod}" in+ chatllm-pod) deployment_name=chatllm ;;+ custom-host-pod) deployment_name=custom-host ;;+ wfc-21352205ce-pod) deployment_name=wfc-21352205ce ;;+ multi-consumer-pod) deployment_name=multi-consumer ;;+ gfsc-writer-pod) deployment_name=gfsc-writer ;;+ gfsc-reader-pod) deployment_name=gfsc-reader ;;+ *) deployment_name="${pod%-pod}" ;;+ esac+ binding="${deployment_name}/${container}/${env_name}"+ if [[ ",${TEST_STALE_BINDINGS:-}," == *",${binding},"* ]]; then+ if [[ ! -f "${TEST_STATE_DIR}/${deployment_name}-restart.count" ]]; then+ expected_key=old-key+ fi+ fi+ if [[ ",${TEST_PERSIST_STALE_BINDINGS:-}," == *",${binding},"* ]]; then+ expected_key=old-key+ fiprintf '%s' "${expected_key}" >"${expected_file}"cmp -s "${input_file}" "${expected_file}"exit@@ -371,6 +549,137 @@ if grep -q 'rollout restart deployment/' "${LOG_FILE}"; thenexit 1fi+# ConfigMap drift and process drift are independent. Both the envFrom-backed+# MCP host and the direct configMapKeyRef-backed WFC already run with the+# source key, so repairing the ConfigMap must not restart either Deployment.+# The HCC-owned unrelated Deployment and the WRC recipe host deliberately use+# other ConfigMaps and must be ignored despite the broad owner label or the+# recipe host's identical effective environment-variable name.+printf 'old-mcp-key' >"${STATE_DIR}/mcp.key"+rm -f "${STATE_DIR}/mcp.applied"+: >"${LOG_FILE}"+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_RUNTIME_KEY_MCP=public-key \+ TEST_RUNTIME_KEY_WFC=public-key TEST_MCP_DEPLOYMENTS=runtime-contract \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo 'FAIL: runtime-contract consumers did not converge after ConfigMap repair' >&2+ exit 1+fi+[[ "$(cat "${STATE_DIR}/mcp.applied")" == "${SOURCE_HASH}" ]]+if grep -q 'rollout restart deployment/' "${LOG_FILE}"; then+ echo 'FAIL: ConfigMap-only drift restarted already-converged runtime consumers' >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi+if grep -q 'deployment/hcc-unrelated' "${LOG_FILE}"; then+ echo 'FAIL: unrelated HCC Deployment was treated as an auth-key consumer' >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi+if grep -q 'deployment/wrc-recipe-host' "${LOG_FILE}"; then+ echo 'FAIL: a WRC recipe host using clerum-wrc-public-key was treated as a platform auth-key consumer' >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi+grep -q -- '-l app=workspace-files-controller,clerum.io/sharedfilesystem=workspace-alpha,clerum.io/sharedfilesystem-namespace=mcp-host' "${LOG_FILE}"+grep -q -- '-c workspace-files-controller -- node -e .* WSF_JWT_PUBLIC_KEY' "${LOG_FILE}"++# A rotation must restart only the stale MCP host. WFC is independently+# verified through its own binding and remains untouched.+printf 'public-key' >"${STATE_DIR}/mcp.key"+rm -f "${STATE_DIR}/mcp.applied" \+ "${STATE_DIR}/chatllm-restart.count" "${STATE_DIR}/wfc-21352205ce-restart.count"+: >"${LOG_FILE}"+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=runtime-contract \+ TEST_STALE_BINDINGS='chatllm/mcp-host/CLERUM_AUTH_JWT_PUBLIC_KEY' \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo 'FAIL: stale MCP host did not converge after its selective rollout' >&2+ exit 1+fi+[[ "$(cat "${STATE_DIR}/chatllm-restart.count")" == 1 ]]+[[ ! -e "${STATE_DIR}/wfc-21352205ce-restart.count" ]]++# The inverse rotation proves the production WFC contract: real selector,+# real container, and effective env name. Only WFC may restart.+rm -f "${STATE_DIR}/mcp.applied" \+ "${STATE_DIR}/chatllm-restart.count" "${STATE_DIR}/wfc-21352205ce-restart.count"+: >"${LOG_FILE}"+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=runtime-contract \+ TEST_STALE_BINDINGS='wfc-21352205ce/workspace-files-controller/WSF_JWT_PUBLIC_KEY' \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo 'FAIL: stale WFC did not converge after its selective rollout' >&2+ exit 1+fi+[[ "$(cat "${STATE_DIR}/wfc-21352205ce-restart.count")" == 1 ]]+[[ ! -e "${STATE_DIR}/chatllm-restart.count" ]]+grep -q 'rollout restart deployment/wfc-21352205ce' "${LOG_FILE}"+grep -q -- '-l app=workspace-files-controller,clerum.io/sharedfilesystem=workspace-alpha,clerum.io/sharedfilesystem-namespace=mcp-host' "${LOG_FILE}"+grep -q -- '-c workspace-files-controller -- node -e .* WSF_JWT_PUBLIC_KEY' "${LOG_FILE}"++# A rollout that still yields no Ready, provable WFC pod must fail closed and+# leave the convergence marker absent.+rm -f "${STATE_DIR}/mcp.applied" \+ "${STATE_DIR}/chatllm-restart.count" "${STATE_DIR}/wfc-21352205ce-restart.count"+: >"${LOG_FILE}"+if run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=runtime-contract \+ TEST_STALE_BINDINGS='wfc-21352205ce/workspace-files-controller/WSF_JWT_PUBLIC_KEY' \+ TEST_NO_READY_AFTER_ROLLOUT=wfc-21352205ce \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ echo 'FAIL: auth-key sync accepted WFC without a Ready pod after rollout' >&2+ exit 1+fi+[[ "$(cat "${STATE_DIR}/wfc-21352205ce-restart.count")" == 1 ]]+[[ ! -e "${STATE_DIR}/mcp.applied" ]]+grep -q 'no active pod proved consumption of the target auth key' "${OUT_FILE}"++# Multiple containers and effective bindings must all be proved. envFrom+# prefix changes the runtime name; an explicit literal env overrides the+# unprefixed envFrom value and therefore is not a target-key binding.+rm -f "${STATE_DIR}/mcp.applied" "${STATE_DIR}/multi-consumer-restart.count"+: >"${LOG_FILE}"+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=multiple-bindings \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo 'FAIL: multiple effective auth-key bindings were not proved' >&2+ exit 1+fi+[[ ! -e "${STATE_DIR}/multi-consumer-restart.count" ]]+grep -q -- '-c prefixed -- node -e .* AUTH_CLERUM_AUTH_JWT_PUBLIC_KEY' "${LOG_FILE}"+grep -q -- '-c direct -- node -e .* DIRECT_JWT_PUBLIC_KEY' "${LOG_FILE}"+if grep -q -- '-c overridden --' "${LOG_FILE}"; then+ echo 'FAIL: an explicit literal env override was treated as a ConfigMap binding' >&2+ cat "${LOG_FILE}" >&2+ exit 1+fi++# One stale binding makes the whole Deployment stale, but it is restarted only+# once and every binding is re-proved after rollout.+rm -f "${STATE_DIR}/mcp.applied" "${STATE_DIR}/multi-consumer-restart.count"+: >"${LOG_FILE}"+if ! run_sync env TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=multiple-bindings \+ TEST_STALE_BINDINGS='multi-consumer/direct/DIRECT_JWT_PUBLIC_KEY' \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ cat "${OUT_FILE}" >&2+ cat "${LOG_FILE}" >&2+ echo 'FAIL: multiple-binding Deployment did not converge after one rollout' >&2+ exit 1+fi+[[ "$(cat "${STATE_DIR}/multi-consumer-restart.count")" == 1 ]]+[[ "$(grep -c 'rollout restart deployment/multi-consumer' "${LOG_FILE}")" == 1 ]]+# A source rotation after consumer proof must invalidate the run before the# final success message, even when the previous convergence marker was already# written. This is the race the resourceVersion+key snapshot closes.@@ -391,4 +700,26 @@ grep -q 'auth source rotated during convergence' "${OUT_FILE}" || {exit 1}+# Preserve the remote fail-closed barriers used by evenfire-infra. A substring+# match is not authorization, and shared-profile bootstrap can never carry the+# remote capability.+: >"${LOG_FILE}"+if GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=not-fake run_sync env \+ TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=none \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ >"${OUT_FILE}" 2>&1; then+ echo 'FAIL: remote auth reconciliation accepted a non-exact context match' >&2+ exit 1+fi+grep -q 'remote auth-key convergence requires an explicit allowlisted context' "${OUT_FILE}"++if GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS=fake run_sync env \+ TEST_RUNTIME_KEY=public-key TEST_MCP_DEPLOYMENTS=none \+ bash "${ROOT}/scripts/minikube/sync-auth-key.sh" --context fake --require-mcp --skip-gfs \+ --shared-profile-bootstrap >"${OUT_FILE}" 2>&1; then+ echo 'FAIL: shared-profile bootstrap accepted remote reconciliation authorization' >&2+ exit 1+fi+grep -q -- '--shared-profile-bootstrap cannot carry remote reconciliation authorization' "${OUT_FILE}"+echo "PASS: auth-key convergence is durable, resumable, consumer-proven, and idempotent"diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.shindex 643bdd1ff..35f95136b 100755--- a/scripts/tests/test-minikube-sync-auth-key.sh+++ b/scripts/tests/test-minikube-sync-auth-key.sh@@ -9,6 +9,8 @@ STATE_DIR="${TMP_DIR}/state"LOG_FILE="${TMP_DIR}/kubectl.log"OUT_FILE="${TMP_DIR}/sync-auth-key.out"mkdir -p "${STATE_DIR}"+printf 'old-key' >"${STATE_DIR}/mcp.key"+printf 'old-gfs-key' >"${STATE_DIR}/gfs.key"# The production helper requires a live parent T2 lease. Keep the behavior# tests isolated with a lease owned by this test shell rather than adding a@@ -99,11 +101,12 @@ fiif [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-config" ]]; thenif has_output_flag "$@"; thenif [[ "$*" == *'-o json' ]]; then- printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"CLERUM_AUTH_JWT_PUBLIC_KEY":"old-key"}}'+ printf '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"CLERUM_AUTH_JWT_PUBLIC_KEY":"%s"}}' \+ "$(cat "${TEST_STATE_DIR}/mcp.key")"elif [[ "$*" == *auth-key-applied-sha256* ]]; thenprintf '%s' "${TEST_MCP_APPLIED_HASH:-old-hash}"else- printf 'old-key'+ cat "${TEST_STATE_DIR}/mcp.key"fifiexit 0@@ -116,21 +119,24 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]fiif has_output_flag "$@"; thenif [[ "$*" == *'-o json' ]]; then- printf '%s' '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"jwt-public-key":"old-gfs-key"}}'+ printf '{"metadata":{"resourceVersion":"42","annotations":{}},"data":{"jwt-public-key":"%s"}}' \+ "$(cat "${TEST_STATE_DIR}/gfs.key")"elif [[ "$*" == *auth-key-applied-sha256* ]]; thenprintf '%s' "${TEST_GFS_APPLIED_HASH:-old-hash}"else- printf 'old-gfs-key'+ cat "${TEST_STATE_DIR}/gfs.key"fifiexit 0fiif [[ "${1:-}" == "patch" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-config" ]]; then+ [[ "$*" == *'"data"'* ]] && printf 'public-key' >"${TEST_STATE_DIR}/mcp.key"exit 0fiif [[ "${1:-}" == "patch" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]]; then+ [[ "$*" == *'"data"'* ]] && printf 'public-key' >"${TEST_STATE_DIR}/gfs.key"exit 0fi@@ -145,8 +151,12 @@ if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "gfs-controller-dexit 0fi-if [[ "${1:-}" == "get" && "${2:-}" == "deployment" && "${3:-}" == "-l" ]]; then- printf 'deployment.apps/chatllm\ndeployment.apps/mcp-host\ndeployment.apps/custom-host\n'+if [[ "${1:-}" == "get" && ( "${2:-}" == "deployment" || "${2:-}" == "deployments" ) && "$*" == *'-o json'* ]]; then+ printf '%s' '{"apiVersion":"v1","kind":"List","items":[+ {"metadata":{"name":"chatllm","namespace":"mcp-host"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"chatllm"}},"template":{"spec":{"containers":[{"name":"mcp-host","envFrom":[{"configMapRef":{"name":"mcp-host-config"}}]}]}}}},+ {"metadata":{"name":"mcp-host","namespace":"mcp-host"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"mcp-host"}},"template":{"spec":{"containers":[{"name":"mcp-host","envFrom":[{"configMapRef":{"name":"mcp-host-config"}}]}]}}}},+ {"metadata":{"name":"custom-host","namespace":"mcp-host"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"custom-host"}},"template":{"spec":{"containers":[{"name":"mcp-host","envFrom":[{"configMapRef":{"name":"mcp-host-config"}}]}]}}}}+ ]}'exit 0fi@@ -175,7 +185,15 @@ fiif [[ "${1:-}" == "exec" ]]; thenpayload="$(cat)"- [[ "${TEST_CONSUMER_KEY_MATCH:-1}" == "1" && "${payload}" == "public-key" ]]+ if [[ "$*" == *chatllm-pod* ]]; then+ [[ -f "${TEST_STATE_DIR}/chatllm.restart-count" && "${payload}" == "public-key" ]]+ elif [[ "$*" == *mcp-host-pod* ]]; then+ [[ -f "${TEST_STATE_DIR}/mcp-host.restart-count" && "${payload}" == "public-key" ]]+ elif [[ "$*" == *custom-host-pod* ]]; then+ [[ -f "${TEST_STATE_DIR}/custom-host.restart-count" && "${payload}" == "public-key" ]]+ else+ [[ "${TEST_CONSUMER_KEY_MATCH:-1}" == "1" && "${payload}" == "public-key" ]]+ fiexitfi@@ -358,6 +376,7 @@ figrep -q 'patch configmap gfs-config' "${LOG_FILE}": >"${LOG_FILE}"+printf 'old-gfs-key' >"${STATE_DIR}/gfs.key"if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \@@ -368,6 +387,7 @@ figrep -q 'gfs-controller-db.connection-string is empty' "${OUT_FILE}": >"${LOG_FILE}"+printf 'old-gfs-key' >"${STATE_DIR}/gfs.key"if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 \GFS_AUTH_SYNC_ALLOW_STAGED=true \@@ -386,6 +406,7 @@ if grep -q 'rollout restart deployment/gfsc-' "${LOG_FILE}"; thenfi: >"${LOG_FILE}"+printf 'old-gfs-key' >"${STATE_DIR}/gfs.key"if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DSN_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 TEST_GFS_STATUS_FAILURE=42 \PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \diff --git a/scripts/tests/test-provision-gfs-runtime.sh b/scripts/tests/test-provision-gfs-runtime.shindex d6fa5ae77..409dce517 100755--- a/scripts/tests/test-provision-gfs-runtime.sh+++ b/scripts/tests/test-provision-gfs-runtime.sh@@ -62,7 +62,7 @@ grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS="$CONTEXT"' \echo 'FAIL: post-overlay credential reconciliation did not carry explicit context authorization' >&2exit 1}-grep -Fq 'sync-auth-key.sh --context "$CONTEXT" --require-gfs' \+grep -Fq 'bash "$ROOT/scripts/minikube/sync-auth-key.sh" --context "$CONTEXT" --require-gfs' \"$ROOT/deploy/scripts/provision-gfs-runtime.sh" || {echo 'FAIL: GFS runtime provisioning does not require its auth source and target' >&2exit 1