Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: close adversarial PR 416 findings
Alfredo Lopez4609609ec68c
diff --git a/Makefile b/Makefileindex 34a206eed..c725d91df 100644--- a/Makefile+++ b/Makefile@@ -155,6 +155,7 @@ minikube-stop: ## Stop minikube cluster# pulls ~25 published images instead of building 28 from source. Build# everything locally with `make minikube-setup-local` (or IMAGE_SOURCE=local).IMAGE_SOURCE ?= ghcr+MINIKUBE_GFS_MUTATION ?= true.PHONY: minikube-setupminikube-setup: ## Clean install from scratch, PULLING published images (IMAGE_SOURCE=local or `make minikube-setup-local` builds instead). Rebuilds the DB; REUSE_DB=true keeps it. SKIP_UIS=true omits Control/Profile UI. Runs 'prereqs' first (SKIP_PREREQS=true to bypass). Needs ADMIN_PASSWORD in .env.@@ -223,18 +224,21 @@ minikube-detect-k8s-api-ip: ## Patch overlays/minikube/patches/k8s-api-ip.yaml w.PHONY: minikube-deploy-allminikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay@$(MAKE) --no-print-directory minikube-detect-k8s-api-ip- @CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-gfs-writer-secret.sh@# Upgrade path: adopt/validate writer and stage reader before HCC cutover.- @writer_dsn="$$(kubectl --context=$(MINIKUBE_PROFILE) -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')" || { \- echo "[minikube-deploy-all] failed to classify the existing GFS writer Secret; refusing HCC cutover" >&2; exit 1; \- }; \- if [ -n "$$writer_dsn" ]; then \- kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \- echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \+ @if [ "$(MINIKUBE_GFS_MUTATION)" != "true" ]; then echo "[minikube-deploy-all] GFS mutation disabled for this non-T2 sync"; fi+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-gfs-writer-secret.sh; \+ writer_dsn="$$(kubectl --context=$(MINIKUBE_PROFILE) -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')" || { \+ echo "[minikube-deploy-all] failed to classify the existing GFS writer Secret; refusing HCC cutover" >&2; exit 1; \}; \- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \- else \- echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \+ if [[ -n "$$writer_dsn" ]]; then \+ kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \+ echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \+ }; \+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \+ else \+ echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \+ fi; \fi@# THE OVERLAY FOLLOWS THE CLUSTER, NOT THIS SHELL. Hardcoding@# deploy/overlays/minikube applied clerum/*:test image refs to a cluster@@ -246,14 +250,22 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay@# above, because an overridden tag renders from a copy of deploy/ that must@# already contain the generated k8s-api-ip.yaml.render_dir="$$(bash scripts/minikube/image-mode.sh --render-dir)" && \- kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | kubectl --context=$(MINIKUBE_PROFILE) apply -f -+ if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | kubectl --context=$(MINIKUBE_PROFILE) apply -f -; \+ else \+ kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | python3 scripts/minikube/filter-gfs-resources.py | kubectl --context=$(MINIKUBE_PROFILE) apply -f -; \+ fiCONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-inter-service-tokens.sh- $(KC) apply -f deploy/overlays/minikube/instances/+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ $(KC) apply -f deploy/overlays/minikube/instances/; \+ else \+ python3 scripts/minikube/filter-gfs-resources.py deploy/overlays/minikube/instances/*.yaml | $(KC) apply -f -; \+ fi@# Kustomize reapplies the persisted mcp-host ConfigMap, which can overwrite@# CLERUM_AUTH_JWT_PUBLIC_KEY with an older repo value. Always re-sync from@# rpc-proxy-secrets after each full overlay apply so Desktop/rpc-proxy/mcp-host@# stay on the same JWT validation key.- @$(MAKE) --no-print-directory minikube-sync-auth-key+ @MINIKUBE_GFS_MUTATION="$(MINIKUBE_GFS_MUTATION)" $(MAKE) --no-print-directory minikube-sync-auth-key@# The pre-overlay helper migrates legacy last-applied ownership without@# removing the provisioning-owned connection-string. When the GFS stack is@# deployed AND control-api is Ready (migration 0048 applied), re-provision@@ -262,13 +274,25 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay@# control-api is not Ready yet (fresh cluster mid-setup), provisioning is@# deferred LOUDLY to the full-setup/pre-gate-sync flow that already orders@# it after control-api migrations.- @if kubectl --context=$(MINIKUBE_PROFILE) get configmap gfs-config -n gfs >/dev/null 2>&1; then \- if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/wait-gfsc-secret-references.sh; \- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \- else \- echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ gfs_config_probe="$$(kubectl --context=$(MINIKUBE_PROFILE) get configmap gfs-config -n gfs 2>&1)" || { \+ if [[ "$$gfs_config_probe" == *NotFound* || "$$gfs_config_probe" == *"not found"* ]]; then \+ echo "[minikube-deploy-all] GFS is not deployed; skipping post-overlay credential reconciliation"; \+ gfs_config_probe=""; \+ else \+ echo "[minikube-deploy-all] unable to inspect GFS configmap; refusing to continue: $$gfs_config_probe" >&2; exit 1; \+ fi; \+ }; \+ if [ -n "$$gfs_config_probe" ]; then \+ if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/wait-gfsc-secret-references.sh; \+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \+ else \+ echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \+ fi; \fi; \+ else \+ echo "[minikube-deploy-all] skipping post-overlay GFS credential reconciliation"; \fi.PHONY: minikube-verify-networkpolicies@@ -364,7 +388,11 @@ minikube-apply-namespaces: ## Create all namespaces.PHONY: minikube-sync-auth-keyminikube-sync-auth-key: ## Sync JWT public key from rpc-proxy-secrets into runtime ConfigMaps when drift exists- @bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE)+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \+ bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE); \+ else \+ bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE) --skip-gfs --require-mcp; \+ fi.PHONY: minikube-sync-auth-key-if-presentminikube-sync-auth-key-if-present: ## Sync JWT public key only when minikube auth resources already existdiff --git a/deploy/scripts/provision-gfs-runtime.sh b/deploy/scripts/provision-gfs-runtime.shindex c4895b1db..bcc7f3527 100755--- a/deploy/scripts/provision-gfs-runtime.sh+++ b/deploy/scripts/provision-gfs-runtime.sh@@ -102,7 +102,7 @@ context_allowed_via_env() {}case "$CONTEXT" in- *example-dev*|minikube|clerum-test|clerum-codex-*|clerum-detached-*|clerum-feat-*|clerum-pr-*)+ minikube|clerum-test|clerum-codex-*|clerum-detached-*|clerum-feat-*|clerum-pr-*);;*)# A non-dev context passes only if explicitly named in ALLOWED_CONTEXTS or@@ -214,7 +214,7 @@ if [ "$SKIP_SYNC" = "1" ]; thenlog "Skipping runtime auth configuration sync"elselog "Syncing runtime auth configuration"- bash scripts/minikube/sync-auth-key.sh --context "$CONTEXT"+ bash scripts/minikube/sync-auth-key.sh --context "$CONTEXT" --require-gfsfiif [ "$SKIP_INSTANCES" = "1" ]; thendiff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.shindex daede2656..76f6cd279 100755--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh@@ -33,15 +33,40 @@ remote_context_allowed() {return 1}-if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == clerum-* ]]; then+if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; then# shellcheck source=scripts/minikube/t2-common.shsource "$ROOT/scripts/minikube/t2-common.sh"if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi+ RECONCILE_CLEANUP_DONE=falsecleanup_reconcile() {+ local status="${1:-$?}"+ if [ "$RECONCILE_CLEANUP_DONE" = true ]; then+ return "$status"+ fi+ RECONCILE_CLEANUP_DONE=true+ trap - EXIT+ trap '' INT TERM+ t2_lock_release "$status" || status=$?+ return "$status"+ }+ handle_reconcile_signal() {+ local signal="$1" status+ case "$signal" in+ INT) status=130 ;;+ TERM) status=143 ;;+ *) status=1 ;;+ esac+ cleanup_reconcile "$status" || status=$?+ exit "$status"+ }+ handle_reconcile_exit() {local status=$?- t2_lock_release "$status"+ cleanup_reconcile "$status" || status=$?+ exit "$status"}- trap cleanup_reconcile EXIT INT TERM+ trap handle_reconcile_exit EXIT+ trap 'handle_reconcile_signal INT' INT+ trap 'handle_reconcile_signal TERM' TERMt2_repo_metadatat2_profile_scopet2_context_check@@ -50,22 +75,17 @@ if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == cle# a second CONTEXT value after the fence has been acquired.CONTEXT="$T2_CONTEXT"else- case "$CONTEXT" in- gke_*)- # GKE deploys are owned by the explicitly selected deploy workflow rather- # than the local Minikube lease. The workflow must opt in explicitly and- # pass the exact context allowlist; the name prefix alone is not- # authorization to mutate a remote cluster.- if ! remote_context_allowed; then- printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2- exit 1- fi- ;;- *)+ # Remote reconciliation is authorized by the explicit workflow decision and+ # exact context membership, never by a provider/name convention. This also+ # supports self-hosted dev contexts whose names are not gke_*.+ if ! remote_context_allowed; then+ if [ -z "${GFS_REMOTE_RECONCILE_AUTHORIZED:-}" ] && [ -z "${ALLOWED_CONTEXTS:-}" ]; thenprintf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2exit 1- ;;- esac+ fi+ printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2+ exit 1+ fifiCONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.shindex b7eae25aa..fb37ef25e 100755--- a/scripts/e2e/minikube-real-postgres.sh+++ b/scripts/e2e/minikube-real-postgres.sh@@ -38,6 +38,7 @@ T1_PENDING_TESTS=0# (e.g. a failed precondition, or this file being sourced by a harness test).T1_GFS_RESTORE_REQUIRED=falseT1_CLEANUP_DONE=false+T1_SIGNAL_STATUS=0die_t1() {local code="$1"@@ -96,13 +97,27 @@ restore_gfs_runtime_credentials() {fi}+handle_t1_signal() {+ local signal="$1" status+ case "$signal" in+ INT) status=130 ;;+ TERM) status=143 ;;+ *) status=1 ;;+ esac+ T1_SIGNAL_STATUS="$status"+ cleanup_t1 "$status"+}+cleanup_t1() {- local status=$?+ local status="${1:-$?}"if [ "$T1_CLEANUP_DONE" = true ]; thenreturn 0fiT1_CLEANUP_DONE=true- trap - EXIT INT TERM+ # Finish the safety restore once it has started. A second signal must not+ # interrupt the lock release and leave a half-restored profile behind.+ trap - EXIT+ trap '' INT TERMif [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; thenlocal command_linecommand_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"@@ -116,10 +131,18 @@ cleanup_t1() {status=1fiif [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi+ if [ "$status" -eq 0 ] && [ "$T1_STATUS" = PASS ]; then+ t2_evidence_write T1 PASS "Real PostgreSQL suites passed; tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS; cleanup=PASS"+ t2_evidence_write complete PASS "T1=PASS tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS cleanup=PASS"+ elif [ "$T1_STATUS" = PASS ]; then+ t2_evidence_write T1 FAIL "Real PostgreSQL suites passed but cleanup failed; cleanup=FAIL"+ fit2_lock_release "$status" || status=$?exit "$status"}-trap cleanup_t1 EXIT INT TERM+trap cleanup_t1 EXIT+trap 'handle_t1_signal INT' INT+trap 'handle_t1_signal TERM' TERMrequire_t1_commands() {local command_name@@ -456,8 +479,9 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.decunset ADMIN_DSN ISOLATED_DSNstop_isolated_postgresT1_STATUS=PASS- t2_evidence_write T1 PASS "Real PostgreSQL suites passed; tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS"- t2_evidence_write complete PASS "T1=PASS tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS"+ # PASS evidence is finalized by cleanup_t1 only after the profile restore and+ # lock release path has completed successfully.+ printf 'T1_STATUS=%s\n' "$T1_STATUS"printf 'T1_TESTS=%s\n' "$T1_TOTAL_TESTS"printf 'T1_PASSED_TESTS=%s\n' "$T1_PASSED_TESTS"printf 'T1_PENDING_TESTS=%s\n' "$T1_PENDING_TESTS"diff --git a/scripts/minikube/filter-gfs-resources.py b/scripts/minikube/filter-gfs-resources.pynew file mode 100755index 000000000..db6e35420--- /dev/null+++ b/scripts/minikube/filter-gfs-resources.py@@ -0,0 +1,56 @@+#!/usr/bin/env python3+"""Remove GFS-owned Kubernetes documents from an intentionally non-GFS apply.++This is used only by non-T2 pre-gate deployment syncs. The filter is explicit+about the small set of cluster-scoped and control-plane GFS boundary objects;+unknown documents are retained rather than guessed away.+"""++from __future__ import annotations++import re+import sys+from pathlib import Path+++def field(document: str, expression: str) -> str:+ match = re.search(expression, document, flags=re.MULTILINE)+ return match.group(1) if match else ""+++def is_gfs_owned(document: str) -> bool:+ kind = field(document, r"^kind:\s*([^\s#]+)\s*$")+ name = field(document, r"^ name:\s*([^\s#]+)\s*$")+ namespace = field(document, r"^ namespace:\s*([^\s#]+)\s*$")+ if kind == "Namespace" and name == "gfs":+ return True+ if namespace == "gfs" or kind == "GlobalFileSystem":+ return True+ if kind == "CustomResourceDefinition" and name == "globalfilesystems.clerum.io":+ return True+ if kind == "NetworkPolicy" and name in {+ "control-api-to-gfsc",+ "control-postgres-from-gfs-controller",+ "deny-all-gfs",+ }:+ return True+ return False+++def read_documents(paths: list[str]) -> str:+ if not paths:+ return sys.stdin.read()+ return "\n---\n".join(Path(path).read_text() for path in paths)+++def main() -> int:+ source = read_documents(sys.argv[1:])+ documents = re.split(r"(?m)^---\s*$", source)+ kept = [document.rstrip() for document in documents if document.strip() and not is_gfs_owned(document)]+ if kept:+ sys.stdout.write("\n---\n".join(kept) + "\n")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.shindex 6a482c709..a375528f7 100755--- a/scripts/minikube/pre-gate-sync.sh+++ b/scripts/minikube/pre-gate-sync.sh@@ -115,9 +115,14 @@ CONTROL_API_FENCED=falseCONTROL_API_REPLICAS=1fence_workflow_reconciler() {- if ! ${KC} get deployment/workflow-recipes -n control-plane >/dev/null 2>&1; then- log "Workflow reconciler is not present; no rollout fence is required"- return 0+ local deployment_probe+ if ! deployment_probe="$(${KC} get deployment/workflow-recipes -n control-plane 2>&1)"; then+ if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then+ log "Workflow reconciler is not present; no rollout fence is required"+ return 0+ fi+ log "ERROR: unable to inspect workflow-recipes before schema migration; refusing to continue without its writer fence"+ return 1fiWRC_REPLICAS="$(${KC} get deployment/workflow-recipes -n control-plane -o jsonpath='{.spec.replicas}')"WRC_REPLICAS="${WRC_REPLICAS:-1}"@@ -147,9 +152,14 @@ restore_workflow_reconciler() {}fence_control_api() {- if ! ${KC} get deployment/control-api -n control-plane >/dev/null 2>&1; then- log "Control API is not present; no writer fence is required"- return 0+ local deployment_probe+ if ! deployment_probe="$(${KC} get deployment/control-api -n control-plane 2>&1)"; then+ if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then+ log "Control API is not present; no writer fence is required"+ return 0+ fi+ log "ERROR: unable to inspect control-api before schema migration; refusing to continue without its writer fence"+ return 1fiCONTROL_API_REPLICAS="$(${KC} get deployment/control-api -n control-plane -o jsonpath='{.spec.replicas}')"CONTROL_API_REPLICAS="${CONTROL_API_REPLICAS:-1}"@@ -320,17 +330,26 @@ ensure_artifact() {}sync_mcp_host_auth_key() {- if ! ${KC} get secret rpc-proxy-secrets -n rpc-proxy >/dev/null 2>&1; then- log "Skipping mcp-host auth key sync (rpc-proxy-secrets not found)"- return 0+ local source_probe target_probe+ if ! source_probe="$(${KC} get secret rpc-proxy-secrets -n rpc-proxy 2>&1)"; then+ if [[ "${source_probe}" == *NotFound* || "${source_probe}" == *"not found"* ]]; then+ log "Skipping mcp-host auth key sync (rpc-proxy-secrets not found)"+ return 0+ fi+ log "ERROR: unable to inspect rpc-proxy/rpc-proxy-secrets before MCP auth sync"+ return 1fi- if ! ${KC} get configmap mcp-host-config -n mcp-host >/dev/null 2>&1; then- log "Skipping mcp-host auth key sync (mcp-host-config not found)"- return 0+ if ! target_probe="$(${KC} get configmap mcp-host-config -n mcp-host 2>&1)"; then+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then+ log "Skipping mcp-host auth key sync (mcp-host-config not found)"+ return 0+ fi+ log "ERROR: unable to inspect mcp-host/mcp-host-config before MCP auth sync"+ return 1fi- bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --skip-gfs+ bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --skip-gfs --require-mcp}sync_gfs_auth_key() {@@ -385,6 +404,10 @@ converge_gfs_reader_after_restore() {log "ERROR: unable to read Ready replicas for gfs/${deployment}"return 1fi+ # Kubernetes omits status.readyReplicas while a Deployment has zero Ready+ # pods. An empty successful read is a real 0; an API failure above remains+ # unknown and fails closed.+ ready="${ready:-0}"if [[ ! "${ready}" =~ ^[0-9]+$ ]]; thenlog "ERROR: Ready replicas for gfs/${deployment} is not numeric"return 1@@ -576,7 +599,11 @@ if [[ "${cluster_changed}" == "true" ]]; thencd "${PROJECT_DIR}"make minikube-deploy-crds)- CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"+ if [[ "${GATE_NAME}" == "minikube-t2" ]]; then+ CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"+ else+ log "Skipping GFS writer Secret mutation before ${GATE_NAME}; only minikube-t2 owns GFS recovery"+ fiwriter_dsn="$(${KC} -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"if [[ -n "${writer_dsn}" ]]; then# Do not reconcile the GFS roles before the migration window. The@@ -643,7 +670,10 @@ if [[ "${cluster_changed}" == "true" ]]; then(cd "${PROJECT_DIR}"make minikube-apply-secrets- make minikube-deploy-all+ gfs_mutation=false+ [[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_mutation=true+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" MINIKUBE_GFS_MUTATION="${gfs_mutation}" \+ make minikube-deploy-allif [[ "${FORCE_RESTART}" == "true" || "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ]]; thenmake minikube-restart-allfidiff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.shindex bb6680103..0d27d5976 100755--- a/scripts/minikube/settle-gfs-reader-rollout.sh+++ b/scripts/minikube/settle-gfs-reader-rollout.sh@@ -41,11 +41,37 @@ log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }die() { log "ERROR: $*"; exit 1; }kc() { kubectl --context="$T2_CONTEXT" "$@"; }+SETTLE_CLEANUP_DONE=false+cleanup_settle() {+ local status="${1:-$?}"+ if [ "$SETTLE_CLEANUP_DONE" = true ]; then+ return "$status"+ fi+ SETTLE_CLEANUP_DONE=true+ trap - EXIT+ trap '' INT TERM+ t2_lock_release "$status" || status=$?+ return "$status"+}+handle_settle_signal() {+ local signal="$1" status+ case "$signal" in+ INT) status=130 ;;+ TERM) status=143 ;;+ *) status=1 ;;+ esac+ cleanup_settle "$status" || status=$?+ exit "$status"+}+handle_settle_exit() {local status=$?- t2_lock_release "$status"+ cleanup_settle "$status" || status=$?+ exit "$status"}-trap cleanup_settle EXIT INT TERM+trap handle_settle_exit EXIT+trap 'handle_settle_signal INT' INT+trap 'handle_settle_signal TERM' TERMt2_repo_metadatat2_profile_scopet2_context_check@@ -58,13 +84,31 @@ source "$ROOT/deploy/scripts/lib/gfs-credential-secret.sh"# shellcheck source=deploy/scripts/lib/gfs-credential-rollout.shsource "$ROOT/deploy/scripts/lib/gfs-credential-rollout.sh"-if ! kc -n "$GFS_NS" get deployment "$DEPLOY" >/dev/null 2>&1; then- exit 0+deployment_probe=""+if ! deployment_probe="$(kc -n "$GFS_NS" get deployment "$DEPLOY" 2>&1)"; then+ if [[ "$deployment_probe" == *NotFound* || "$deployment_probe" == *"not found"* ]]; then+ exit 0+ fi+ die "unable to inspect gfs/${DEPLOY}: ${deployment_probe}"fi-desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.spec.replicas}')"-ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.status.readyReplicas}')"-if [ -z "$desired" ] || [ "$desired" = "0" ] || [ "${ready:-0}" != "$desired" ]; then+if ! desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.spec.replicas}' 2>&1)"; then+ die "unable to read desired replicas for gfs/${DEPLOY}: ${desired}"+fi+if [ -z "$desired" ] || ! [[ "$desired" =~ ^[0-9]+$ ]]; then+ die "desired replicas for gfs/${DEPLOY} is not numeric"+fi+if [ "$desired" = "0" ]; then+ exit 0+fi+if ! ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.status.readyReplicas}' 2>&1)"; then+ die "unable to read Ready replicas for gfs/${DEPLOY}: ${ready}"+fi+ready="${ready:-0}"+if ! [[ "$ready" =~ ^[0-9]+$ ]]; then+ die "Ready replicas for gfs/${DEPLOY} is not numeric"+fi+if [ "$ready" != "$desired" ]; thenexit 0fi@@ -73,7 +117,7 @@ current_rev="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \-o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"rs_rows="$(kc -n "$GFS_NS" get rs -l "$SELECTOR" -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.spec.replicas}{"|"}{.status.readyReplicas}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' \- 2>/dev/null || true)"+ 2>/dev/null)"while IFS='|' read -r rs_name rs_owner rs_replicas rs_ready rs_rev; do[ -n "$rs_name" ] || continue[ "$rs_owner" = "$DEPLOY" ] || continue@@ -87,7 +131,7 @@ done <<<"$rs_rows"pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.containerStatuses[0].state.waiting.reason}{"\n"}{end}' \- 2>/dev/null || true)"+ 2>/dev/null)"while IFS='|' read -r pod_name pod_ready pod_deleting pod_reason; do[ -n "$pod_name" ] || continue[ -z "$pod_deleting" ] || continue@@ -97,8 +141,12 @@ while IFS='|' read -r pod_name pod_ready pod_deleting pod_reason; dokc -n "$GFS_NS" delete pod "$pod_name" --wait=false >/dev/nulldone <<<"$pod_rows"-if ! kc -n "$GFS_NS" get secret "$SECRET" >/dev/null 2>&1; then- exit 0+secret_probe=""+if ! secret_probe="$(kc -n "$GFS_NS" get secret "$SECRET" 2>&1)"; then+ if [[ "$secret_probe" == *NotFound* || "$secret_probe" == *"not found"* ]]; then+ exit 0+ fi+ die "unable to inspect ${GFS_NS}/${SECRET}: ${secret_probe}"fistate="$(kc -n "$GFS_NS" get secret "$SECRET" \diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.shindex 661e9b978..ad9168795 100755--- a/scripts/minikube/sync-auth-key.sh+++ b/scripts/minikube/sync-auth-key.sh@@ -1,8 +1,10 @@#!/usr/bin/env bashset -euo pipefail+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"PROFILE="clerum-test"REQUIRE_GFS=false+REQUIRE_MCP=falseSYNC_GFS=truewhile [[ $# -gt 0 ]]; do@@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; doREQUIRE_GFS=trueshift;;+ --require-mcp)+ REQUIRE_MCP=true+ shift+ ;;--skip-gfs)# Pre-gate security lanes need the MCP host key refresh without mutating# the optional GFS plane. The T2 GFS path uses --require-gfs instead.@@ -29,7 +35,7 @@ while [[ $# -gt 0 ]]; doshift;;-h|--help)- echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-gfs|--skip-gfs]" >&2+ echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-mcp] [--require-gfs|--skip-gfs]" >&2exit 0;;*)@@ -101,12 +107,19 @@ PY# The public key is the SAME for every consumer (one platform keypair), so it is# fetched once and synced into each consumer's ConfigMap below.-if ! "${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" >/dev/null 2>&1; then- if [[ "${REQUIRE_GFS}" == "true" ]]; then- die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"+source_probe=""+if ! source_probe="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" 2>&1)"; then+ if [[ "${REQUIRE_GFS}" == "true" || "${REQUIRE_MCP}" == "true" ]]; then+ if [[ "${REQUIRE_GFS}" == "true" ]]; then+ die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"+ fi+ die "required MCP auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"+ fi+ if [ -z "${source_probe}" ] || [[ "${source_probe}" == *NotFound* || "${source_probe}" == *"not found"* ]]; then+ log "Skipping auth key sync (${SOURCE_SECRET} not found)"+ exit 0fi- log "Skipping auth key sync (${SOURCE_SECRET} not found)"- exit 0+ die "cannot inspect auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET}: ${source_probe}"fisource_key="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}" | base64 -d)"if [[ -z "${source_key}" ]]; then@@ -116,18 +129,31 @@ if [[ -z "${source_key}" ]]; thendie "auth source key ${SOURCE_KEY} is empty; refusing to mutate consumers"fiif [[ "${REQUIRE_GFS}" == "true" ]] && \- ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then- die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable"+ ! gfs_target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; then+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable: ${gfs_target_probe}"+fi+if [[ "${REQUIRE_MCP}" == "true" ]] && \+ ! mcp_target_probe="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" 2>&1)"; then+ die "required MCP auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} is missing or unreadable: ${mcp_target_probe}"fi# ── Target 1: mcp-host-config (restart named chatllm/mcp-host with retry) ──────sync_mcp_host() {- if ! "${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" >/dev/null 2>&1; then- log "Skipping ${CONFIGMAP} sync (not found)"- return 0+ local target_probe=""+ if ! target_probe="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" 2>&1)"; then+ if [[ "${REQUIRE_MCP}" == "true" ]]; then+ die "required MCP auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} disappeared or became unreadable during sync"+ fi+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then+ log "Skipping ${CONFIGMAP} sync (not found)"+ return 0+ fi+ die "cannot inspect auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${target_probe}"filocal current_key- current_key="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" -o "jsonpath={.data.${TARGET_KEY}}" 2>/dev/null || true)"+ if ! current_key="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" -o "jsonpath={.data.${TARGET_KEY}}" 2>&1)"; then+ die "cannot read auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${current_key}"+ fiif [[ "${source_key}" == "${current_key}" ]]; thenlog "${CONFIGMAP}.${TARGET_KEY} already matches source; no patch needed"return 0@@ -150,33 +176,62 @@ sync_mcp_host() {# ── Target 2: gfs-config (gfsc fails closed without the verification key) ──────sync_gfs() {- if ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then+ local target_probe=""+ if ! target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; thenif [[ "${REQUIRE_GFS}" == "true" ]]; then- die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared or became unreadable during sync"+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared or became unreadable during sync: ${target_probe}"fi- log "Skipping ${GFS_CONFIGMAP} sync (not found)"- return 0+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then+ log "Skipping ${GFS_CONFIGMAP} sync (not found)"+ return 0+ fi+ die "cannot inspect auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${target_probe}"filocal current_key- current_key="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" -o "jsonpath={.data.${GFS_TARGET_KEY}}" 2>/dev/null || true)"+ if ! current_key="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" -o "jsonpath={.data.${GFS_TARGET_KEY}}" 2>&1)"; then+ die "cannot read auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${current_key}"+ fiif [[ "${source_key}" == "${current_key}" ]]; thenlog "${GFS_CONFIGMAP}.${GFS_TARGET_KEY} already matches source; no patch needed"return 0filog "Patching ${GFS_CONFIGMAP}.${GFS_TARGET_KEY}""${KCTL[@]}" patch configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${GFS_TARGET_KEY}")" >/dev/null- local dsn- dsn="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>/dev/null | base64 -d || true)"+ local dsn_encoded dsn+ if ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then+ if [[ "${dsn_encoded}" == *NotFound* || "${dsn_encoded}" == *"not found"* ]]; then+ log "Skipping gfsc restart after auth key drift (gfs-controller-db Secret is not populated yet)"+ return 0+ fi+ die "cannot inspect ${GFS_NAMESPACE}/gfs-controller-db before gfsc restart: ${dsn_encoded}"+ fi+ if [[ -z "${dsn_encoded}" ]]; then+ log "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"+ return 0+ fi+ if ! dsn="$(printf '%s' "${dsn_encoded}" | base64 -d 2>&1)"; then+ die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is not valid base64: ${dsn}"+ fiif [[ -z "${dsn}" ]]; thenlog "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"return 0fi# gfsc reads the key at boot; restart the writer+reader so the new key takes# effect (label-based — the reconciler owns the deployment names).- if "${KCTL[@]}" get deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" -o name 2>/dev/null | grep -q .; then+ local deployments deployment rollout_output+ if ! deployments="$("${KCTL[@]}" get deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" -o name 2>&1)"; then+ die "cannot inspect gfsc deployments after auth key drift: ${deployments}"+ fi+ if [[ -n "${deployments}" ]]; thenlog "Restarting gfsc (${GFS_DEPLOY_SELECTOR}) after auth key drift""${KCTL[@]}" rollout restart deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" >/dev/null- "${KCTL[@]}" rollout status deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" --timeout=90s || true+ while IFS= read -r deployment; do+ [[ -n "${deployment}" ]] || continue+ if ! rollout_output="$(PATH="${SCRIPT_DIR}/gfs-rollout-shim:${PATH}" CONTEXT="${PROFILE}" \+ "${KCTL[@]}" rollout status "${deployment}" -n "${GFS_NAMESPACE}" --timeout=90s 2>&1)"; then+ die "gfsc deployment ${deployment} did not become Ready after auth key drift: ${rollout_output}"+ fi+ done <<<"${deployments}"elselog "No gfsc deployments exist yet; auth key is synced for future pods"fidiff --git a/scripts/minikube/t0.sh b/scripts/minikube/t0.shnew file mode 100755index 000000000..1521503e8--- /dev/null+++ b/scripts/minikube/t0.sh@@ -0,0 +1,44 @@+#!/usr/bin/env bash+# Canonical T0 checks beyond the structural contract suite.+set -euo pipefail+set +x++PROJECT_DIR="${T0_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)}"+ORIGIN_DEV="${T0_ORIGIN_DEV:-origin/dev}"+HEAD="${T0_HEAD:-HEAD}"++shell_files=()+package_dirs=()+seen_packages="|"+while IFS= read -r path; do+ [ -n "$path" ] || continue+ case "$path" in+ *.sh|*.bash) shell_files+=("$PROJECT_DIR/$path") ;;+ esac+ top="${path%%/*}"+ if [ "$top" != "$path" ] && [ -f "$PROJECT_DIR/$top/package.json" ] && [[ "$seen_packages" != *"|$top|"* ]]; then+ seen_packages="${seen_packages}${top}|"+ package_dirs+=("$top")+ fi+done < <(git -C "$PROJECT_DIR" diff --name-only "$ORIGIN_DEV...$HEAD")++if [ "${#shell_files[@]}" -gt 0 ]; then+ if command -v shellcheck >/dev/null 2>&1; then+ shellcheck --severity=error "${shell_files[@]}"+ printf 'T0_SHELLCHECK=PASS\n'+ else+ printf 'T0_SHELLCHECK=NOT_AVAILABLE\n'+ fi+else+ printf 'T0_SHELLCHECK=NOT_APPLICABLE\n'+fi++for package in "${package_dirs[@]}"; do+ printf '[minikube-t0] affected package: %s test\n' "$package"+ (cd "$PROJECT_DIR/$package" && npm test)+ printf '[minikube-t0] affected package: %s build/typecheck\n' "$package"+ (cd "$PROJECT_DIR/$package" && npm run build)+done++printf 'T0_AFFECTED_PACKAGES=%s\n' "${package_dirs[*]:-none}"+printf 'T0_AFFECTED_CHECKS=PASS\n'diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex b25186628..45dbc7627 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -63,6 +63,7 @@ T2_LOCK_KEY=""T2_EVIDENCE_DIR=""T2_EVIDENCE_FILE=""T2_LOCK_HELD=false+T2_LOCK_RELEASED=falseT2_LOCK_TOKEN="$T2_LOCK_TOKEN"T2_RUN_ID="$T2_RUN_ID"T2_GATE_ID="$T2_GATE_ID"@@ -489,7 +490,7 @@ t2_deployment_check() {T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is unavailable'fi- if ! unready="$(python3 - "$deployment_json" <<'PY'+ if ! unready="$(python3 - "$deployment_json" 2>/dev/null <<'PY'import jsonimport syspayload = json.loads(sys.argv[1])@@ -708,6 +709,7 @@ PROCESS_START=$process_startSTARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)EOFT2_LOCK_HELD=true+ T2_LOCK_RELEASED=false}t2_lock_validate_inherited() {@@ -746,6 +748,7 @@ t2_lock_validate_inherited() {t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'fiT2_LOCK_HELD=false+ T2_LOCK_RELEASED=false}t2_mutation_lock() {@@ -759,7 +762,10 @@ t2_mutation_lock() {t2_lock_release() {local incoming_status=$? status owner_pid owner_tokenif [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fi- trap - EXIT INT TERM+ if [ "$T2_LOCK_RELEASED" = true ]; then+ return "$status"+ fi+ T2_LOCK_RELEASED=trueif [ "$T2_LOCK_HELD" = true ] && [ -n "$T2_LOCK_DIR" ]; thenowner_pid="$(t2_lock_owner_value PID || true)"owner_token="$(t2_lock_owner_value TOKEN || true)"diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.shindex efccd6b02..7fb2fa672 100755--- a/scripts/minikube/t2.sh+++ b/scripts/minikube/t2.sh@@ -56,13 +56,14 @@ run_t0() {printf '[minikube-t2] T0: shell syntax and contract checks\n'git -C "$T2_PROJECT_DIR" diff --check "$T2_ORIGIN_DEV...$T2_HEAD"bash -n "$SCRIPT_DIR/t2-common.sh" "$SCRIPT_DIR/t2-preflight.sh" "$SCRIPT_DIR/t2.sh"+ T0_PROJECT_DIR="$T2_PROJECT_DIR" T0_ORIGIN_DEV="$T2_ORIGIN_DEV" T0_HEAD="$T2_HEAD" \+ bash "$T2_PROJECT_DIR/scripts/minikube/t0.sh"if [ -n "$T2_T0_COMMAND" ]; thenbash -c "$T2_T0_COMMAND"- else- bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"fi+ bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"T2_T0_STATUS=PASS- t2_evidence_write T0 PASS 'syntax, contract, and diff checks passed'+ t2_evidence_write T0 PASS 'syntax, ShellCheck when available, affected package test/build/typecheck, contract, and diff checks passed'}run_preflight_plan() {@@ -210,6 +211,20 @@ run_t1() {if T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \bash "$T2_PROJECT_DIR/scripts/e2e/minikube-real-postgres.sh" >"$T2_T1_OUTPUT" 2>&1; thencat "$T2_T1_OUTPUT"+ local t1_status t1_tests t1_passed t1_pending t1_evidence+ t1_status="$(awk -F= '$1 == "T1_STATUS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"+ t1_tests="$(awk -F= '$1 == "T1_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"+ t1_passed="$(awk -F= '$1 == "T1_PASSED_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"+ t1_pending="$(awk -F= '$1 == "T1_PENDING_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"+ t1_evidence="$(awk -F= '$1 == "T1_EVIDENCE" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"+ if [ "$t1_status" != PASS ] || ! [[ "$t1_tests" =~ ^[1-9][0-9]*$ ]] || \+ ! [[ "$t1_passed" =~ ^[1-9][0-9]*$ ]] || [ "$t1_passed" -ne "$t1_tests" ] || \+ [ "$t1_pending" != 0 ] || [ -z "$t1_evidence" ]; then+ T2_T1_STATUS=FAIL+ t2_evidence_write T1 FAIL 'Real PostgreSQL lane exited zero but did not emit a complete green T1 result'+ T2_NEXT_COMMAND='repair the T1 result contract; it must emit PASS, positive total/passed counts, zero pending tests, and an evidence path'+ t2_fail REAL_PG_SUITE_FAILED 'T1 result was incomplete despite a zero process exit'+ fiT2_T1_STATUS=PASST2_T1_COUNTS="$(awk -F= '/^T1_TESTS=|^T1_PASSED_TESTS=|^T1_PENDING_TESTS=/{printf "%s%s", (n++ ? "," : ""), $0}' "$T2_T1_OUTPUT")"t2_evidence_write T1 PASS "Real PostgreSQL suites executed with no skips; $T2_T1_COUNTS"diff --git a/scripts/minikube/verify-gfs.sh b/scripts/minikube/verify-gfs.shindex 06d95b105..bda048773 100755--- a/scripts/minikube/verify-gfs.sh+++ b/scripts/minikube/verify-gfs.sh@@ -14,12 +14,17 @@ PG_DB="${PG_DB:-profiles}"PG_HOST="${PG_HOST:-control-postgres.control-plane.svc.cluster.local}"PG_PORT="${PG_PORT:-5432}"VERIFY_ROLLOUT_TIMEOUT="${VERIFY_ROLLOUT_TIMEOUT:-60s}"+VERIFY_AUTH_RETRIES="${VERIFY_AUTH_RETRIES:-3}"+VERIFY_AUTH_RETRY_DELAY_SECONDS="${VERIFY_AUTH_RETRY_DELAY_SECONDS:-2}"kc() { kubectl --context="$CONTEXT" "$@"; }log() { printf '[verify-gfs] %s\n' "$*"; }fail() { printf '[verify-gfs] FAIL: %s\n' "$*" >&2; exit 1; }kc_read() { local out; out="$(kc "$@" 2>&1)" || fail "kubectl $* failed: $out"; printf '%s' "$out"; }+[[ "$VERIFY_AUTH_RETRIES" =~ ^[1-9][0-9]*$ ]] || fail "VERIFY_AUTH_RETRIES must be a positive integer"+[[ "$VERIFY_AUTH_RETRY_DELAY_SECONDS" =~ ^[0-9]+$ ]] || fail "VERIFY_AUTH_RETRY_DELAY_SECONDS must be a non-negative integer"+if crd_out="$(kc get crd globalfilesystems.clerum.io -o name 2>&1)"; thenlog "${crd_out} is installed"elif grep -qiE 'not ?found' <<<"$crd_out"; then@@ -41,7 +46,7 @@ kc_read -n "$GFS_NS" get configmap gfs-config -o name >/dev/nullverify_role() {local role="$1" secret="$2" deployment="$3" selector="$4"- local encoded dsn lifecycle pending rotated_at rows live+ local encoded dsn lifecycle pending rotated_at rows live auth_rc attemptlifecycle="$(kc_read -n "$GFS_NS" get secret "$secret" -o \'jsonpath={.metadata.annotations.clerum\.io/gfs-dsn-state}')"@@ -53,8 +58,27 @@ verify_role() {encoded="$(kc_read -n "$GFS_NS" get secret "$secret" -o 'jsonpath={.data.connection-string}')"[ -n "$encoded" ] || fail "${GFS_NS}/${secret}.connection-string is empty"dsn="$(printf '%s' "$encoded" | base64 -d)" || fail "${GFS_NS}/${secret} has invalid encoding"- gfs_dsn_authenticates_as "$dsn" "$role" \- || fail "${secret} cannot authenticate through the expected PostgreSQL Service and role"+ auth_rc=2+ for ((attempt=1; attempt<=VERIFY_AUTH_RETRIES; attempt++)); do+ set +e+ gfs_dsn_authenticates_as "$dsn" "$role"+ auth_rc=$?+ set -e+ if [ "$auth_rc" -eq 0 ]; then+ break+ fi+ if [ "$auth_rc" -eq 1 ]; then+ fail "${secret} was rejected by the expected PostgreSQL Service or role"+ fi+ if [ "$auth_rc" -ne 2 ]; then+ fail "${secret} authentication probe returned unexpected status ${auth_rc}"+ fi+ if [ "$attempt" -lt "$VERIFY_AUTH_RETRIES" ]; then+ log "${secret} authentication probe unavailable after rollout; retry ${attempt}/${VERIFY_AUTH_RETRIES}"+ sleep "$VERIFY_AUTH_RETRY_DELAY_SECONDS"+ fi+ done+ [ "$auth_rc" -eq 0 ] || fail "${secret} authentication probe remained unavailable after ${VERIFY_AUTH_RETRIES} attempts"unset dsnlog "${secret} authenticates as ${role}"diff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.shindex 9343cf304..c0f9f6b8a 100755--- a/scripts/minikube/wait-gfs-reader-ready.sh+++ b/scripts/minikube/wait-gfs-reader-ready.sh@@ -39,14 +39,35 @@ fideadline=$((SECONDS + TIMEOUT_SECONDS))while :; do- desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \- -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"- ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \- -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)"- if [ -n "$desired" ] && [ "$desired" != 0 ] && [ "${ready:-0}" -ge "$desired" ]; then- unready="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \+ if ! desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \+ -o jsonpath='{.spec.replicas}' 2>&1)"; then+ log "unable to read desired replicas for ${GFS_NS}/${DEPLOY}: ${desired}"+ exit 1+ fi+ if [ -z "$desired" ] || ! [[ "$desired" =~ ^[0-9]+$ ]]; then+ log "desired replicas for ${GFS_NS}/${DEPLOY} is not numeric"+ exit 1+ fi+ if ! ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \+ -o jsonpath='{.status.readyReplicas}' 2>&1)"; then+ log "unable to read Ready replicas for ${GFS_NS}/${DEPLOY}: ${ready}"+ exit 1+ fi+ ready="${ready:-0}"+ if ! [[ "$ready" =~ ^[0-9]+$ ]]; then+ log "Ready replicas for ${GFS_NS}/${DEPLOY} is not numeric"+ exit 1+ fi+ if [ "$desired" -gt 0 ] && [ "$ready" -ge "$desired" ]; then+ pod_rows=""+ unready=""+ if ! pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \'jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \- 2>/dev/null | awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }')"+ 2>&1)"; then+ log "unable to inspect reader pods for ${GFS_NS}/${DEPLOY}: ${pod_rows}"+ exit 1+ fi+ unready="$(awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }' <<<"$pod_rows")"if [ "${unready:-0}" -eq 0 ]; thenlog "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with no live unready reader pod"exit 0diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.shindex 48237d75c..8bd6bff5a 100755--- a/scripts/tests/test-minikube-sync-auth-key.sh+++ b/scripts/tests/test-minikube-sync-auth-key.sh@@ -63,7 +63,10 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-conffiif [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]]; then- [[ "${TEST_GFS_CONFIG_PRESENT:-}" == "1" ]] || exit 1+ if [[ "${TEST_GFS_CONFIG_PRESENT:-}" != "1" ]]; then+ printf 'Error from server (NotFound): configmaps "gfs-config" not found\n' >&2+ exit 1+ fiif has_output_flag "$@"; thenprintf 'old-gfs-key'fi@@ -79,20 +82,32 @@ if [[ "${1:-}" == "patch" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config"fiif [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "gfs-controller-db" ]]; then- [[ "${TEST_GFS_CONFIG_PRESENT:-}" == "1" ]] || exit 1+ if [[ "${TEST_GFS_CONFIG_PRESENT:-}" != "1" ]]; then+ printf 'Error from server (NotFound): secrets "gfs-controller-db" not found\n' >&2+ exit 1+ fiif has_output_flag "$@"; then- printf ''+ [[ "${TEST_GFS_DSN_PRESENT:-0}" == "1" ]] && printf 'ZHNuLXZhbHVl'fiexit 0fi+if [[ "${1:-}" == "get" && "${2:-}" == "deployment" && "${3:-}" == "-l" ]]; then+ [[ "${TEST_GFS_DEPLOYMENTS:-0}" == "1" ]] || exit 0+ printf 'deployment.apps/gfsc-writer\ndeployment.apps/gfsc-reader\n'+ exit 0+fi+if [[ "${1:-}" == "get" && "${2:-}" == deployment/* ]]; thenexit 0fiif [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment" && "${4:-}" == "-l" ]]; then- echo "unexpected gfsc rollout restart before DSN provisioning" >&2- exit 98+ [[ "${TEST_GFS_DSN_PRESENT:-0}" == "1" ]] || {+ echo "unexpected gfsc rollout restart before DSN provisioning" >&2+ exit 98+ }+ exit "${TEST_GFS_RESTART_STATUS:-0}"fiif [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/chatllm" ]]; then@@ -112,6 +127,10 @@ if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/fiif [[ "${1:-}" == "rollout" && "${2:-}" == "status" && "${3:-}" == deployment/* ]]; then+ if [[ "${3:-}" == *gfsc-writer* && "${TEST_GFS_STATUS_FAILURE:-0}" != "0" ]]; then+ echo 'simulated gfsc writer rollout failure' >&2+ exit "${TEST_GFS_STATUS_FAILURE}"+ fiexit 0fi@@ -247,4 +266,18 @@ if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFIfigrep -q 'patch configmap gfs-config' "${LOG_FILE}"+: >"${LOG_FILE}"+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DSN_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 TEST_GFS_STATUS_FAILURE=42 \+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then+ echo "FAIL: GFS auth sync swallowed a failed rollout status" >&2+ exit 1+fi+grep -q 'gfsc deployment deployment.apps/gfsc-writer did not become Ready after auth key drift' "${OUT_FILE}" || {+ cat "${OUT_FILE}" >&2+ echo "FAIL: GFS rollout failure did not produce a stable diagnostic" >&2+ exit 1+}+echo "PASS: auth-key sync retries transient rollout restart race and skips gfsc restart until DSN provisioning"diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex 5da0cfc9b..6cecc55f2 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -9,12 +9,14 @@ MINIKUBE_DIR="$ROOT/scripts/minikube"T1="$ROOT/scripts/e2e/minikube-real-postgres.sh"PREFLIGHT="$MINIKUBE_DIR/t2-preflight.sh"T2="$MINIKUBE_DIR/t2.sh"+T0="$MINIKUBE_DIR/t0.sh"+GFS_FILTER="$MINIKUBE_DIR/filter-gfs-resources.py"COMMON="$MINIKUBE_DIR/t2-common.sh"TMP_ROOT="$TMPDIR"if [ -z "$TMP_ROOT" ]; then TMP_ROOT=/tmp; fiset -u-for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \+for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" "$T0" \"$ROOT/scripts/e2e/e2e-np08-hcc-authorization.sh" \"$ROOT/scripts/minikube/settle-gfs-reader-rollout.sh" \"$ROOT/scripts/minikube/wait-gfs-reader-ready.sh" \@@ -28,6 +30,12 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \"$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"; dobash -n "$file"done+python3 -m py_compile "$GFS_FILTER"+grep -Fq 't0.sh' "$T2"+grep -Fq 'T0_SHELLCHECK=PASS' "$T0"+grep -Fq 'npm run build' "$T0"+grep -Fq 'npm test' "$T0"+grep -Fq 'bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"' "$T2"required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY PROFILE_LOCK_REQUIRED HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED CERTIFICATION_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT NP08_HCC_AUTHORIZATION_FAILED"for code in $required_codes; do@@ -104,10 +112,33 @@ grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$T1"grep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$T1"grep -Fq 'reconcile-gfs-deploy-credentials.sh' "$T1"grep -Fq 'T1_GFS_RESTORE_REQUIRED=true' "$T1"+grep -Fq "trap 'handle_t1_signal INT' INT" "$T1"+grep -Fq "trap 'handle_t1_signal TERM' TERM" "$T1"+grep -Fq 'T1_STATUS=%s' "$T1"+grep -Fq 'cleanup=PASS' "$T1"+if grep -Fq 'trap cleanup_t1 EXIT INT TERM' "$T1"; then+ echo 'FAIL: T1 still treats a signal as an ordinary EXIT cleanup' >&2+ exit 1+figrep -Fq 'settle-gfs-reader-rollout.sh' "$T1"grep -Fq 'gfs-rollout-shim' "$T1"grep -Fq 'T2_UNREADY_DEPLOYMENTS' "$COMMON"grep -Fq 'required gfs-controller-db Secret is missing or unreadable' "$T1"+grep -Fq 'T1_STATUS' "$T2"+grep -Fq 'T1_PENDING_TESTS' "$T2"+grep -Fq 'T1 result was incomplete despite a zero process exit' "$T2"+grep -Fq 'VERIFY_AUTH_RETRIES' "$ROOT/scripts/minikube/verify-gfs.sh"+grep -Fq 'authentication probe remained unavailable' "$ROOT/scripts/minikube/verify-gfs.sh"+grep -Fq 'GFS mutation disabled for this non-T2 sync' "$ROOT/Makefile"+grep -Fq 'filter-gfs-resources.py' "$ROOT/Makefile"+grep -Fq 'MINIKUBE_GFS_MUTATION' "$ROOT/scripts/minikube/pre-gate-sync.sh"+grep -Fq 'T2_LOCK_TOKEN="${T2_LOCK_TOKEN}"' "$ROOT/scripts/minikube/pre-gate-sync.sh"+grep -Fq 'T2_LOCK_RELEASED' "$COMMON"+grep -Fq 'leaving the writer fence armed' "$ROOT/scripts/minikube/pre-gate-sync.sh"+if grep -Fq 'gke_*)' "$ROOT/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then+ echo 'FAIL: remote GFS authorization still depends on a gke_* name prefix' >&2+ exit 1+fi# pre-gate-sync GFS serving provisioning must opt into restoring a NOLOGIN# role from the committed Secret DSN. Only the T2 transition owns thisdiff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex d7db81126..568eb76bd 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -268,6 +268,11 @@ expect_code PROFILE_UNHEALTHY unready-final-preflight unready-final-preflight \expect_code PROFILE_UNHEALTHY invalid-deployment-inventory invalid-deployment-inventory \env "${repo_env[@]}" INVALID_INVENTORY='{not-json' \bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"+if grep -Fq 'Traceback' "$tmp/invalid-deployment-inventory"; then+ fail 'invalid deployment inventory leaked a Python traceback instead of the stable failure contract'+fi+grep -Fq 'next:' "$tmp/invalid-deployment-inventory" ||+ fail 'invalid deployment inventory omitted the stable next-step guidance'expect_code PROFILE_UNHEALTHY malformed-deployment-inventory malformed-deployment-inventory \env "${repo_env[@]}" INVALID_INVENTORY='{"items":[{"spec":{"replicas":"not-an-integer"}}]}' \bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"diff --git a/scripts/tests/test-provision-gfs-runtime.sh b/scripts/tests/test-provision-gfs-runtime.shindex 29d80fbba..d6fa5ae77 100755--- a/scripts/tests/test-provision-gfs-runtime.sh+++ b/scripts/tests/test-provision-gfs-runtime.sh@@ -62,6 +62,11 @@ grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS="$CONTEXT"' \echo 'FAIL: post-overlay credential reconciliation did not carry explicit context authorization' >&2exit 1}+grep -Fq 'sync-auth-key.sh --context "$CONTEXT" --require-gfs' \+ "$ROOT/deploy/scripts/provision-gfs-runtime.sh" || {+ echo 'FAIL: GFS runtime provisioning does not require its auth source and target' >&2+ exit 1+}[ "$(grep -Fc "rollout status deployment/host-context-controller --timeout=${HCC_ROLLOUT_TIMEOUT_S}s" "$CALL_LOG")" -eq 3 ] || {echo "FAIL: HCC post-overlay rollout did not use its dedicated ${HCC_ROLLOUT_TIMEOUT_S}s timeout" >&2exit 1