Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: close adversarial PR 416 findings

Alfredo Lopez4609609ec68c
diff --git a/Makefile b/Makefile
index 34a206eed..c725d91df 100644
--- a/Makefile
+++ b/Makefile
@@ -155,6 +155,7 @@ minikube-stop: ## Stop minikube cluster
# pulls ~25 published images instead of building 28 from source. Build
# everything locally with `make minikube-setup-local` (or IMAGE_SOURCE=local).
IMAGE_SOURCE ?= ghcr
+MINIKUBE_GFS_MUTATION ?= true
.PHONY: minikube-setup
minikube-setup: ## Clean install from scratch, PULLING published images (IMAGE_SOURCE=local or `make minikube-setup-local` builds instead). Rebuilds the DB; REUSE_DB=true keeps it. SKIP_UIS=true omits Control/Profile UI. Runs 'prereqs' first (SKIP_PREREQS=true to bypass). Needs ADMIN_PASSWORD in .env.
@@ -223,18 +224,21 @@ minikube-detect-k8s-api-ip: ## Patch overlays/minikube/patches/k8s-api-ip.yaml w
.PHONY: minikube-deploy-all
minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
@$(MAKE) --no-print-directory minikube-detect-k8s-api-ip
- @CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-gfs-writer-secret.sh
@# Upgrade path: adopt/validate writer and stage reader before HCC cutover.
- @writer_dsn="$$(kubectl --context=$(MINIKUBE_PROFILE) -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')" || { \
- echo "[minikube-deploy-all] failed to classify the existing GFS writer Secret; refusing HCC cutover" >&2; exit 1; \
- }; \
- if [ -n "$$writer_dsn" ]; then \
- kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \
- echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \
+ @if [ "$(MINIKUBE_GFS_MUTATION)" != "true" ]; then echo "[minikube-deploy-all] GFS mutation disabled for this non-T2 sync"; fi
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-gfs-writer-secret.sh; \
+ writer_dsn="$$(kubectl --context=$(MINIKUBE_PROFILE) -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')" || { \
+ echo "[minikube-deploy-all] failed to classify the existing GFS writer Secret; refusing HCC cutover" >&2; exit 1; \
}; \
- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
- else \
- echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \
+ if [[ -n "$$writer_dsn" ]]; then \
+ kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1 || { \
+ echo "[minikube-deploy-all] existing GFS writer detected but control-api is not Ready; refusing HCC cutover" >&2; exit 1; \
+ }; \
+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
+ else \
+ echo "[minikube-deploy-all] fresh bootstrap: reader staging deferred until post-migration full-setup (GFSC fail-closed)"; \
+ fi; \
fi
@# THE OVERLAY FOLLOWS THE CLUSTER, NOT THIS SHELL. Hardcoding
@# deploy/overlays/minikube applied clerum/*:test image refs to a cluster
@@ -246,14 +250,22 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
@# above, because an overridden tag renders from a copy of deploy/ that must
@# already contain the generated k8s-api-ip.yaml.
render_dir="$$(bash scripts/minikube/image-mode.sh --render-dir)" && \
- kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | kubectl --context=$(MINIKUBE_PROFILE) apply -f -
+ if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | kubectl --context=$(MINIKUBE_PROFILE) apply -f -; \
+ else \
+ kubectl --context=$(MINIKUBE_PROFILE) kustomize "$$render_dir" | python3 scripts/minikube/filter-gfs-resources.py | kubectl --context=$(MINIKUBE_PROFILE) apply -f -; \
+ fi
CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/apply-inter-service-tokens.sh
- $(KC) apply -f deploy/overlays/minikube/instances/
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ $(KC) apply -f deploy/overlays/minikube/instances/; \
+ else \
+ python3 scripts/minikube/filter-gfs-resources.py deploy/overlays/minikube/instances/*.yaml | $(KC) apply -f -; \
+ fi
@# Kustomize reapplies the persisted mcp-host ConfigMap, which can overwrite
@# CLERUM_AUTH_JWT_PUBLIC_KEY with an older repo value. Always re-sync from
@# rpc-proxy-secrets after each full overlay apply so Desktop/rpc-proxy/mcp-host
@# stay on the same JWT validation key.
- @$(MAKE) --no-print-directory minikube-sync-auth-key
+ @MINIKUBE_GFS_MUTATION="$(MINIKUBE_GFS_MUTATION)" $(MAKE) --no-print-directory minikube-sync-auth-key
@# The pre-overlay helper migrates legacy last-applied ownership without
@# removing the provisioning-owned connection-string. When the GFS stack is
@# deployed AND control-api is Ready (migration 0048 applied), re-provision
@@ -262,13 +274,25 @@ minikube-deploy-all: ## Deploy ALL services via Kustomize minikube overlay
@# control-api is not Ready yet (fresh cluster mid-setup), provisioning is
@# deferred LOUDLY to the full-setup/pre-gate-sync flow that already orders
@# it after control-api migrations.
- @if kubectl --context=$(MINIKUBE_PROFILE) get configmap gfs-config -n gfs >/dev/null 2>&1; then \
- if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \
- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/wait-gfsc-secret-references.sh; \
- CONTEXT=$(MINIKUBE_PROFILE) bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
- else \
- echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ gfs_config_probe="$$(kubectl --context=$(MINIKUBE_PROFILE) get configmap gfs-config -n gfs 2>&1)" || { \
+ if [[ "$$gfs_config_probe" == *NotFound* || "$$gfs_config_probe" == *"not found"* ]]; then \
+ echo "[minikube-deploy-all] GFS is not deployed; skipping post-overlay credential reconciliation"; \
+ gfs_config_probe=""; \
+ else \
+ echo "[minikube-deploy-all] unable to inspect GFS configmap; refusing to continue: $$gfs_config_probe" >&2; exit 1; \
+ fi; \
+ }; \
+ if [ -n "$$gfs_config_probe" ]; then \
+ if kubectl --context=$(MINIKUBE_PROFILE) -n control-plane rollout status deployment/control-api --timeout=5s >/dev/null 2>&1; then \
+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/wait-gfsc-secret-references.sh; \
+ CONTEXT=$(MINIKUBE_PROFILE) T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" bash deploy/scripts/reconcile-gfs-deploy-credentials.sh; \
+ else \
+ echo "[minikube-deploy-all] control-api not Ready — gfs DSN provisioning DEFERRED to full-setup/pre-gate-sync ordering (gfsc stays fail-closed until then)"; \
+ fi; \
fi; \
+ else \
+ echo "[minikube-deploy-all] skipping post-overlay GFS credential reconciliation"; \
fi
.PHONY: minikube-verify-networkpolicies
@@ -364,7 +388,11 @@ minikube-apply-namespaces: ## Create all namespaces
.PHONY: minikube-sync-auth-key
minikube-sync-auth-key: ## Sync JWT public key from rpc-proxy-secrets into runtime ConfigMaps when drift exists
- @bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE)
+ @if [ "$(MINIKUBE_GFS_MUTATION)" = "true" ]; then \
+ bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE); \
+ else \
+ bash scripts/minikube/sync-auth-key.sh --context=$(MINIKUBE_PROFILE) --skip-gfs --require-mcp; \
+ fi
.PHONY: minikube-sync-auth-key-if-present
minikube-sync-auth-key-if-present: ## Sync JWT public key only when minikube auth resources already exist
diff --git a/deploy/scripts/provision-gfs-runtime.sh b/deploy/scripts/provision-gfs-runtime.sh
index c4895b1db..bcc7f3527 100755
--- a/deploy/scripts/provision-gfs-runtime.sh
+++ b/deploy/scripts/provision-gfs-runtime.sh
@@ -102,7 +102,7 @@ context_allowed_via_env() {
}
case "$CONTEXT" in
- *example-dev*|minikube|clerum-test|clerum-codex-*|clerum-detached-*|clerum-feat-*|clerum-pr-*)
+ minikube|clerum-test|clerum-codex-*|clerum-detached-*|clerum-feat-*|clerum-pr-*)
;;
*)
# A non-dev context passes only if explicitly named in ALLOWED_CONTEXTS or
@@ -214,7 +214,7 @@ if [ "$SKIP_SYNC" = "1" ]; then
log "Skipping runtime auth configuration sync"
else
log "Syncing runtime auth configuration"
- bash scripts/minikube/sync-auth-key.sh --context "$CONTEXT"
+ bash scripts/minikube/sync-auth-key.sh --context "$CONTEXT" --require-gfs
fi
if [ "$SKIP_INSTANCES" = "1" ]; then
diff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
index daede2656..76f6cd279 100755
--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh
+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
@@ -33,15 +33,40 @@ remote_context_allowed() {
return 1
}
-if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == clerum-* ]]; then
+if [ -n "$T2_LOCK_TOKEN" ] || [[ "$CONTEXT" == clerum-* ]]; then
# shellcheck source=scripts/minikube/t2-common.sh
source "$ROOT/scripts/minikube/t2-common.sh"
if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
+ RECONCILE_CLEANUP_DONE=false
cleanup_reconcile() {
+ local status="${1:-$?}"
+ if [ "$RECONCILE_CLEANUP_DONE" = true ]; then
+ return "$status"
+ fi
+ RECONCILE_CLEANUP_DONE=true
+ trap - EXIT
+ trap '' INT TERM
+ t2_lock_release "$status" || status=$?
+ return "$status"
+ }
+ handle_reconcile_signal() {
+ local signal="$1" status
+ case "$signal" in
+ INT) status=130 ;;
+ TERM) status=143 ;;
+ *) status=1 ;;
+ esac
+ cleanup_reconcile "$status" || status=$?
+ exit "$status"
+ }
+ handle_reconcile_exit() {
local status=$?
- t2_lock_release "$status"
+ cleanup_reconcile "$status" || status=$?
+ exit "$status"
}
- trap cleanup_reconcile EXIT INT TERM
+ trap handle_reconcile_exit EXIT
+ trap 'handle_reconcile_signal INT' INT
+ trap 'handle_reconcile_signal TERM' TERM
t2_repo_metadata
t2_profile_scope
t2_context_check
@@ -50,22 +75,17 @@ if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == cle
# a second CONTEXT value after the fence has been acquired.
CONTEXT="$T2_CONTEXT"
else
- case "$CONTEXT" in
- gke_*)
- # GKE deploys are owned by the explicitly selected deploy workflow rather
- # than the local Minikube lease. The workflow must opt in explicitly and
- # pass the exact context allowlist; the name prefix alone is not
- # authorization to mutate a remote cluster.
- if ! remote_context_allowed; then
- printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2
- exit 1
- fi
- ;;
- *)
+ # Remote reconciliation is authorized by the explicit workflow decision and
+ # exact context membership, never by a provider/name convention. This also
+ # supports self-hosted dev contexts whose names are not gke_*.
+ if ! remote_context_allowed; then
+ if [ -z "${GFS_REMOTE_RECONCILE_AUTHORIZED:-}" ] && [ -z "${ALLOWED_CONTEXTS:-}" ]; then
printf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2
exit 1
- ;;
- esac
+ fi
+ printf '[reconcile-gfs-deploy] ERROR: refusing remote context without explicit authorization and exact ALLOWED_CONTEXTS membership: %s\n' "$CONTEXT" >&2
+ exit 1
+ fi
fi
CONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.sh
index b7eae25aa..fb37ef25e 100755
--- a/scripts/e2e/minikube-real-postgres.sh
+++ b/scripts/e2e/minikube-real-postgres.sh
@@ -38,6 +38,7 @@ T1_PENDING_TESTS=0
# (e.g. a failed precondition, or this file being sourced by a harness test).
T1_GFS_RESTORE_REQUIRED=false
T1_CLEANUP_DONE=false
+T1_SIGNAL_STATUS=0
die_t1() {
local code="$1"
@@ -96,13 +97,27 @@ restore_gfs_runtime_credentials() {
fi
}
+handle_t1_signal() {
+ local signal="$1" status
+ case "$signal" in
+ INT) status=130 ;;
+ TERM) status=143 ;;
+ *) status=1 ;;
+ esac
+ T1_SIGNAL_STATUS="$status"
+ cleanup_t1 "$status"
+}
+
cleanup_t1() {
- local status=$?
+ local status="${1:-$?}"
if [ "$T1_CLEANUP_DONE" = true ]; then
return 0
fi
T1_CLEANUP_DONE=true
- trap - EXIT INT TERM
+ # Finish the safety restore once it has started. A second signal must not
+ # interrupt the lock release and leave a half-restored profile behind.
+ trap - EXIT
+ trap '' INT TERM
if [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
local command_line
command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"
@@ -116,10 +131,18 @@ cleanup_t1() {
status=1
fi
if [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi
+ if [ "$status" -eq 0 ] && [ "$T1_STATUS" = PASS ]; then
+ t2_evidence_write T1 PASS "Real PostgreSQL suites passed; tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS; cleanup=PASS"
+ t2_evidence_write complete PASS "T1=PASS tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS cleanup=PASS"
+ elif [ "$T1_STATUS" = PASS ]; then
+ t2_evidence_write T1 FAIL "Real PostgreSQL suites passed but cleanup failed; cleanup=FAIL"
+ fi
t2_lock_release "$status" || status=$?
exit "$status"
}
-trap cleanup_t1 EXIT INT TERM
+trap cleanup_t1 EXIT
+trap 'handle_t1_signal INT' INT
+trap 'handle_t1_signal TERM' TERM
require_t1_commands() {
local command_name
@@ -456,8 +479,9 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.dec
unset ADMIN_DSN ISOLATED_DSN
stop_isolated_postgres
T1_STATUS=PASS
- t2_evidence_write T1 PASS "Real PostgreSQL suites passed; tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS"
- t2_evidence_write complete PASS "T1=PASS tests=$T1_TOTAL_TESTS passed=$T1_PASSED_TESTS pending=$T1_PENDING_TESTS"
+ # PASS evidence is finalized by cleanup_t1 only after the profile restore and
+ # lock release path has completed successfully.
+ printf 'T1_STATUS=%s\n' "$T1_STATUS"
printf 'T1_TESTS=%s\n' "$T1_TOTAL_TESTS"
printf 'T1_PASSED_TESTS=%s\n' "$T1_PASSED_TESTS"
printf 'T1_PENDING_TESTS=%s\n' "$T1_PENDING_TESTS"
diff --git a/scripts/minikube/filter-gfs-resources.py b/scripts/minikube/filter-gfs-resources.py
new file mode 100755
index 000000000..db6e35420
--- /dev/null
+++ b/scripts/minikube/filter-gfs-resources.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Remove GFS-owned Kubernetes documents from an intentionally non-GFS apply.
+
+This is used only by non-T2 pre-gate deployment syncs. The filter is explicit
+about the small set of cluster-scoped and control-plane GFS boundary objects;
+unknown documents are retained rather than guessed away.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+
+def field(document: str, expression: str) -> str:
+ match = re.search(expression, document, flags=re.MULTILINE)
+ return match.group(1) if match else ""
+
+
+def is_gfs_owned(document: str) -> bool:
+ kind = field(document, r"^kind:\s*([^\s#]+)\s*$")
+ name = field(document, r"^ name:\s*([^\s#]+)\s*$")
+ namespace = field(document, r"^ namespace:\s*([^\s#]+)\s*$")
+ if kind == "Namespace" and name == "gfs":
+ return True
+ if namespace == "gfs" or kind == "GlobalFileSystem":
+ return True
+ if kind == "CustomResourceDefinition" and name == "globalfilesystems.clerum.io":
+ return True
+ if kind == "NetworkPolicy" and name in {
+ "control-api-to-gfsc",
+ "control-postgres-from-gfs-controller",
+ "deny-all-gfs",
+ }:
+ return True
+ return False
+
+
+def read_documents(paths: list[str]) -> str:
+ if not paths:
+ return sys.stdin.read()
+ return "\n---\n".join(Path(path).read_text() for path in paths)
+
+
+def main() -> int:
+ source = read_documents(sys.argv[1:])
+ documents = re.split(r"(?m)^---\s*$", source)
+ kept = [document.rstrip() for document in documents if document.strip() and not is_gfs_owned(document)]
+ if kept:
+ sys.stdout.write("\n---\n".join(kept) + "\n")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.sh
index 6a482c709..a375528f7 100755
--- a/scripts/minikube/pre-gate-sync.sh
+++ b/scripts/minikube/pre-gate-sync.sh
@@ -115,9 +115,14 @@ CONTROL_API_FENCED=false
CONTROL_API_REPLICAS=1
fence_workflow_reconciler() {
- if ! ${KC} get deployment/workflow-recipes -n control-plane >/dev/null 2>&1; then
- log "Workflow reconciler is not present; no rollout fence is required"
- return 0
+ local deployment_probe
+ if ! deployment_probe="$(${KC} get deployment/workflow-recipes -n control-plane 2>&1)"; then
+ if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
+ log "Workflow reconciler is not present; no rollout fence is required"
+ return 0
+ fi
+ log "ERROR: unable to inspect workflow-recipes before schema migration; refusing to continue without its writer fence"
+ return 1
fi
WRC_REPLICAS="$(${KC} get deployment/workflow-recipes -n control-plane -o jsonpath='{.spec.replicas}')"
WRC_REPLICAS="${WRC_REPLICAS:-1}"
@@ -147,9 +152,14 @@ restore_workflow_reconciler() {
}
fence_control_api() {
- if ! ${KC} get deployment/control-api -n control-plane >/dev/null 2>&1; then
- log "Control API is not present; no writer fence is required"
- return 0
+ local deployment_probe
+ if ! deployment_probe="$(${KC} get deployment/control-api -n control-plane 2>&1)"; then
+ if [[ "${deployment_probe}" == *NotFound* || "${deployment_probe}" == *"not found"* ]]; then
+ log "Control API is not present; no writer fence is required"
+ return 0
+ fi
+ log "ERROR: unable to inspect control-api before schema migration; refusing to continue without its writer fence"
+ return 1
fi
CONTROL_API_REPLICAS="$(${KC} get deployment/control-api -n control-plane -o jsonpath='{.spec.replicas}')"
CONTROL_API_REPLICAS="${CONTROL_API_REPLICAS:-1}"
@@ -320,17 +330,26 @@ ensure_artifact() {
}
sync_mcp_host_auth_key() {
- if ! ${KC} get secret rpc-proxy-secrets -n rpc-proxy >/dev/null 2>&1; then
- log "Skipping mcp-host auth key sync (rpc-proxy-secrets not found)"
- return 0
+ local source_probe target_probe
+ if ! source_probe="$(${KC} get secret rpc-proxy-secrets -n rpc-proxy 2>&1)"; then
+ if [[ "${source_probe}" == *NotFound* || "${source_probe}" == *"not found"* ]]; then
+ log "Skipping mcp-host auth key sync (rpc-proxy-secrets not found)"
+ return 0
+ fi
+ log "ERROR: unable to inspect rpc-proxy/rpc-proxy-secrets before MCP auth sync"
+ return 1
fi
- if ! ${KC} get configmap mcp-host-config -n mcp-host >/dev/null 2>&1; then
- log "Skipping mcp-host auth key sync (mcp-host-config not found)"
- return 0
+ if ! target_probe="$(${KC} get configmap mcp-host-config -n mcp-host 2>&1)"; then
+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then
+ log "Skipping mcp-host auth key sync (mcp-host-config not found)"
+ return 0
+ fi
+ log "ERROR: unable to inspect mcp-host/mcp-host-config before MCP auth sync"
+ return 1
fi
- bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --skip-gfs
+ bash "${PROJECT_DIR}/scripts/minikube/sync-auth-key.sh" --context "${PROFILE}" --skip-gfs --require-mcp
}
sync_gfs_auth_key() {
@@ -385,6 +404,10 @@ converge_gfs_reader_after_restore() {
log "ERROR: unable to read Ready replicas for gfs/${deployment}"
return 1
fi
+ # Kubernetes omits status.readyReplicas while a Deployment has zero Ready
+ # pods. An empty successful read is a real 0; an API failure above remains
+ # unknown and fails closed.
+ ready="${ready:-0}"
if [[ ! "${ready}" =~ ^[0-9]+$ ]]; then
log "ERROR: Ready replicas for gfs/${deployment} is not numeric"
return 1
@@ -576,7 +599,11 @@ if [[ "${cluster_changed}" == "true" ]]; then
cd "${PROJECT_DIR}"
make minikube-deploy-crds
)
- CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
+ if [[ "${GATE_NAME}" == "minikube-t2" ]]; then
+ CONTEXT="${PROFILE}" bash "${PROJECT_DIR}/deploy/scripts/apply-gfs-writer-secret.sh"
+ else
+ log "Skipping GFS writer Secret mutation before ${GATE_NAME}; only minikube-t2 owns GFS recovery"
+ fi
writer_dsn="$(${KC} -n gfs get secret gfs-controller-db -o 'jsonpath={.data.connection-string}')"
if [[ -n "${writer_dsn}" ]]; then
# Do not reconcile the GFS roles before the migration window. The
@@ -643,7 +670,10 @@ if [[ "${cluster_changed}" == "true" ]]; then
(
cd "${PROJECT_DIR}"
make minikube-apply-secrets
- make minikube-deploy-all
+ gfs_mutation=false
+ [[ "${GATE_NAME}" == "minikube-t2" ]] && gfs_mutation=true
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="${T2_LOCK_TOKEN}" MINIKUBE_GFS_MUTATION="${gfs_mutation}" \
+ make minikube-deploy-all
if [[ "${FORCE_RESTART}" == "true" || "${INCREMENTAL_FULL_IMAGE_BUILD}" == "true" ]]; then
make minikube-restart-all
fi
diff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.sh
index bb6680103..0d27d5976 100755
--- a/scripts/minikube/settle-gfs-reader-rollout.sh
+++ b/scripts/minikube/settle-gfs-reader-rollout.sh
@@ -41,11 +41,37 @@ log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }
die() { log "ERROR: $*"; exit 1; }
kc() { kubectl --context="$T2_CONTEXT" "$@"; }
+SETTLE_CLEANUP_DONE=false
+
cleanup_settle() {
+ local status="${1:-$?}"
+ if [ "$SETTLE_CLEANUP_DONE" = true ]; then
+ return "$status"
+ fi
+ SETTLE_CLEANUP_DONE=true
+ trap - EXIT
+ trap '' INT TERM
+ t2_lock_release "$status" || status=$?
+ return "$status"
+}
+handle_settle_signal() {
+ local signal="$1" status
+ case "$signal" in
+ INT) status=130 ;;
+ TERM) status=143 ;;
+ *) status=1 ;;
+ esac
+ cleanup_settle "$status" || status=$?
+ exit "$status"
+}
+handle_settle_exit() {
local status=$?
- t2_lock_release "$status"
+ cleanup_settle "$status" || status=$?
+ exit "$status"
}
-trap cleanup_settle EXIT INT TERM
+trap handle_settle_exit EXIT
+trap 'handle_settle_signal INT' INT
+trap 'handle_settle_signal TERM' TERM
t2_repo_metadata
t2_profile_scope
t2_context_check
@@ -58,13 +84,31 @@ source "$ROOT/deploy/scripts/lib/gfs-credential-secret.sh"
# shellcheck source=deploy/scripts/lib/gfs-credential-rollout.sh
source "$ROOT/deploy/scripts/lib/gfs-credential-rollout.sh"
-if ! kc -n "$GFS_NS" get deployment "$DEPLOY" >/dev/null 2>&1; then
- exit 0
+deployment_probe=""
+if ! deployment_probe="$(kc -n "$GFS_NS" get deployment "$DEPLOY" 2>&1)"; then
+ if [[ "$deployment_probe" == *NotFound* || "$deployment_probe" == *"not found"* ]]; then
+ exit 0
+ fi
+ die "unable to inspect gfs/${DEPLOY}: ${deployment_probe}"
fi
-desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.spec.replicas}')"
-ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.status.readyReplicas}')"
-if [ -z "$desired" ] || [ "$desired" = "0" ] || [ "${ready:-0}" != "$desired" ]; then
+if ! desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.spec.replicas}' 2>&1)"; then
+ die "unable to read desired replicas for gfs/${DEPLOY}: ${desired}"
+fi
+if [ -z "$desired" ] || ! [[ "$desired" =~ ^[0-9]+$ ]]; then
+ die "desired replicas for gfs/${DEPLOY} is not numeric"
+fi
+if [ "$desired" = "0" ]; then
+ exit 0
+fi
+if ! ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" -o jsonpath='{.status.readyReplicas}' 2>&1)"; then
+ die "unable to read Ready replicas for gfs/${DEPLOY}: ${ready}"
+fi
+ready="${ready:-0}"
+if ! [[ "$ready" =~ ^[0-9]+$ ]]; then
+ die "Ready replicas for gfs/${DEPLOY} is not numeric"
+fi
+if [ "$ready" != "$desired" ]; then
exit 0
fi
@@ -73,7 +117,7 @@ current_rev="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
-o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
rs_rows="$(kc -n "$GFS_NS" get rs -l "$SELECTOR" -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.spec.replicas}{"|"}{.status.readyReplicas}{"|"}{.metadata.annotations.deployment\.kubernetes\.io/revision}{"\n"}{end}' \
- 2>/dev/null || true)"
+ 2>/dev/null)"
while IFS='|' read -r rs_name rs_owner rs_replicas rs_ready rs_rev; do
[ -n "$rs_name" ] || continue
[ "$rs_owner" = "$DEPLOY" ] || continue
@@ -87,7 +131,7 @@ done <<<"$rs_rows"
pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \
'jsonpath={range .items[*]}{.metadata.name}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.containerStatuses[0].state.waiting.reason}{"\n"}{end}' \
- 2>/dev/null || true)"
+ 2>/dev/null)"
while IFS='|' read -r pod_name pod_ready pod_deleting pod_reason; do
[ -n "$pod_name" ] || continue
[ -z "$pod_deleting" ] || continue
@@ -97,8 +141,12 @@ while IFS='|' read -r pod_name pod_ready pod_deleting pod_reason; do
kc -n "$GFS_NS" delete pod "$pod_name" --wait=false >/dev/null
done <<<"$pod_rows"
-if ! kc -n "$GFS_NS" get secret "$SECRET" >/dev/null 2>&1; then
- exit 0
+secret_probe=""
+if ! secret_probe="$(kc -n "$GFS_NS" get secret "$SECRET" 2>&1)"; then
+ if [[ "$secret_probe" == *NotFound* || "$secret_probe" == *"not found"* ]]; then
+ exit 0
+ fi
+ die "unable to inspect ${GFS_NS}/${SECRET}: ${secret_probe}"
fi
state="$(kc -n "$GFS_NS" get secret "$SECRET" \
diff --git a/scripts/minikube/sync-auth-key.sh b/scripts/minikube/sync-auth-key.sh
index 661e9b978..ad9168795 100755
--- a/scripts/minikube/sync-auth-key.sh
+++ b/scripts/minikube/sync-auth-key.sh
@@ -1,8 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
PROFILE="clerum-test"
REQUIRE_GFS=false
+REQUIRE_MCP=false
SYNC_GFS=true
while [[ $# -gt 0 ]]; do
@@ -22,6 +24,10 @@ while [[ $# -gt 0 ]]; do
REQUIRE_GFS=true
shift
;;
+ --require-mcp)
+ REQUIRE_MCP=true
+ shift
+ ;;
--skip-gfs)
# Pre-gate security lanes need the MCP host key refresh without mutating
# the optional GFS plane. The T2 GFS path uses --require-gfs instead.
@@ -29,7 +35,7 @@ while [[ $# -gt 0 ]]; do
shift
;;
-h|--help)
- echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-gfs|--skip-gfs]" >&2
+ echo "Usage: scripts/minikube/sync-auth-key.sh [--context <kubectl-context>] [--require-mcp] [--require-gfs|--skip-gfs]" >&2
exit 0
;;
*)
@@ -101,12 +107,19 @@ PY
# The public key is the SAME for every consumer (one platform keypair), so it is
# fetched once and synced into each consumer's ConfigMap below.
-if ! "${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" >/dev/null 2>&1; then
- if [[ "${REQUIRE_GFS}" == "true" ]]; then
- die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"
+source_probe=""
+if ! source_probe="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" 2>&1)"; then
+ if [[ "${REQUIRE_GFS}" == "true" || "${REQUIRE_MCP}" == "true" ]]; then
+ if [[ "${REQUIRE_GFS}" == "true" ]]; then
+ die "required GFS auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"
+ fi
+ die "required MCP auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET} is missing or unreadable"
+ fi
+ if [ -z "${source_probe}" ] || [[ "${source_probe}" == *NotFound* || "${source_probe}" == *"not found"* ]]; then
+ log "Skipping auth key sync (${SOURCE_SECRET} not found)"
+ exit 0
fi
- log "Skipping auth key sync (${SOURCE_SECRET} not found)"
- exit 0
+ die "cannot inspect auth source ${SOURCE_NAMESPACE}/${SOURCE_SECRET}: ${source_probe}"
fi
source_key="$("${KCTL[@]}" get secret "${SOURCE_SECRET}" -n "${SOURCE_NAMESPACE}" -o "jsonpath={.data.${SOURCE_KEY}}" | base64 -d)"
if [[ -z "${source_key}" ]]; then
@@ -116,18 +129,31 @@ if [[ -z "${source_key}" ]]; then
die "auth source key ${SOURCE_KEY} is empty; refusing to mutate consumers"
fi
if [[ "${REQUIRE_GFS}" == "true" ]] && \
- ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then
- die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable"
+ ! gfs_target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; then
+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} is missing or unreadable: ${gfs_target_probe}"
+fi
+if [[ "${REQUIRE_MCP}" == "true" ]] && \
+ ! mcp_target_probe="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" 2>&1)"; then
+ die "required MCP auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} is missing or unreadable: ${mcp_target_probe}"
fi
# ── Target 1: mcp-host-config (restart named chatllm/mcp-host with retry) ──────
sync_mcp_host() {
- if ! "${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" >/dev/null 2>&1; then
- log "Skipping ${CONFIGMAP} sync (not found)"
- return 0
+ local target_probe=""
+ if ! target_probe="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" 2>&1)"; then
+ if [[ "${REQUIRE_MCP}" == "true" ]]; then
+ die "required MCP auth target ${CONFIG_NAMESPACE}/${CONFIGMAP} disappeared or became unreadable during sync"
+ fi
+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then
+ log "Skipping ${CONFIGMAP} sync (not found)"
+ return 0
+ fi
+ die "cannot inspect auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${target_probe}"
fi
local current_key
- current_key="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" -o "jsonpath={.data.${TARGET_KEY}}" 2>/dev/null || true)"
+ if ! current_key="$("${KCTL[@]}" get configmap "${CONFIGMAP}" -n "${CONFIG_NAMESPACE}" -o "jsonpath={.data.${TARGET_KEY}}" 2>&1)"; then
+ die "cannot read auth target ${CONFIG_NAMESPACE}/${CONFIGMAP}: ${current_key}"
+ fi
if [[ "${source_key}" == "${current_key}" ]]; then
log "${CONFIGMAP}.${TARGET_KEY} already matches source; no patch needed"
return 0
@@ -150,33 +176,62 @@ sync_mcp_host() {
# ── Target 2: gfs-config (gfsc fails closed without the verification key) ──────
sync_gfs() {
- if ! "${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" >/dev/null 2>&1; then
+ local target_probe=""
+ if ! target_probe="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" 2>&1)"; then
if [[ "${REQUIRE_GFS}" == "true" ]]; then
- die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared or became unreadable during sync"
+ die "required GFS auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP} disappeared or became unreadable during sync: ${target_probe}"
fi
- log "Skipping ${GFS_CONFIGMAP} sync (not found)"
- return 0
+ if [[ "${target_probe}" == *NotFound* || "${target_probe}" == *"not found"* ]]; then
+ log "Skipping ${GFS_CONFIGMAP} sync (not found)"
+ return 0
+ fi
+ die "cannot inspect auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${target_probe}"
fi
local current_key
- current_key="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" -o "jsonpath={.data.${GFS_TARGET_KEY}}" 2>/dev/null || true)"
+ if ! current_key="$("${KCTL[@]}" get configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" -o "jsonpath={.data.${GFS_TARGET_KEY}}" 2>&1)"; then
+ die "cannot read auth target ${GFS_NAMESPACE}/${GFS_CONFIGMAP}: ${current_key}"
+ fi
if [[ "${source_key}" == "${current_key}" ]]; then
log "${GFS_CONFIGMAP}.${GFS_TARGET_KEY} already matches source; no patch needed"
return 0
fi
log "Patching ${GFS_CONFIGMAP}.${GFS_TARGET_KEY}"
"${KCTL[@]}" patch configmap "${GFS_CONFIGMAP}" -n "${GFS_NAMESPACE}" --type=merge -p "$(make_patch "${source_key}" "${GFS_TARGET_KEY}")" >/dev/null
- local dsn
- dsn="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>/dev/null | base64 -d || true)"
+ local dsn_encoded dsn
+ if ! dsn_encoded="$("${KCTL[@]}" get secret gfs-controller-db -n "${GFS_NAMESPACE}" -o 'jsonpath={.data.connection-string}' 2>&1)"; then
+ if [[ "${dsn_encoded}" == *NotFound* || "${dsn_encoded}" == *"not found"* ]]; then
+ log "Skipping gfsc restart after auth key drift (gfs-controller-db Secret is not populated yet)"
+ return 0
+ fi
+ die "cannot inspect ${GFS_NAMESPACE}/gfs-controller-db before gfsc restart: ${dsn_encoded}"
+ fi
+ if [[ -z "${dsn_encoded}" ]]; then
+ log "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"
+ return 0
+ fi
+ if ! dsn="$(printf '%s' "${dsn_encoded}" | base64 -d 2>&1)"; then
+ die "${GFS_NAMESPACE}/gfs-controller-db.connection-string is not valid base64: ${dsn}"
+ fi
if [[ -z "${dsn}" ]]; then
log "Skipping gfsc restart after auth key drift (gfs-controller-db.connection-string not populated yet)"
return 0
fi
# gfsc reads the key at boot; restart the writer+reader so the new key takes
# effect (label-based — the reconciler owns the deployment names).
- if "${KCTL[@]}" get deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" -o name 2>/dev/null | grep -q .; then
+ local deployments deployment rollout_output
+ if ! deployments="$("${KCTL[@]}" get deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" -o name 2>&1)"; then
+ die "cannot inspect gfsc deployments after auth key drift: ${deployments}"
+ fi
+ if [[ -n "${deployments}" ]]; then
log "Restarting gfsc (${GFS_DEPLOY_SELECTOR}) after auth key drift"
"${KCTL[@]}" rollout restart deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" >/dev/null
- "${KCTL[@]}" rollout status deployment -l "${GFS_DEPLOY_SELECTOR}" -n "${GFS_NAMESPACE}" --timeout=90s || true
+ while IFS= read -r deployment; do
+ [[ -n "${deployment}" ]] || continue
+ if ! rollout_output="$(PATH="${SCRIPT_DIR}/gfs-rollout-shim:${PATH}" CONTEXT="${PROFILE}" \
+ "${KCTL[@]}" rollout status "${deployment}" -n "${GFS_NAMESPACE}" --timeout=90s 2>&1)"; then
+ die "gfsc deployment ${deployment} did not become Ready after auth key drift: ${rollout_output}"
+ fi
+ done <<<"${deployments}"
else
log "No gfsc deployments exist yet; auth key is synced for future pods"
fi
diff --git a/scripts/minikube/t0.sh b/scripts/minikube/t0.sh
new file mode 100755
index 000000000..1521503e8
--- /dev/null
+++ b/scripts/minikube/t0.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Canonical T0 checks beyond the structural contract suite.
+set -euo pipefail
+set +x
+
+PROJECT_DIR="${T0_PROJECT_DIR:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)}"
+ORIGIN_DEV="${T0_ORIGIN_DEV:-origin/dev}"
+HEAD="${T0_HEAD:-HEAD}"
+
+shell_files=()
+package_dirs=()
+seen_packages="|"
+while IFS= read -r path; do
+ [ -n "$path" ] || continue
+ case "$path" in
+ *.sh|*.bash) shell_files+=("$PROJECT_DIR/$path") ;;
+ esac
+ top="${path%%/*}"
+ if [ "$top" != "$path" ] && [ -f "$PROJECT_DIR/$top/package.json" ] && [[ "$seen_packages" != *"|$top|"* ]]; then
+ seen_packages="${seen_packages}${top}|"
+ package_dirs+=("$top")
+ fi
+done < <(git -C "$PROJECT_DIR" diff --name-only "$ORIGIN_DEV...$HEAD")
+
+if [ "${#shell_files[@]}" -gt 0 ]; then
+ if command -v shellcheck >/dev/null 2>&1; then
+ shellcheck --severity=error "${shell_files[@]}"
+ printf 'T0_SHELLCHECK=PASS\n'
+ else
+ printf 'T0_SHELLCHECK=NOT_AVAILABLE\n'
+ fi
+else
+ printf 'T0_SHELLCHECK=NOT_APPLICABLE\n'
+fi
+
+for package in "${package_dirs[@]}"; do
+ printf '[minikube-t0] affected package: %s test\n' "$package"
+ (cd "$PROJECT_DIR/$package" && npm test)
+ printf '[minikube-t0] affected package: %s build/typecheck\n' "$package"
+ (cd "$PROJECT_DIR/$package" && npm run build)
+done
+
+printf 'T0_AFFECTED_PACKAGES=%s\n' "${package_dirs[*]:-none}"
+printf 'T0_AFFECTED_CHECKS=PASS\n'
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index b25186628..45dbc7627 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -63,6 +63,7 @@ T2_LOCK_KEY=""
T2_EVIDENCE_DIR=""
T2_EVIDENCE_FILE=""
T2_LOCK_HELD=false
+T2_LOCK_RELEASED=false
T2_LOCK_TOKEN="$T2_LOCK_TOKEN"
T2_RUN_ID="$T2_RUN_ID"
T2_GATE_ID="$T2_GATE_ID"
@@ -489,7 +490,7 @@ t2_deployment_check() {
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is unavailable'
fi
- if ! unready="$(python3 - "$deployment_json" <<'PY'
+ if ! unready="$(python3 - "$deployment_json" 2>/dev/null <<'PY'
import json
import sys
payload = json.loads(sys.argv[1])
@@ -708,6 +709,7 @@ PROCESS_START=$process_start
STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
EOF
T2_LOCK_HELD=true
+ T2_LOCK_RELEASED=false
}
t2_lock_validate_inherited() {
@@ -746,6 +748,7 @@ t2_lock_validate_inherited() {
t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'
fi
T2_LOCK_HELD=false
+ T2_LOCK_RELEASED=false
}
t2_mutation_lock() {
@@ -759,7 +762,10 @@ t2_mutation_lock() {
t2_lock_release() {
local incoming_status=$? status owner_pid owner_token
if [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fi
- trap - EXIT INT TERM
+ if [ "$T2_LOCK_RELEASED" = true ]; then
+ return "$status"
+ fi
+ T2_LOCK_RELEASED=true
if [ "$T2_LOCK_HELD" = true ] && [ -n "$T2_LOCK_DIR" ]; then
owner_pid="$(t2_lock_owner_value PID || true)"
owner_token="$(t2_lock_owner_value TOKEN || true)"
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index efccd6b02..7fb2fa672 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -56,13 +56,14 @@ run_t0() {
printf '[minikube-t2] T0: shell syntax and contract checks\n'
git -C "$T2_PROJECT_DIR" diff --check "$T2_ORIGIN_DEV...$T2_HEAD"
bash -n "$SCRIPT_DIR/t2-common.sh" "$SCRIPT_DIR/t2-preflight.sh" "$SCRIPT_DIR/t2.sh"
+ T0_PROJECT_DIR="$T2_PROJECT_DIR" T0_ORIGIN_DEV="$T2_ORIGIN_DEV" T0_HEAD="$T2_HEAD" \
+ bash "$T2_PROJECT_DIR/scripts/minikube/t0.sh"
if [ -n "$T2_T0_COMMAND" ]; then
bash -c "$T2_T0_COMMAND"
- else
- bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"
fi
+ bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"
T2_T0_STATUS=PASS
- t2_evidence_write T0 PASS 'syntax, contract, and diff checks passed'
+ t2_evidence_write T0 PASS 'syntax, ShellCheck when available, affected package test/build/typecheck, contract, and diff checks passed'
}
run_preflight_plan() {
@@ -210,6 +211,20 @@ run_t1() {
if T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \
bash "$T2_PROJECT_DIR/scripts/e2e/minikube-real-postgres.sh" >"$T2_T1_OUTPUT" 2>&1; then
cat "$T2_T1_OUTPUT"
+ local t1_status t1_tests t1_passed t1_pending t1_evidence
+ t1_status="$(awk -F= '$1 == "T1_STATUS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"
+ t1_tests="$(awk -F= '$1 == "T1_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"
+ t1_passed="$(awk -F= '$1 == "T1_PASSED_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"
+ t1_pending="$(awk -F= '$1 == "T1_PENDING_TESTS" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"
+ t1_evidence="$(awk -F= '$1 == "T1_EVIDENCE" { value=$2 } END { print value }' "$T2_T1_OUTPUT")"
+ if [ "$t1_status" != PASS ] || ! [[ "$t1_tests" =~ ^[1-9][0-9]*$ ]] || \
+ ! [[ "$t1_passed" =~ ^[1-9][0-9]*$ ]] || [ "$t1_passed" -ne "$t1_tests" ] || \
+ [ "$t1_pending" != 0 ] || [ -z "$t1_evidence" ]; then
+ T2_T1_STATUS=FAIL
+ t2_evidence_write T1 FAIL 'Real PostgreSQL lane exited zero but did not emit a complete green T1 result'
+ T2_NEXT_COMMAND='repair the T1 result contract; it must emit PASS, positive total/passed counts, zero pending tests, and an evidence path'
+ t2_fail REAL_PG_SUITE_FAILED 'T1 result was incomplete despite a zero process exit'
+ fi
T2_T1_STATUS=PASS
T2_T1_COUNTS="$(awk -F= '/^T1_TESTS=|^T1_PASSED_TESTS=|^T1_PENDING_TESTS=/{printf "%s%s", (n++ ? "," : ""), $0}' "$T2_T1_OUTPUT")"
t2_evidence_write T1 PASS "Real PostgreSQL suites executed with no skips; $T2_T1_COUNTS"
diff --git a/scripts/minikube/verify-gfs.sh b/scripts/minikube/verify-gfs.sh
index 06d95b105..bda048773 100755
--- a/scripts/minikube/verify-gfs.sh
+++ b/scripts/minikube/verify-gfs.sh
@@ -14,12 +14,17 @@ PG_DB="${PG_DB:-profiles}"
PG_HOST="${PG_HOST:-control-postgres.control-plane.svc.cluster.local}"
PG_PORT="${PG_PORT:-5432}"
VERIFY_ROLLOUT_TIMEOUT="${VERIFY_ROLLOUT_TIMEOUT:-60s}"
+VERIFY_AUTH_RETRIES="${VERIFY_AUTH_RETRIES:-3}"
+VERIFY_AUTH_RETRY_DELAY_SECONDS="${VERIFY_AUTH_RETRY_DELAY_SECONDS:-2}"
kc() { kubectl --context="$CONTEXT" "$@"; }
log() { printf '[verify-gfs] %s\n' "$*"; }
fail() { printf '[verify-gfs] FAIL: %s\n' "$*" >&2; exit 1; }
kc_read() { local out; out="$(kc "$@" 2>&1)" || fail "kubectl $* failed: $out"; printf '%s' "$out"; }
+[[ "$VERIFY_AUTH_RETRIES" =~ ^[1-9][0-9]*$ ]] || fail "VERIFY_AUTH_RETRIES must be a positive integer"
+[[ "$VERIFY_AUTH_RETRY_DELAY_SECONDS" =~ ^[0-9]+$ ]] || fail "VERIFY_AUTH_RETRY_DELAY_SECONDS must be a non-negative integer"
+
if crd_out="$(kc get crd globalfilesystems.clerum.io -o name 2>&1)"; then
log "${crd_out} is installed"
elif grep -qiE 'not ?found' <<<"$crd_out"; then
@@ -41,7 +46,7 @@ kc_read -n "$GFS_NS" get configmap gfs-config -o name >/dev/null
verify_role() {
local role="$1" secret="$2" deployment="$3" selector="$4"
- local encoded dsn lifecycle pending rotated_at rows live
+ local encoded dsn lifecycle pending rotated_at rows live auth_rc attempt
lifecycle="$(kc_read -n "$GFS_NS" get secret "$secret" -o \
'jsonpath={.metadata.annotations.clerum\.io/gfs-dsn-state}')"
@@ -53,8 +58,27 @@ verify_role() {
encoded="$(kc_read -n "$GFS_NS" get secret "$secret" -o 'jsonpath={.data.connection-string}')"
[ -n "$encoded" ] || fail "${GFS_NS}/${secret}.connection-string is empty"
dsn="$(printf '%s' "$encoded" | base64 -d)" || fail "${GFS_NS}/${secret} has invalid encoding"
- gfs_dsn_authenticates_as "$dsn" "$role" \
- || fail "${secret} cannot authenticate through the expected PostgreSQL Service and role"
+ auth_rc=2
+ for ((attempt=1; attempt<=VERIFY_AUTH_RETRIES; attempt++)); do
+ set +e
+ gfs_dsn_authenticates_as "$dsn" "$role"
+ auth_rc=$?
+ set -e
+ if [ "$auth_rc" -eq 0 ]; then
+ break
+ fi
+ if [ "$auth_rc" -eq 1 ]; then
+ fail "${secret} was rejected by the expected PostgreSQL Service or role"
+ fi
+ if [ "$auth_rc" -ne 2 ]; then
+ fail "${secret} authentication probe returned unexpected status ${auth_rc}"
+ fi
+ if [ "$attempt" -lt "$VERIFY_AUTH_RETRIES" ]; then
+ log "${secret} authentication probe unavailable after rollout; retry ${attempt}/${VERIFY_AUTH_RETRIES}"
+ sleep "$VERIFY_AUTH_RETRY_DELAY_SECONDS"
+ fi
+ done
+ [ "$auth_rc" -eq 0 ] || fail "${secret} authentication probe remained unavailable after ${VERIFY_AUTH_RETRIES} attempts"
unset dsn
log "${secret} authenticates as ${role}"
diff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.sh
index 9343cf304..c0f9f6b8a 100755
--- a/scripts/minikube/wait-gfs-reader-ready.sh
+++ b/scripts/minikube/wait-gfs-reader-ready.sh
@@ -39,14 +39,35 @@ fi
deadline=$((SECONDS + TIMEOUT_SECONDS))
while :; do
- desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
- -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
- ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
- -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)"
- if [ -n "$desired" ] && [ "$desired" != 0 ] && [ "${ready:-0}" -ge "$desired" ]; then
- unready="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \
+ if ! desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
+ -o jsonpath='{.spec.replicas}' 2>&1)"; then
+ log "unable to read desired replicas for ${GFS_NS}/${DEPLOY}: ${desired}"
+ exit 1
+ fi
+ if [ -z "$desired" ] || ! [[ "$desired" =~ ^[0-9]+$ ]]; then
+ log "desired replicas for ${GFS_NS}/${DEPLOY} is not numeric"
+ exit 1
+ fi
+ if ! ready="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
+ -o jsonpath='{.status.readyReplicas}' 2>&1)"; then
+ log "unable to read Ready replicas for ${GFS_NS}/${DEPLOY}: ${ready}"
+ exit 1
+ fi
+ ready="${ready:-0}"
+ if ! [[ "$ready" =~ ^[0-9]+$ ]]; then
+ log "Ready replicas for ${GFS_NS}/${DEPLOY} is not numeric"
+ exit 1
+ fi
+ if [ "$desired" -gt 0 ] && [ "$ready" -ge "$desired" ]; then
+ pod_rows=""
+ unready=""
+ if ! pod_rows="$(kc -n "$GFS_NS" get pods -l "$SELECTOR" -o \
'jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \
- 2>/dev/null | awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }')"
+ 2>&1)"; then
+ log "unable to inspect reader pods for ${GFS_NS}/${DEPLOY}: ${pod_rows}"
+ exit 1
+ fi
+ unready="$(awk -F'|' 'NF && $2 == "" && $1 != "True" { n++ } END { print n+0 }' <<<"$pod_rows")"
if [ "${unready:-0}" -eq 0 ]; then
log "${GFS_NS}/${DEPLOY} is Ready (${ready}/${desired}) with no live unready reader pod"
exit 0
diff --git a/scripts/tests/test-minikube-sync-auth-key.sh b/scripts/tests/test-minikube-sync-auth-key.sh
index 48237d75c..8bd6bff5a 100755
--- a/scripts/tests/test-minikube-sync-auth-key.sh
+++ b/scripts/tests/test-minikube-sync-auth-key.sh
@@ -63,7 +63,10 @@ if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "mcp-host-conf
fi
if [[ "${1:-}" == "get" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config" ]]; then
- [[ "${TEST_GFS_CONFIG_PRESENT:-}" == "1" ]] || exit 1
+ if [[ "${TEST_GFS_CONFIG_PRESENT:-}" != "1" ]]; then
+ printf 'Error from server (NotFound): configmaps "gfs-config" not found\n' >&2
+ exit 1
+ fi
if has_output_flag "$@"; then
printf 'old-gfs-key'
fi
@@ -79,20 +82,32 @@ if [[ "${1:-}" == "patch" && "${2:-}" == "configmap" && "${3:-}" == "gfs-config"
fi
if [[ "${1:-}" == "get" && "${2:-}" == "secret" && "${3:-}" == "gfs-controller-db" ]]; then
- [[ "${TEST_GFS_CONFIG_PRESENT:-}" == "1" ]] || exit 1
+ if [[ "${TEST_GFS_CONFIG_PRESENT:-}" != "1" ]]; then
+ printf 'Error from server (NotFound): secrets "gfs-controller-db" not found\n' >&2
+ exit 1
+ fi
if has_output_flag "$@"; then
- printf ''
+ [[ "${TEST_GFS_DSN_PRESENT:-0}" == "1" ]] && printf 'ZHNuLXZhbHVl'
fi
exit 0
fi
+if [[ "${1:-}" == "get" && "${2:-}" == "deployment" && "${3:-}" == "-l" ]]; then
+ [[ "${TEST_GFS_DEPLOYMENTS:-0}" == "1" ]] || exit 0
+ printf 'deployment.apps/gfsc-writer\ndeployment.apps/gfsc-reader\n'
+ exit 0
+fi
+
if [[ "${1:-}" == "get" && "${2:-}" == deployment/* ]]; then
exit 0
fi
if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment" && "${4:-}" == "-l" ]]; then
- echo "unexpected gfsc rollout restart before DSN provisioning" >&2
- exit 98
+ [[ "${TEST_GFS_DSN_PRESENT:-0}" == "1" ]] || {
+ echo "unexpected gfsc rollout restart before DSN provisioning" >&2
+ exit 98
+ }
+ exit "${TEST_GFS_RESTART_STATUS:-0}"
fi
if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/chatllm" ]]; then
@@ -112,6 +127,10 @@ if [[ "${1:-}" == "rollout" && "${2:-}" == "restart" && "${3:-}" == "deployment/
fi
if [[ "${1:-}" == "rollout" && "${2:-}" == "status" && "${3:-}" == deployment/* ]]; then
+ if [[ "${3:-}" == *gfsc-writer* && "${TEST_GFS_STATUS_FAILURE:-0}" != "0" ]]; then
+ echo 'simulated gfsc writer rollout failure' >&2
+ exit "${TEST_GFS_STATUS_FAILURE}"
+ fi
exit 0
fi
@@ -247,4 +266,18 @@ if ! TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" TEST_GFS_CONFI
fi
grep -q 'patch configmap gfs-config' "${LOG_FILE}"
+: >"${LOG_FILE}"
+if TEST_KUBECTL_LOG="${LOG_FILE}" TEST_STATE_DIR="${STATE_DIR}" \
+ TEST_GFS_CONFIG_PRESENT=1 TEST_GFS_DSN_PRESENT=1 TEST_GFS_DEPLOYMENTS=1 TEST_GFS_STATUS_FAILURE=42 \
+ PATH="${TMP_DIR}:$PATH" bash "${ROOT}/scripts/minikube/sync-auth-key.sh" \
+ --context fake --require-gfs >"${OUT_FILE}" 2>&1; then
+ echo "FAIL: GFS auth sync swallowed a failed rollout status" >&2
+ exit 1
+fi
+grep -q 'gfsc deployment deployment.apps/gfsc-writer did not become Ready after auth key drift' "${OUT_FILE}" || {
+ cat "${OUT_FILE}" >&2
+ echo "FAIL: GFS rollout failure did not produce a stable diagnostic" >&2
+ exit 1
+}
+
echo "PASS: auth-key sync retries transient rollout restart race and skips gfsc restart until DSN provisioning"
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 5da0cfc9b..6cecc55f2 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -9,12 +9,14 @@ MINIKUBE_DIR="$ROOT/scripts/minikube"
T1="$ROOT/scripts/e2e/minikube-real-postgres.sh"
PREFLIGHT="$MINIKUBE_DIR/t2-preflight.sh"
T2="$MINIKUBE_DIR/t2.sh"
+T0="$MINIKUBE_DIR/t0.sh"
+GFS_FILTER="$MINIKUBE_DIR/filter-gfs-resources.py"
COMMON="$MINIKUBE_DIR/t2-common.sh"
TMP_ROOT="$TMPDIR"
if [ -z "$TMP_ROOT" ]; then TMP_ROOT=/tmp; fi
set -u
-for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \
+for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" "$T0" \
"$ROOT/scripts/e2e/e2e-np08-hcc-authorization.sh" \
"$ROOT/scripts/minikube/settle-gfs-reader-rollout.sh" \
"$ROOT/scripts/minikube/wait-gfs-reader-ready.sh" \
@@ -28,6 +30,12 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \
"$ROOT/scripts/tests/test-minikube-t1-gfs-restore.sh"; do
bash -n "$file"
done
+python3 -m py_compile "$GFS_FILTER"
+grep -Fq 't0.sh' "$T2"
+grep -Fq 'T0_SHELLCHECK=PASS' "$T0"
+grep -Fq 'npm run build' "$T0"
+grep -Fq 'npm test' "$T0"
+grep -Fq 'bash "$T2_PROJECT_DIR/scripts/tests/test-minikube-t2-contract.sh"' "$T2"
required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY PROFILE_LOCK_REQUIRED HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED CERTIFICATION_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT NP08_HCC_AUTHORIZATION_FAILED"
for code in $required_codes; do
@@ -104,10 +112,33 @@ grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$T1"
grep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$T1"
grep -Fq 'reconcile-gfs-deploy-credentials.sh' "$T1"
grep -Fq 'T1_GFS_RESTORE_REQUIRED=true' "$T1"
+grep -Fq "trap 'handle_t1_signal INT' INT" "$T1"
+grep -Fq "trap 'handle_t1_signal TERM' TERM" "$T1"
+grep -Fq 'T1_STATUS=%s' "$T1"
+grep -Fq 'cleanup=PASS' "$T1"
+if grep -Fq 'trap cleanup_t1 EXIT INT TERM' "$T1"; then
+ echo 'FAIL: T1 still treats a signal as an ordinary EXIT cleanup' >&2
+ exit 1
+fi
grep -Fq 'settle-gfs-reader-rollout.sh' "$T1"
grep -Fq 'gfs-rollout-shim' "$T1"
grep -Fq 'T2_UNREADY_DEPLOYMENTS' "$COMMON"
grep -Fq 'required gfs-controller-db Secret is missing or unreadable' "$T1"
+grep -Fq 'T1_STATUS' "$T2"
+grep -Fq 'T1_PENDING_TESTS' "$T2"
+grep -Fq 'T1 result was incomplete despite a zero process exit' "$T2"
+grep -Fq 'VERIFY_AUTH_RETRIES' "$ROOT/scripts/minikube/verify-gfs.sh"
+grep -Fq 'authentication probe remained unavailable' "$ROOT/scripts/minikube/verify-gfs.sh"
+grep -Fq 'GFS mutation disabled for this non-T2 sync' "$ROOT/Makefile"
+grep -Fq 'filter-gfs-resources.py' "$ROOT/Makefile"
+grep -Fq 'MINIKUBE_GFS_MUTATION' "$ROOT/scripts/minikube/pre-gate-sync.sh"
+grep -Fq 'T2_LOCK_TOKEN="${T2_LOCK_TOKEN}"' "$ROOT/scripts/minikube/pre-gate-sync.sh"
+grep -Fq 'T2_LOCK_RELEASED' "$COMMON"
+grep -Fq 'leaving the writer fence armed' "$ROOT/scripts/minikube/pre-gate-sync.sh"
+if grep -Fq 'gke_*)' "$ROOT/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then
+ echo 'FAIL: remote GFS authorization still depends on a gke_* name prefix' >&2
+ exit 1
+fi
# pre-gate-sync GFS serving provisioning must opt into restoring a NOLOGIN
# role from the committed Secret DSN. Only the T2 transition owns this
diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.sh
index d7db81126..568eb76bd 100755
--- a/scripts/tests/test-minikube-t2-scenarios.sh
+++ b/scripts/tests/test-minikube-t2-scenarios.sh
@@ -268,6 +268,11 @@ expect_code PROFILE_UNHEALTHY unready-final-preflight unready-final-preflight \
expect_code PROFILE_UNHEALTHY invalid-deployment-inventory invalid-deployment-inventory \
env "${repo_env[@]}" INVALID_INVENTORY='{not-json' \
bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"
+if grep -Fq 'Traceback' "$tmp/invalid-deployment-inventory"; then
+ fail 'invalid deployment inventory leaked a Python traceback instead of the stable failure contract'
+fi
+grep -Fq 'next:' "$tmp/invalid-deployment-inventory" ||
+ fail 'invalid deployment inventory omitted the stable next-step guidance'
expect_code PROFILE_UNHEALTHY malformed-deployment-inventory malformed-deployment-inventory \
env "${repo_env[@]}" INVALID_INVENTORY='{"items":[{"spec":{"replicas":"not-an-integer"}}]}' \
bash -c 'source "$1"; T2_PLAN_MODE=false; T2_BOOTSTRAP_REQUIRED=false; t2_kc(){ printf "%s" "$INVALID_INVENTORY"; }; t2_deployment_check' bash "$COMMON"
diff --git a/scripts/tests/test-provision-gfs-runtime.sh b/scripts/tests/test-provision-gfs-runtime.sh
index 29d80fbba..d6fa5ae77 100755
--- a/scripts/tests/test-provision-gfs-runtime.sh
+++ b/scripts/tests/test-provision-gfs-runtime.sh
@@ -62,6 +62,11 @@ grep -Fq 'GFS_REMOTE_RECONCILE_AUTHORIZED=true ALLOWED_CONTEXTS="$CONTEXT"' \
echo 'FAIL: post-overlay credential reconciliation did not carry explicit context authorization' >&2
exit 1
}
+grep -Fq 'sync-auth-key.sh --context "$CONTEXT" --require-gfs' \
+ "$ROOT/deploy/scripts/provision-gfs-runtime.sh" || {
+ echo 'FAIL: GFS runtime provisioning does not require its auth source and target' >&2
+ exit 1
+}
[ "$(grep -Fc "rollout status deployment/host-context-controller --timeout=${HCC_ROLLOUT_TIMEOUT_S}s" "$CALL_LOG")" -eq 3 ] || {
echo "FAIL: HCC post-overlay rollout did not use its dedicated ${HCC_ROLLOUT_TIMEOUT_S}s timeout" >&2
exit 1