Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: close PR 416 rollout gate findings
Alfredo Lopeze26e24dc4ea9
diff --git a/.gitignore b/.gitignoreindex 1fcc0af27..02b789b6d 100644--- a/.gitignore+++ b/.gitignore@@ -95,6 +95,7 @@ docs/analysis/# Local developer notes (untracked).local-notes/+.ralph/plans/# Agent session transcripts can contain internal infrastructure details.session-ses_*.mddiff --git a/deploy/scripts/lib/gfs-credential-rollout.sh b/deploy/scripts/lib/gfs-credential-rollout.shindex 9b13065c7..1c8bf071e 100755--- a/deploy/scripts/lib/gfs-credential-rollout.sh+++ b/deploy/scripts/lib/gfs-credential-rollout.sh@@ -1,5 +1,6 @@#!/usr/bin/env bash# Targeted GFSC rollout transitions after a credential is committed.+# shellcheck disable=SC2034deployment_uses_secret() {local deployment="$1" secret="$2" refs error@@ -61,6 +62,11 @@ credential_rollout_required() {credential_adoption_timestamp() {local deployment="$1" selector="$2" expected_dsn="$3" rows live oldest="" pod created readyif ! deployment_exists "$deployment"; then+ GFS_ROLLOUT_PROOF_KIND='deployment-absent'+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION='future-pod'+ GFS_ROLLOUT_PROOF_POD_COUNT=0+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"date -u '+%Y-%m-%dT%H:%M:%SZ'return 0fi@@ -88,6 +94,104 @@ rollout_exact() {kc -n "$GFS_NS" rollout status "deployment/${deployment}" --timeout="$ROLLOUT_TIMEOUT"}+credential_rollout_proof() {+ local secret="$1" deployment="$2" rotated_at="$3"+ local secret_json secret_rv active_dsn template_revision generation observed_revision+ local selector rows pod_name pod_created pod_ready pod_deleting pod_owner pod_revision+ local owner_revision+ local pod_count=0 current_rv+ if ! deployment_uses_secret "$deployment" "$secret"; then+ die "$GFS_NS/$deployment does not reference $GFS_NS/$secret; refusing to certify credential consumption"+ fi+ if ! secret_json="$(kc -n "$GFS_NS" get secret "$secret" -o json)"; then+ die "cannot snapshot $GFS_NS/$secret for rollout proof"+ fi+ if ! IFS=$'\t' read -r secret_rv active_dsn < <(printf '%s' "$secret_json" | python3 -c 'import base64, json, sys+obj = json.load(sys.stdin)+metadata = obj.get("metadata") or {}+encoded = (obj.get("data") or {}).get("connection-string") or ""+if not encoded:+ raise SystemExit(1)+try:+ dsn = base64.b64decode(encoded, validate=True).decode()+except (ValueError, UnicodeDecodeError):+ raise SystemExit(1)+print(str(metadata.get("resourceVersion", "")) + "\t" + dsn)'); then+ die "cannot decode the committed $GFS_NS/$secret credential for rollout proof"+ fi+ [ -n "$secret_rv" ] || die "the $GFS_NS/$secret rollout proof has no resourceVersion"+ # The standard Deployment revision is the portable template identity. A+ # custom annotation is not guaranteed to be copied to Pods by Kubernetes or+ # by HCC's reconciler, so it cannot be the sole rollout proof.+ template_revision="$(kc -n "$GFS_NS" get deployment "$deployment" \+ -o 'jsonpath={.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"+ generation="$(kc -n "$GFS_NS" get deployment "$deployment" \+ -o 'jsonpath={.metadata.generation}' 2>/dev/null || true)"+ observed_revision="$(kc -n "$GFS_NS" get deployment "$deployment" \+ -o 'jsonpath={.status.observedGeneration}' 2>/dev/null || true)"+ if [[ ! "$generation" =~ ^[1-9][0-9]*$ ]] ||+ [[ ! "$observed_revision" =~ ^[1-9][0-9]*$ ]] ||+ [ "$generation" != "$observed_revision" ]; then+ die "$GFS_NS/$deployment generation is not observed; refusing a pre-rollout credential proof"+ fi+ [ -n "$template_revision" ] ||+ die "$GFS_NS/$deployment has no observed Deployment revision; refusing an unverifiable credential proof"+ case "$deployment" in+ gfsc-reader) selector='app=gfs-controller,clerum.io/gfsc-role=reader' ;;+ gfsc-writer) selector='app=gfs-controller,clerum.io/gfsc-role=writer' ;;+ *) die "unsupported GFSC deployment for credential proof: $deployment" ;;+ esac+ if ! rows="$(kc -n "$GFS_NS" get pods -l "$selector" -o \+ 'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.metadata.annotations.clerum\.io/gfs-template-hash}{"\n"}{end}')"; then+ die "cannot list $GFS_NS/$deployment pods for credential proof"+ fi+ while IFS='|' read -r pod_name pod_created pod_ready pod_deleting pod_owner pod_revision; do+ [ -n "$pod_name" ] || continue+ [ -z "$pod_deleting" ] || continue+ if ! python3 - "$pod_created" "$rotated_at" <<'PY'+from datetime import datetime, timezone+import sys+created, rotated = sys.argv[1:]+def parse(value):+ return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)+try:+ ok = parse(created) >= parse(rotated)+except ValueError:+ ok = False+raise SystemExit(0 if ok else 1)+PY+ then+ die "$GFS_NS/$deployment Ready pod predates the committed credential rotation"+ fi+ [ "$pod_ready" = True ] || die "$GFS_NS/$deployment has a live pod that is not Ready"+ owner_revision="$(kc -n "$GFS_NS" get rs "$pod_owner" \+ -o 'jsonpath={.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"+ [ -n "$owner_revision" ] || owner_revision="$pod_revision"+ [ "$owner_revision" = "$template_revision" ] ||+ die "$GFS_NS/$deployment Ready pod is not on the observed template revision"+ if ! printf '%s' "$active_dsn" |+ kc -n "$GFS_NS" exec -i "$pod_name" -c gfsc -- node -e '+let input = "";+process.stdin.setEncoding("utf8");+process.stdin.on("data", chunk => input += chunk);+process.stdin.on("end", () => process.exit(input === process.env.GFS_PG_CONNECTION_STRING ? 0 : 1));' \+ >/dev/null 2>&1; then+ die "$GFS_NS/$deployment Ready pod did not consume the committed credential"+ fi+ pod_count=$((pod_count + 1))+ done <<<"$rows"+ [ "$pod_count" -gt 0 ] || die "no live $GFS_NS/$deployment pod exists for credential proof"+ current_rv="$(kc -n "$GFS_NS" get secret "$secret" -o \+ 'jsonpath={.metadata.resourceVersion}' 2>/dev/null || true)"+ [ "$current_rv" = "$secret_rv" ] ||+ die "$GFS_NS/$secret changed during rollout proof; refusing stale readiness"+ GFS_ROLLOUT_PROOF_KIND='pod-env'+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION="$secret_rv"+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION="$template_revision"+ GFS_ROLLOUT_PROOF_POD_COUNT="$pod_count"+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"+}+complete_rollout() {local secret="$1" deployment="$2" rotated state rcrotated="$(gfs_secret_rotated_at "$secret")"@@ -106,11 +210,17 @@ complete_rollout() {esacif ! deployment_exists "$deployment"; thenlog "${GFS_NS}/${deployment} is not present; credential is staged for its future pod"+ GFS_ROLLOUT_PROOF_KIND='deployment-absent'+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION='future-pod'+ GFS_ROLLOUT_PROOF_POD_COUNT=0+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"mark_secret_rollout_ready "$secret" "$rotated" rollout-running \|| die "staged credential state was superseded; refusing a stale completion"return 0fiif rollout_exact "$deployment"; then+ credential_rollout_proof "$secret" "$deployment" "$rotated"mark_secret_rollout_ready "$secret" "$rotated" rollout-running \|| die "${deployment} rolled out but credential state was superseded; refusing a stale completion"return 0diff --git a/deploy/scripts/lib/gfs-credential-secret.sh b/deploy/scripts/lib/gfs-credential-secret.shindex d6cb34291..be130bfff 100755--- a/deploy/scripts/lib/gfs-credential-secret.sh+++ b/deploy/scripts/lib/gfs-credential-secret.sh@@ -3,6 +3,11 @@# Candidate connection material is accepted only on stdin.# Snapshot globals are consumed by the provisioning script that sources this file.# shellcheck disable=SC2034+GFS_ROLLOUT_PROOF_KIND=""+GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""+GFS_ROLLOUT_PROOF_TEMPLATE_REVISION=""+GFS_ROLLOUT_PROOF_POD_COUNT=""+GFS_ROLLOUT_PROOF_AT=""gfs_secret_state() {kc -n "$GFS_NS" get secret "$1" -o 'jsonpath={.metadata.annotations.clerum\.io/gfs-dsn-state}'@@ -189,12 +194,37 @@ print(json.dumps([mark_secret_rollout_ready() {local secret="$1" expected_rotated_at="$2" expected_state="$3"+ local proof_kind proof_resource_version proof_template_revision proof_pod_count proof_at+ proof_kind="$GFS_ROLLOUT_PROOF_KIND"+ proof_resource_version="$GFS_ROLLOUT_PROOF_RESOURCE_VERSION"+ proof_template_revision="$GFS_ROLLOUT_PROOF_TEMPLATE_REVISION"+ proof_pod_count="$GFS_ROLLOUT_PROOF_POD_COUNT"+ proof_at="$GFS_ROLLOUT_PROOF_AT"python3 -c 'import json, sys-state, stamp = sys.argv[1:]-print(json.dumps([+state, stamp, proof_kind, proof_resource_version, proof_template_revision, proof_pod_count, proof_at = sys.argv[1:]+ops = [{"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": state},{"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-rotated-at", "value": stamp},{"op": "replace", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": "ready"},-]))' "$expected_state" "$expected_rotated_at" \+]+if proof_kind:+ if proof_resource_version:+ ops.insert(2, {"op": "test", "path": "/metadata/resourceVersion", "value": proof_resource_version})+ values = {+ "clerum.io/gfs-dsn-proof-kind": proof_kind,+ "clerum.io/gfs-dsn-proof-resource-version": proof_resource_version,+ "clerum.io/gfs-dsn-proof-template-revision": proof_template_revision,+ "clerum.io/gfs-dsn-proof-pod-count": proof_pod_count,+ "clerum.io/gfs-dsn-proof-at": proof_at,+ }+ for key, value in values.items():+ ops.append({"op": "add", "path": "/metadata/annotations/" + key.replace("/", "~1"), "value": value})+print(json.dumps(ops))' "$expected_state" "$expected_rotated_at" \+ "$proof_kind" "$proof_resource_version" "$proof_template_revision" "$proof_pod_count" "$proof_at" \| kc -n "$GFS_NS" patch secret "$secret" --type=json --patch-file=/dev/stdin >/dev/null+ GFS_ROLLOUT_PROOF_KIND=""+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION=""+ GFS_ROLLOUT_PROOF_POD_COUNT=""+ GFS_ROLLOUT_PROOF_AT=""}diff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.shindex 6d821c606..a9a718a21 100755--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh@@ -5,6 +5,49 @@ set -euo pipefailCONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"GFS_NS="${GFS_NS:-gfs}"+set +u+T2_PROJECT_DIR="$T2_PROJECT_DIR"+T2_PROFILE="$T2_PROFILE"+T2_CONTEXT="$T2_CONTEXT"+MINIKUBE_PROFILE="$MINIKUBE_PROFILE"+CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"+T2_LOCK_TOKEN="$T2_LOCK_TOKEN"+set -u+if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fi+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$CONTEXT"; fi+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"; fi+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTEXT"; fi+if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == clerum-* ]]; then+ # shellcheck source=scripts/minikube/t2-common.sh+ source "$ROOT/scripts/minikube/t2-common.sh"+ if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi+ cleanup_reconcile() {+ local status=$?+ t2_lock_release "$status"+ }+ trap cleanup_reconcile EXIT INT TERM+ t2_repo_metadata+ t2_profile_scope+ t2_context_check+ t2_mutation_lock+ # The verified profile context is authoritative. Do not let a caller keep+ # a second CONTEXT value after the fence has been acquired.+ CONTEXT="$T2_CONTEXT"+else+ case "$CONTEXT" in+ gke_*)+ # GKE deploys are owned by the explicitly selected deploy workflow rather+ # than the local Minikube lease. Keep this exception narrow: an arbitrary+ # CONTEXT must never turn this credential mutator into a remote-cluster+ # command without either the local fence or the deploy-context shape.+ ;;+ *)+ printf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2+ exit 1+ ;;+ esac+fiCONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"writer_dsn_b64="$(kubectl --context="$CONTEXT" -n "$GFS_NS" get secret gfs-controller-db \diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.shindex 66faa8f91..4451df0d1 100755--- a/scripts/e2e/minikube-real-postgres.sh+++ b/scripts/e2e/minikube-real-postgres.sh@@ -37,6 +37,7 @@ T1_PENDING_TESTS=0# reconcile cluster credentials for a run that never reached the suites# (e.g. a failed precondition, or this file being sourced by a harness test).T1_GFS_RESTORE_REQUIRED=false+T1_CLEANUP_DONE=falsedie_t1() {local code="$1"@@ -73,11 +74,19 @@ restore_gfs_runtime_credentials() {# rollout restart it, and let any wait reconcile still needs judge# readiness via the gfs-rollout-shim instead of the template generation# HCC's gfsReconciler keeps rewriting.- if ! CONTEXT="$T2_CONTEXT" \+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \+ CONTEXT="$T2_CONTEXT" \bash "$PROJECT_DIR/scripts/minikube/settle-gfs-reader-rollout.sh"; thenprintf '[minikube-t1] WARN: could not settle a leftover gfsc-reader rollout claim before restore\n' >&2fiif ! PATH="$PROJECT_DIR/scripts/minikube/gfs-rollout-shim:$PATH" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \CONTEXT="$T2_CONTEXT" \bash "$PROJECT_DIR/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then@@ -88,6 +97,11 @@ restore_gfs_runtime_credentials() {cleanup_t1() {local status=$?+ if [ "$T1_CLEANUP_DONE" = true ]; then+ return 0+ fi+ T1_CLEANUP_DONE=true+ trap - EXIT INT TERMif [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; thenlocal command_linecommand_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"@@ -101,6 +115,7 @@ cleanup_t1() {status=1fiif [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi+ t2_lock_release "$status" || status=$?exit "$status"}trap cleanup_t1 EXIT INT TERM@@ -183,7 +198,7 @@ list_real_pg_files() {is_isolated_control_api_file() {local path="$1"case "$path" in- */db.realPostgresMigration.integration.test.ts|*/gfsReaderRole.realPostgres.integration.test.ts)+ "$PROJECT_DIR"/control-api/test/*realPostgres*.test.ts)return 0;;*)@@ -193,20 +208,19 @@ is_isolated_control_api_file() {}require_isolated_control_api_files() {- local relative path- for relative in \- test/db.realPostgresMigration.integration.test.ts \- test/gfsReaderRole.realPostgres.integration.test.ts; do- path="$PROJECT_DIR/control-api/$relative"- if [ ! -f "$path" ]; then- T1_NEXT_COMMAND='restore the isolated Real PostgreSQL suite files, then re-run T1'- die_t1 ZERO_TESTS_EXECUTED "isolated suite file missing or renamed: $relative"- fi+ local path count=0+ while IFS= read -r path; do+ [ -n "$path" ] || continue+ count=$((count + 1))is_isolated_control_api_file "$path" || {- T1_NEXT_COMMAND='keep the isolated Real PostgreSQL suite filenames in the T1 harness'- die_t1 ZERO_TESTS_EXECUTED "isolated suite file is no longer routed away from control-postgres: $relative"+ T1_NEXT_COMMAND='route every control-api Real PostgreSQL suite through the disposable PostgreSQL instance'+ die_t1 ZERO_TESTS_EXECUTED "control-api Real PostgreSQL suite is not isolated: $path"}- done+ done < <(list_real_pg_files control-api)+ if [ "$count" -le 0 ]; then+ T1_NEXT_COMMAND='restore the control-api Real PostgreSQL suite files, then re-run T1'+ die_t1 ZERO_TESTS_EXECUTED 'no control-api Real PostgreSQL suites were found'+ fi}start_isolated_postgres() {@@ -254,8 +268,12 @@ run_suite() {while IFS= read -r file; do[ -z "$file" ] && continue- if [ "$package" = control-api ] && is_isolated_control_api_file "$file"; then- [ "$lane" = isolated ] || continue+ if [ "$package" = control-api ]; then+ if [ "$lane" = isolated ]; then+ is_isolated_control_api_file "$file" || continue+ else+ is_isolated_control_api_file "$file" && continue+ fielse[ "$lane" = shared ] || continuefi@@ -277,9 +295,10 @@ run_suite() {json_file="$T1_TMP_DIR/$(printf '%s' "$package-$lane" | tr / _).json"printf '[minikube-t1] running %s %s Real PostgreSQL suites (%s files)\n' \"$package" "$lane" "$expected_files"- # T1 verdict is the JSON reporter (expected files, executed/passed, zero- # failures, zero pending). Capture npm's exit but do not treat a leftover- # process exit after a complete green reporter as a failed suite.+ # T1 requires both a complete green JSON reporter and a successful process+ # exit. A reporter can be written before Vitest/npm fails during teardown,+ # worker shutdown, OOM handling, or signal cleanup; accepting that partial+ # observation would certify a broken lane.suite_status=0(cd "$PROJECT_DIR/$package"@@ -332,8 +351,10 @@ PYdie_t1 ZERO_TESTS_EXECUTED "Real PostgreSQL lane reported zero tests or skips in $package ($lane)"fiif [ "$suite_status" -ne 0 ]; then- printf '[minikube-t1] vitest process exited %s after a complete green reporter for %s %s; T1 follows the reporter\n' \- "$suite_status" "$package" "$lane"+ cat "$log_file" >&2 || true+ cat "$json_file" >&2 || true+ T1_NEXT_COMMAND='repair the failed or incomplete Real PostgreSQL lane, then re-run T1'+ die_t1 REAL_PG_SUITE_FAILED "Vitest process exited $suite_status for $package ($lane)"fiprintf '[minikube-t1] PASS %s %s files=%s tests=%s skipped=0\n' \"$package" "$lane" "$passed_files" "$passed_tests"@@ -345,6 +366,7 @@ main() {t2_repo_metadatat2_profile_scopet2_context_check+ t2_mutation_lockif [ -z "$T2_CONTEXT" ]; thenT1_NEXT_COMMAND='set CONTROL_API_REAL_PG_CONTEXT to the verified Kubernetes context'die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'local Real PostgreSQL requires an explicit Minikube context'@@ -409,7 +431,6 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.decstart_isolated_postgresT1_GFS_RESTORE_REQUIRED=truerun_suite control-api isolated "$ISOLATED_DSN"- run_suite control-api shared "$ADMIN_DSN"run_suite gfs-controller shared "$ADMIN_DSN"unset PG_USER PG_PASSWORD PG_DATABASE T1_REDACT_PASSWORDunset ADMIN_DSN ISOLATED_DSNdiff --git a/scripts/minikube/gfs-rollout-shim/kubectl b/scripts/minikube/gfs-rollout-shim/kubectlindex 133e6dd75..1b6749264 100755--- a/scripts/minikube/gfs-rollout-shim/kubectl+++ b/scripts/minikube/gfs-rollout-shim/kubectl@@ -64,20 +64,32 @@ while [ "$i" -lt "${#args[@]}" ]; doi=$((i + 1))done-timeout_seconds=600-case "$timeout_raw" in- '') ;;- *m) timeout_seconds=$((${timeout_raw%m} * 60)) ;;- *s) timeout_seconds="${timeout_raw%s}" ;;- *[!0-9]*) ;;- *) timeout_seconds="$timeout_raw" ;;-esac-# A CrashLoopBackOff retry delay reaches five minutes; a shorter caller-# timeout cannot observe the credential-restore recovery this wait exists-# for, so enforce a floor. Overridable only for the stubbed contract test.-min_timeout="${GFS_ROLLOUT_SHIM_MIN_TIMEOUT_SECONDS:-600}"-if [ "$timeout_seconds" -lt "$min_timeout" ]; then- timeout_seconds="$min_timeout"+if ! timeout_seconds="$(python3 - "$timeout_raw" "${GFS_ROLLOUT_SHIM_MIN_TIMEOUT_SECONDS:-600}" <<'PY'+import re+import sys++timeout_raw, minimum_raw = sys.argv[1:]++def parse_duration(raw, default, label):+ if raw == "":+ return default+ match = re.fullmatch(r"[1-9][0-9]{0,4}([sm]?)", raw)+ if not match:+ raise SystemExit(f"invalid {label}: expected a positive integer with optional s/m suffix")+ value = int(raw[:-1] if raw[-1:] in {"s", "m"} else raw)+ if raw.endswith("m"):+ value *= 60+ if value < 1 or value > 86400:+ raise SystemExit(f"invalid {label}: value must be between 1 and 86400 seconds")+ return value++timeout = parse_duration(timeout_raw, 600, "rollout timeout")+minimum = parse_duration(minimum_raw, 600, "minimum rollout timeout")+print(max(timeout, minimum))+PY+)"; then+ printf '[gfs-rollout-shim] refusing an invalid rollout timeout\n' >&2+ exit 2fiif [ -z "$context" ]; thendiff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.shindex abcec6931..625acefba 100755--- a/scripts/minikube/pf-all-stack.sh+++ b/scripts/minikube/pf-all-stack.sh@@ -7,8 +7,20 @@ PROFILE="${MINIKUBE_PROFILE:-clerum-test}"KC=(kubectl --context="${PROFILE}")HOLD=falsePIDS=()+PIDFILES=()+PIDFILE_SERVICES=()SAFE_PROFILE="${PROFILE//[^A-Za-z0-9_.-]/_}"HAS_PROFILE_OWNED_PORTS=false+if [[ ! "$PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then+ echo "ERROR: invalid Minikube profile identifier: $PROFILE" >&2+ exit 1+fi+set +u+PROFILE_PID_ROOT="$CLERUM_PROFILE_CACHE_ROOT"+set -u+if [[ -z "$PROFILE_PID_ROOT" ]]; then+ PROFILE_PID_ROOT="$HOME/.cache/clerum/minikube-profiles"+fiload_branch_profile_ports_env() {local file="$1"@@ -110,11 +122,37 @@ cleanup() {if [[ "${HOLD}" != "true" ]]; thenreturn 0fi- for pid in "${PIDS[@]}"; do- kill "${pid}" 2>/dev/null || true+ local index+ for index in "${!PIDFILES[@]}"; do+ kill_owned_pidfile "${PIDFILES[$index]}" "${PIDFILE_SERVICES[$index]}" || truedone}+kill_owned_pidfile() {+ local pidfile="$1"+ local service="$2"+ local pid command_line+ [[ -f "$pidfile" ]] || return 0+ pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"+ if [[ ! "$pid" =~ ^[1-9][0-9]*$ ]]; then+ rm -f -- "$pidfile"+ return 0+ fi+ if ! kill -0 "$pid" 2>/dev/null; then+ rm -f -- "$pidfile"+ return 0+ fi+ command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)"+ if [[ "$command_line" != *port-forward* ||+ "$command_line" != *"svc/$service"* ||+ "$command_line" != *"$PROFILE"* ]]; then+ echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the $PROFILE $service port-forward" >&2+ return 1+ fi+ kill "$pid" 2>/dev/null || true+ rm -f -- "$pidfile"+}+start_pf() {local name="$1"local namespace="$2"@@ -123,23 +161,19 @@ start_pf() {local health_url="${5:-}"local log="/tmp/pf-${SAFE_PROFILE}-${name}.log"local pidfile="/tmp/pf-${SAFE_PROFILE}-${name}.pid"- local profile_pids_dir="${CLERUM_PROFILE_CACHE_ROOT:-${HOME}/.cache/clerum/minikube-profiles}/${PROFILE}/pids"+ local profile_pids_dir="$PROFILE_PID_ROOT/$SAFE_PROFILE/pids"local profile_pidfile="${profile_pids_dir}/${name}.pid"- mkdir -p "${profile_pids_dir}"- if [[ -f "${pidfile}" ]]; then- kill "$(cat "${pidfile}")" 2>/dev/null || true- rm -f "${pidfile}"- fi- if [[ -f "${profile_pidfile}" ]]; then- kill "$(cat "${profile_pidfile}")" 2>/dev/null || true- rm -f "${profile_pidfile}"- fi+ mkdir -p "$profile_pids_dir"+ kill_owned_pidfile "$pidfile" "$service"+ kill_owned_pidfile "$profile_pidfile" "$service"nohup "${KC[@]}" -n "${namespace}" port-forward --address=127.0.0.1 "svc/${service}" "${ports}" >"${log}" 2>&1 </dev/null &echo $! >"${pidfile}"echo $! >"${profile_pidfile}"PIDS+=("$!")+ PIDFILES+=("$pidfile" "$profile_pidfile")+ PIDFILE_SERVICES+=("$service" "$service")echo " ${name}: pid=$(cat "${pidfile}") ns=${namespace} svc=${service} ports=${ports}"sleep 0.2if ! kill -0 "$!" 2>/dev/null; thendiff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.shindex 213956ebf..1b4a45b07 100755--- a/scripts/minikube/pre-gate-sync.sh+++ b/scripts/minikube/pre-gate-sync.sh@@ -2,6 +2,10 @@# Enforces the "sync to minikube before every gate" rule from the platform E2E plan.set -euo pipefail+set +u+PRE_GATE_SYNC_CONFIG_ONLY="$PRE_GATE_SYNC_CONFIG_ONLY"+set -u+if [ -z "$PRE_GATE_SYNC_CONFIG_ONLY" ]; then PRE_GATE_SYNC_CONFIG_ONLY=false; fiSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"@@ -43,6 +47,23 @@ while [[ $# -gt 0 ]]; doesacdone+# The pre-gate entrypoint is also a mutating recovery boundary. It must carry+# the same branch/profile identity as the T2 orchestrator, or acquire that+# identity itself when invoked standalone.+T2_PROJECT_DIR="$PROJECT_DIR"+T2_PROFILE="$PROFILE"+T2_CONTEXT="$PROFILE"+T2_GATE_ID="$GATE_NAME"+# shellcheck source=scripts/minikube/t2-common.sh+source "$SCRIPT_DIR/t2-common.sh"+if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi+if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then+ t2_require_commands+ t2_repo_metadata+ t2_profile_scope+ t2_context_check+fi+log() {printf '[pre-gate-sync] %s\n' "$*"}@@ -154,13 +175,19 @@ restore_control_api() {}restore_pre_gate_writers() {+ local status=$?# Restore Control API first so it can serve the final readiness checks; WRC# remains last because it is the consumer that must stay stopped until every# migration, writer restart, and legacy-policy gate has completed.- restore_control_api- restore_workflow_reconciler+ restore_control_api || status=1+ restore_workflow_reconciler || status=1+ t2_lock_release "$status" || status=$?+ return "$status"}+if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then+ t2_mutation_lock+fitrap restore_pre_gate_writers EXITfingerprint_dir() {@@ -375,11 +402,19 @@ provision_gfs_serving() {# still needs a reader rollout, the gfs-rollout-shim PATH prefix makes# that wait judge readiness instead of the template generation HCC keeps# rewriting.- CONTEXT="${PROFILE}" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \+ CONTEXT="$T2_CONTEXT" \bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"if ! PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \- CONTEXT="${PROFILE}" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \+ CONTEXT="$T2_CONTEXT" \bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; thenlog "ERROR: gfs DB provisioning FAILED — gfsc cannot authorize any operation. Aborting ${GATE_NAME} pre-gate sync."exit 1diff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.shindex bcdd71141..bb6680103 100755--- a/scripts/minikube/settle-gfs-reader-rollout.sh+++ b/scripts/minikube/settle-gfs-reader-rollout.sh@@ -13,14 +13,50 @@# times out; not restarting a Ready reader is the only safe path.set -euo pipefail-CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"+ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)"+set +u+T2_PROJECT_DIR="$T2_PROJECT_DIR"+T2_PROFILE="$T2_PROFILE"+T2_CONTEXT="$T2_CONTEXT"+MINIKUBE_PROFILE="$MINIKUBE_PROFILE"+CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"+CONTEXT="$CONTEXT"+set -u+if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fi+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$CONTEXT"; fi+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"; fi+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTEXT"; fi+# shellcheck source=scripts/minikube/t2-common.sh+source "$ROOT/scripts/minikube/t2-common.sh"+if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi+T2_CONTEXT="$T2_CONTEXT"+CONTEXT="$T2_CONTEXT"GFS_NS="${GFS_NS:-gfs}"DEPLOY="${GFS_READER_DEPLOYMENT:-gfsc-reader}"SECRET="${GFS_READER_DB_SECRET:-gfs-controller-reader-db}"SELECTOR="${GFS_READER_SELECTOR:-app=gfs-controller,clerum.io/gfsc-role=reader}"-kc() { kubectl --context="$CONTEXT" "$@"; }log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }+die() { log "ERROR: $*"; exit 1; }+kc() { kubectl --context="$T2_CONTEXT" "$@"; }++cleanup_settle() {+ local status=$?+ t2_lock_release "$status"+}+trap cleanup_settle EXIT INT TERM+t2_repo_metadata+t2_profile_scope+t2_context_check+t2_mutation_lock++# The proof library is sourced only after the profile fence is established. It+# reads the committed DSN in memory and never prints it.+# shellcheck source=deploy/scripts/lib/gfs-credential-secret.sh+source "$ROOT/deploy/scripts/lib/gfs-credential-secret.sh"+# shellcheck source=deploy/scripts/lib/gfs-credential-rollout.sh+source "$ROOT/deploy/scripts/lib/gfs-credential-rollout.sh"if ! kc -n "$GFS_NS" get deployment "$DEPLOY" >/dev/null 2>&1; thenexit 0@@ -77,11 +113,5 @@ if [ -z "$rotated" ]; thenfilog "gfsc-reader is Ready with an interrupted rollout claim; marking the reader Secret ready without restart"-python3 -c 'import json, sys-state, stamp = sys.argv[1:]-print(json.dumps([- {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": state},- {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-rotated-at", "value": stamp},- {"op": "replace", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": "ready"},-]))' "$state" "$rotated" \- | kc -n "$GFS_NS" patch secret "$SECRET" --type=json --patch-file=/dev/stdin >/dev/null+credential_rollout_proof "$SECRET" "$DEPLOY" "$rotated"+mark_secret_rollout_ready "$SECRET" "$rotated" rollout-runningdiff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex 5ab1a12be..bcce90fa8 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -63,6 +63,16 @@ T2_LOCK_KEY=""T2_EVIDENCE_DIR=""T2_EVIDENCE_FILE=""T2_LOCK_HELD=false+T2_LOCK_TOKEN="$T2_LOCK_TOKEN"+T2_RUN_ID="$T2_RUN_ID"+T2_GATE_ID="$T2_GATE_ID"+if [ -z "$T2_GATE_ID" ]; then T2_GATE_ID=minikube-t2; fi+T2_CERTIFICATION_VERSION=1+T2_T0_CERTIFIED=false+T2_T1_CERTIFIED=false+T2_PRIOR_ATTESTATION=""+T2_PRIOR_T0_ATTESTATION=""+T2_PRIOR_T1_ATTESTATION=""T2_REQUIRED_NAMESPACES="$T2_REQUIRED_NAMESPACES"if [ -z "$T2_REQUIRED_NAMESPACES" ]; then T2_REQUIRED_NAMESPACES="control-plane gfs mcp-host mcp-server profiles rpc-proxy channels sandbox-recipes sandbox-ui webhook-ingress registry"; fi@@ -220,7 +230,9 @@ t2_profile_scope() {}t2_context_check() {- if ! kubectl --context="$T2_CONTEXT" config current-context >/dev/null 2>&1; then+ local context_name+ context_name="$(kubectl config get-contexts -o name 2>/dev/null | awk -v target="$T2_CONTEXT" '$0 == target {print; exit}')"+ if [ "$context_name" != "$T2_CONTEXT" ]; thenT2_NEXT_COMMAND='select the explicit Kubernetes context generated with the profile't2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context is unavailable: $T2_CONTEXT"fi@@ -593,60 +605,159 @@ t2_classify_transition() {fi}+t2_lock_owner_value() {+ local key="$1"+ [ -f "$T2_LOCK_DIR/owner.env" ] || return 1+ sed -n "s/^$key=//p" "$T2_LOCK_DIR/owner.env" | head -1+}++t2_lock_key() {+ printf '%s\0%s\0%s\0%s\0%s' \+ "$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_HEAD" "$T2_PROFILE" "$T2_CONTEXT" |+ shasum | awk '{print $1}'+}++t2_lock_profile_id_check() {+ if [[ ! "$T2_PROFILE" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then+ T2_NEXT_COMMAND='use the generated profile name without path separators or shell metacharacters'+ t2_fail DEVELOPMENT_SCOPE_REQUIRED 'profile lock identifier is not a safe local profile name'+ fi+}++t2_lock_process_matches() {+ local pid="$1" expected_start actual_start state+ [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1+ kill -0 "$pid" >/dev/null 2>&1 || return 1+ state="$(ps -p "$pid" -o state= 2>/dev/null | tr -d '[:space:]' || true)"+ [ -z "$state" ] || [[ "$state" != Z* ]] || return 1+ expected_start="$(t2_lock_owner_value PROCESS_START || true)"+ [ -n "$expected_start" ] || return 1+ # Some macOS/sandboxed ps implementations hide lstart/state. The opaque+ # token and live PID still fence the child in that environment; use the+ # stronger start-time comparison whenever the platform exposes it.+ [ "$expected_start" = unavailable ] && return 0+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [ -z "$actual_start" ] || [ "$actual_start" = "$expected_start" ]+}+t2_lock_acquire() {+ local process_start+ t2_lock_profile_id_checkmkdir -p "$T2_LOCK_ROOT"T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"- T2_LOCK_KEY="$(printf '%s\0%s\0%s\0%s' "$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_HEAD" "$T2_PROFILE" | shasum | awk '{print $1}')"- if ! mkdir "$T2_LOCK_DIR" 2>/dev/null; then- local existing_pid- existing_pid="$(sed -n 's/^PID=//p' "$T2_LOCK_DIR/owner.env" 2>/dev/null | head -1 || true)"- if [[ ! "$existing_pid" =~ ^[0-9]+$ ]]; then+ T2_LOCK_KEY="$(t2_lock_key)"+ if [ -e "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR" ]; then+ if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ]; then+ T2_NEXT_COMMAND='remove the unsafe profile-lock path manually after verifying the profile owner'+ t2_fail PROFILE_BUSY "profile lock path is not a directory: $T2_LOCK_DIR"+ fi+ local existing_pid existing_token existing_start+ existing_pid="$(t2_lock_owner_value PID || true)"+ existing_token="$(t2_lock_owner_value TOKEN || true)"+ existing_start="$(t2_lock_owner_value PROCESS_START || true)"+ if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; thenT2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'- t2_fail PROFILE_BUSY "profile $T2_PROFILE has a lock without a valid owner PID"+ t2_fail PROFILE_BUSY "profile $T2_PROFILE has an orphaned lock without a valid owner identity"fi- if [ -n "$existing_pid" ] && kill -0 "$existing_pid" >/dev/null 2>&1; then+ if t2_lock_process_matches "$existing_pid"; thenT2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"t2_fail PROFILE_BUSY "profile $T2_PROFILE is locked by PID $existing_pid"fi- rm -rf "$T2_LOCK_DIR"- mkdir "$T2_LOCK_DIR" 2>/dev/null || {- T2_NEXT_COMMAND='retry after the profile lock is released'- t2_fail PROFILE_BUSY "unable to reclaim the profile lock: $T2_PROFILE"- }+ rm -rf -- "$T2_LOCK_DIR"fi+ mkdir "$T2_LOCK_DIR" 2>/dev/null || {+ T2_NEXT_COMMAND='retry after the profile lock is released'+ t2_fail PROFILE_BUSY "unable to acquire the profile lock: $T2_PROFILE"+ }umask 077+ T2_LOCK_TOKEN="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"+ process_start="$(ps -p $$ -o lstart= 2>/dev/null | sed 's/^ *//' || true)"+ [ -n "$process_start" ] || process_start=unavailablecat >"$T2_LOCK_DIR/owner.env" <<EOFREPOSITORY=$T2_PROJECT_DIRBRANCH=$T2_BRANCHHEAD=$T2_HEADPROFILE=$T2_PROFILECONTEXT=$T2_CONTEXT+WORKTREE_ID=$T2_WORKTREE_IDLOCK_KEY=$T2_LOCK_KEY+TOKEN=$T2_LOCK_TOKENPID=$$+PROCESS_START=$process_startSTARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)EOFT2_LOCK_HELD=true- trap t2_lock_release EXIT INT TERM+}++t2_lock_validate_inherited() {+ local expected_pid expected_token owner_key owner_value+ t2_lock_profile_id_check+ T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"+ T2_LOCK_KEY="$(t2_lock_key)"+ if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR/owner.env" ]; then+ T2_NEXT_COMMAND='run the parent T2 command with its profile lock token, then retry this child operation'+ t2_fail PROFILE_LOCK_REQUIRED "inherited profile lock is missing for $T2_PROFILE"+ fi+ expected_token="$T2_LOCK_TOKEN"+ owner_value="$(t2_lock_owner_value TOKEN || true)"+ if [ -z "$expected_token" ] || [ "$owner_value" != "$expected_token" ]; then+ T2_NEXT_COMMAND='pass the opaque T2_LOCK_TOKEN from the owning T2 process; do not bypass the profile lock'+ t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock token is missing or does not match'+ fi+ for owner_key in REPOSITORY BRANCH HEAD PROFILE CONTEXT WORKTREE_ID LOCK_KEY; do+ case "$owner_key" in+ REPOSITORY) expected_pid="$T2_PROJECT_DIR" ;;+ BRANCH) expected_pid="$T2_BRANCH" ;;+ HEAD) expected_pid="$T2_HEAD" ;;+ PROFILE) expected_pid="$T2_PROFILE" ;;+ CONTEXT) expected_pid="$T2_CONTEXT" ;;+ WORKTREE_ID) expected_pid="$T2_WORKTREE_ID" ;;+ LOCK_KEY) expected_pid="$T2_LOCK_KEY" ;;+ esac+ if [ "$(t2_lock_owner_value "$owner_key" || true)" != "$expected_pid" ]; then+ T2_NEXT_COMMAND='re-run the operation from the worktree/profile that owns the T2 lock'+ t2_fail PROFILE_OWNERSHIP_MISMATCH "profile lock owner does not match $owner_key"+ fi+ done+ expected_pid="$(t2_lock_owner_value PID || true)"+ if ! t2_lock_process_matches "$expected_pid"; then+ T2_NEXT_COMMAND='restart the parent T2 operation so it can acquire a live profile lock'+ t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'+ fi+ T2_LOCK_HELD=false+}++t2_mutation_lock() {+ if [ "$T2_SKIP_LOCK" = true ]; then+ t2_lock_validate_inherited+ else+ t2_lock_acquire+ fi}t2_lock_release() {- local status=$?+ local incoming_status=$? status owner_pid owner_token+ if [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fitrap - EXIT INT TERMif [ "$T2_LOCK_HELD" = true ] && [ -n "$T2_LOCK_DIR" ]; then- local owner_pid- owner_pid="$(sed -n 's/^PID=//p' "$T2_LOCK_DIR/owner.env" 2>/dev/null | head -1 || true)"- if [ "$owner_pid" = "$$" ]; then rm -rf "$T2_LOCK_DIR"; fi+ owner_pid="$(t2_lock_owner_value PID || true)"+ owner_token="$(t2_lock_owner_value TOKEN || true)"+ if [ "$owner_pid" = "$$" ] && [ -n "$T2_LOCK_TOKEN" ] && [ "$owner_token" = "$T2_LOCK_TOKEN" ]; then+ rm -rf -- "$T2_LOCK_DIR"+ fifi+ T2_LOCK_HELD=falseif [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; thent2_evidence_write failure FAIL "$T2_ERROR_CODE"fi- exit "$status"+ return "$status"}t2_evidence_init() {local stampstamp="$(date -u +%Y%m%dT%H%M%SZ)"- T2_EVIDENCE_DIR="$T2_EVIDENCE_ROOT/$stamp-$(printf '%s' "$T2_HEAD" | cut -c1-12)"+ T2_RUN_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"+ T2_EVIDENCE_DIR="$T2_EVIDENCE_ROOT/$stamp-$(printf '%s' "$T2_HEAD" | cut -c1-12)-$T2_RUN_ID"T2_EVIDENCE_FILE="$T2_EVIDENCE_DIR/evidence.json"umask 077mkdir -p "$T2_EVIDENCE_DIR/logs"@@ -660,12 +771,14 @@ t2_evidence_write() {PHASE="$phase" STATUS="$status" DETAIL="$detail" \T2_PROJECT_DIR="$T2_PROJECT_DIR" T2_BRANCH="$T2_BRANCH" T2_HEAD="$T2_HEAD" \T2_ORIGIN_DEV="$T2_ORIGIN_DEV" T2_MERGE_BASE="$T2_MERGE_BASE" \+ T2_WORKTREE_ID="$T2_WORKTREE_ID" T2_RUN_ID="$T2_RUN_ID" T2_GATE_ID="$T2_GATE_ID" \T2_PROFILE="$T2_PROFILE" T2_CONTEXT="$T2_CONTEXT" T2_CLUSTER_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \T2_EVIDENCE_DIR="$T2_EVIDENCE_DIR" \T2_IMAGE_MANIFEST="$T2_IMAGE_MANIFEST" T2_IMAGE_SOURCE="$T2_IMAGE_SOURCE" T2_IMAGE_TAG="$T2_IMAGE_TAG" \python3 - "$file" "$T2_EVIDENCE_FILE" <<'PY'import jsonimport os+from datetime import datetime, timedelta, timezonefrom pathlib import Pathpath = Path(os.sys.argv[1])prior_path = Path(os.sys.argv[2])@@ -674,6 +787,14 @@ if prior_path.exists():try: prior = json.loads(prior_path.read_text())except ValueError: prior = {}prior.setdefault("evidenceVersion", 1)+prior["certificationVersion"] = 1+prior.setdefault("runId", os.environ.get("T2_RUN_ID", ""))+now = datetime.now(timezone.utc)+prior.setdefault("attestationStartedAt", now.isoformat().replace("+00:00", "Z"))+prior.setdefault(+ "attestationExpiresAt",+ (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),+)def redact(value):import revalue = str(value or "")@@ -689,6 +810,9 @@ prior.update({"head": os.environ.get("T2_HEAD", ""),"originDev": os.environ.get("T2_ORIGIN_DEV", ""),"mergeBase": os.environ.get("T2_MERGE_BASE", ""),+ "worktreeId": os.environ.get("T2_WORKTREE_ID", ""),+ "runId": os.environ.get("T2_RUN_ID", ""),+ "gateId": os.environ.get("T2_GATE_ID", ""),"worktree": os.environ.get("T2_PROJECT_DIR", ""),"profile": os.environ.get("T2_PROFILE", ""),"context": os.environ.get("T2_CONTEXT", ""),@@ -701,6 +825,14 @@ prior.update({"status": os.environ.get("STATUS", ""),"detail": redact(os.environ.get("DETAIL", "")),})+phase = os.environ.get("PHASE", "")+status = os.environ.get("STATUS", "")+attestation = prior.get("attestationStatus", "IN_PROGRESS")+if status in {"FAIL", "INVALIDATED"}:+ attestation = "INVALIDATED"+elif phase == "complete" and status == "PASS":+ attestation = "PASS"+prior["attestationStatus"] = attestationprior.setdefault("phases", []).append({"name": os.environ.get("PHASE", ""),"status": os.environ.get("STATUS", ""),@@ -711,10 +843,83 @@ PYmv -- "$file" "$T2_EVIDENCE_FILE"}+t2_certification_validate_prior_lanes() {+ local result+ if [ -z "$T2_CLUSTER_FINGERPRINT" ]; then+ T2_NEXT_COMMAND='run the exact-head preflight and regenerate a profile marker with a cluster fingerprint'+ t2_fail CERTIFICATION_REQUIRED 'current exact-head marker has no cluster fingerprint'+ fi+ if ! result="$(+ CERTIFICATION_ROOT="$T2_EVIDENCE_ROOT" \+ EXPECTED_REPOSITORY="$T2_PROJECT_DIR" EXPECTED_BRANCH="$T2_BRANCH" \+ EXPECTED_HEAD="$T2_HEAD" EXPECTED_ORIGIN_DEV="$T2_ORIGIN_DEV" \+ EXPECTED_WORKTREE_ID="$T2_WORKTREE_ID" EXPECTED_PROFILE="$T2_PROFILE" \+ EXPECTED_CONTEXT="$T2_CONTEXT" EXPECTED_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \+ EXPECTED_GATE_ID="$T2_GATE_ID" python3 - <<'PY'+import json+import os+from datetime import datetime, timezone+from pathlib import Path++root = Path(os.environ["CERTIFICATION_ROOT"])+expected = {+ "repository": os.environ["EXPECTED_REPOSITORY"],+ "branch": os.environ["EXPECTED_BRANCH"],+ "head": os.environ["EXPECTED_HEAD"],+ "originDev": os.environ["EXPECTED_ORIGIN_DEV"],+ "worktreeId": os.environ["EXPECTED_WORKTREE_ID"],+ "profile": os.environ["EXPECTED_PROFILE"],+ "context": os.environ["EXPECTED_CONTEXT"],+ "clusterFingerprintRef": os.environ["EXPECTED_FINGERPRINT"],+ "gateId": os.environ["EXPECTED_GATE_ID"],+}+for candidate in sorted(root.glob("*/evidence.json"), reverse=True):+ try:+ data = json.loads(candidate.read_text())+ except (OSError, ValueError):+ continue+ if data.get("certificationVersion") != 1:+ continue+ if data.get("attestationStatus") != "PASS":+ continue+ if not data.get("runId"):+ continue+ try:+ expiry = datetime.fromisoformat(str(data.get("attestationExpiresAt", "")).replace("Z", "+00:00"))+ if expiry <= datetime.now(timezone.utc):+ continue+ except (TypeError, ValueError):+ continue+ if any(data.get(key, "") != value for key, value in expected.items()):+ continue+ latest = {}+ for phase in data.get("phases", []):+ name = phase.get("name")+ if name in {"T0", "T0_ATTESTED"}:+ latest["T0"] = phase.get("status")+ if name in {"T1", "T1_ATTESTED"}:+ latest["T1"] = phase.get("status")+ if latest.get("T0") == "PASS" and latest.get("T1") == "PASS":+ print("\t".join([str(candidate), "PASS", "PASS"]))+ raise SystemExit(0)+raise SystemExit("no valid exact-head T0/T1 attestation was found")+PY+ )"; then+ T2_NEXT_COMMAND='run make minikube-t2 with T0=true and T1=true to create exact-head lane attestations'+ t2_fail CERTIFICATION_REQUIRED "$result"+ fi+ IFS=$'\t' read -r T2_PRIOR_ATTESTATION T2_PRIOR_T0_ATTESTATION T2_PRIOR_T1_ATTESTATION <<< "$result"+ T2_T0_CERTIFIED=true+ T2_T1_CERTIFIED=true+ t2_evidence_write T0_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"+ t2_evidence_write T1_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"+}+t2_write_plan() {local plan_file="$1"[ -n "$plan_file" ] || return 0- PLAN_STATE="$T2_PLAN_STATE" PLAN_REASON="$T2_PLAN_REASON" python3 - "$plan_file" <<'PY'+ local tmp_file="$plan_file.tmp.$$"+ PLAN_STATE="$T2_PLAN_STATE" PLAN_REASON="$T2_PLAN_REASON" python3 - "$tmp_file" <<'PY'import jsonimport osfrom pathlib import Path@@ -723,4 +928,5 @@ Path(os.sys.argv[1]).write_text(json.dumps({"reason": os.environ.get("PLAN_REASON", ""),}, sort_keys=True) + "\n")PY+ mv -- "$tmp_file" "$plan_file"}diff --git a/scripts/minikube/t2-preflight.sh b/scripts/minikube/t2-preflight.shindex 18ee423b0..4a90e54bf 100755--- a/scripts/minikube/t2-preflight.sh+++ b/scripts/minikube/t2-preflight.sh@@ -19,14 +19,18 @@ T2_PLAN_FILE="$T2_PLAN_FILE"if [ -z "$T2_PLAN_FILE" ]; then T2_PLAN_FILE="$T2_EVIDENCE_ROOT/next-plan.json"; fiT2_NEXT_COMMAND='re-run the canonical command with the verified profile'+cleanup_preflight() {+ local status=$?+ t2_lock_release "$status"+}+trap cleanup_preflight EXIT+main() {t2_require_commandst2_repo_metadatat2_profile_scopet2_context_check- if [ "$T2_SKIP_LOCK" != true ]; then- t2_lock_acquire- fi+ t2_mutation_lockt2_evidence_init# Profile status is read before any resource check. A missing/stopped profilediff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.shindex 7ff2832dd..b4af7c8b2 100755--- a/scripts/minikube/t2.sh+++ b/scripts/minikube/t2.sh@@ -30,23 +30,26 @@ T2_TMP_ROOT="$T2_TMP_ROOT"if [ -z "$T2_TMP_ROOT" ]; then T2_TMP_ROOT=/tmp; fiT2_T1_OUTPUT="$T2_TMP_ROOT/evenfire-t1.$$.out"T2_FINAL_PREFLIGHT_OUTPUT="$T2_TMP_ROOT/evenfire-t2-final-preflight.$$.out"+export T2_PROJECT_DIR T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_CONTEXT T2_LOCK_TOKENcleanup_plan() {+ local status=$?if [ -n "$T2_PLAN_TMP" ] && [ -f "$T2_PLAN_TMP" ]; then rm -f "$T2_PLAN_TMP"; fiif [ -f "$T2_FINAL_PREFLIGHT_OUTPUT" ]; then rm -f "$T2_FINAL_PREFLIGHT_OUTPUT"; fiif [ -f "$T2_T1_OUTPUT" ]; then rm -f "$T2_T1_OUTPUT"; fi+ t2_lock_release "$status"}trap cleanup_plan EXITt2_lane_completed() {local status="$1"- [ "$status" = PASS ] || [ "$status" = SKIPPED ]+ [ "$status" = PASS ]}run_t0() {if [ "$T2_RUN_T0" != true ]; thenT2_T0_STATUS=SKIPPED- t2_evidence_write T0 SKIPPED 'T2_RUN_T0=false; static lane already certified on this HEAD'+ t2_evidence_write T0 SKIPPED 'T2_RUN_T0=false; an exact-head T0 attestation is required before runtime-only certification'return 0fiprintf '[minikube-t2] T0: shell syntax and contract checks\n'@@ -63,7 +66,7 @@ run_t0() {run_preflight_plan() {T2_PLAN_TMP="$(mktemp "$T2_TMP_ROOT/evenfire-t2-plan.XXXXXX")"- if ! T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE="$T2_PLAN_TMP" \+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE="$T2_PLAN_TMP" \T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \@@ -115,20 +118,24 @@ run_bootstrap_or_reconcile() {full-bootstrap)printf '[minikube-t2] bootstrap: full setup, then PostgreSQL/migrations/roles checks\n'if [ "$T2_RESET_PVC" = true ]; then- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \- REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setup+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \+ REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setupelse- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \REUSE_DB=true make minikube-setupfi;;full-reconcile)printf '[minikube-t2] full reconcile: infrastructure input changed\n'if [ "$T2_RESET_PVC" = true ]; then- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setupelse- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \REUSE_DB=true make minikube-setupfi;;@@ -187,17 +194,19 @@ run_pre_gate() {return 0fiprintf '[minikube-t2] pre-gate-sync after bootstrap/reconcile\n'- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-pre-gate-sync GATE=minikube-t2 ARGS='--skip-port-forwards'+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ MINIKUBE_PROFILE="$T2_PROFILE" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" \+ make minikube-pre-gate-sync GATE=minikube-t2 ARGS='--skip-port-forwards'}run_t1() {if [ "$T2_RUN_T1" != true ]; thenT2_T1_STATUS=SKIPPED- t2_evidence_write T1 SKIPPED 'T2_RUN_T1=false; Real PostgreSQL lane already certified on this HEAD'+ t2_evidence_write T1 SKIPPED 'T2_RUN_T1=false; an exact-head T1 attestation is required before runtime-only certification'return 0fiprintf '[minikube-t2] T1: Real PostgreSQL suites\n'- if T2_SKIP_LOCK=true CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \+ if T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \bash "$T2_PROJECT_DIR/scripts/e2e/minikube-real-postgres.sh" >"$T2_T1_OUTPUT" 2>&1; thencat "$T2_T1_OUTPUT"T2_T1_STATUS=PASS@@ -214,7 +223,7 @@ run_t1() {run_final_preflight() {printf '[minikube-t2] final exact-head readiness preflight\n'# Fail-loud: a stale marker or missing bootstrap must not count as T2.- if ! T2_SKIP_LOCK=true T2_PLAN_MODE=false T2_PLAN_FILE="$T2_PLAN_TMP" \+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE="$T2_PLAN_TMP" \T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \@@ -234,6 +243,7 @@ PYT2_NEXT_COMMAND='re-run make minikube-t2 so pre-gate-sync updates the marker to this HEAD't2_fail HEAD_MARKER_MISMATCH "final T2 preflight selected $T2_PLAN_STATE instead of already-synced"fi+ t2_marker_checkT2_T2_STATUS=PASSt2_evidence_write T2 PASS 'exact marker, image, PostgreSQL, namespace, Service, and deployment readiness passed'}@@ -277,11 +287,11 @@ main() {t2_repo_metadatat2_profile_scopet2_context_check- if [ "$T2_SKIP_LOCK" != true ]; then t2_lock_acquire; fi+ t2_mutation_lockt2_evidence_initrun_t0- if ! t2_lane_completed "$T2_T0_STATUS"; then+ if [ "$T2_RUN_T0" = true ] && ! t2_lane_completed "$T2_T0_STATUS"; thenT2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T0=true so the static lane is executed't2_fail ZERO_TESTS_EXECUTED 'T0 was not executed'fi@@ -290,13 +300,21 @@ main() {T2_NEXT_COMMAND='run make minikube-t2 so bootstrap/reconcile and T0/T1 execute on this HEAD't2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2-only mode requires an already-synced exact-head profile'fi+ if [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; then+ t2_marker_check+ t2_certification_validate_prior_lanes+ firun_bootstrap_or_reconcilerun_pre_gaterun_t1- if ! t2_lane_completed "$T2_T1_STATUS"; then+ if [ "$T2_RUN_T1" = true ] && ! t2_lane_completed "$T2_T1_STATUS"; thenT2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T1=true so the Real PostgreSQL lane is executed't2_fail ZERO_TESTS_EXECUTED 'T1 was not executed'fi+ if [ "$T2_RUN_T1" != true ] && [ "$T2_T1_CERTIFIED" != true ]; then+ T2_NEXT_COMMAND='run make minikube-t2 with T2_RUN_T1=true to create an exact-head T1 attestation'+ t2_fail CERTIFICATION_REQUIRED 'T1 was skipped without a validated exact-head attestation'+ firun_final_preflightrun_healthcheck_if_requestedrun_playwright_if_requesteddiff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.shindex 0a5d652fb..9343cf304 100755--- a/scripts/minikube/wait-gfs-reader-ready.sh+++ b/scripts/minikube/wait-gfs-reader-ready.sh@@ -20,6 +20,23 @@ POLL_SECONDS="${GFS_READER_WAIT_POLL_SECONDS:-5}"kc() { kubectl --context="$CONTEXT" "$@"; }log() { printf '[wait-gfs-reader-ready] %s\n' "$*" >&2; }+if ! TIMEOUT_SECONDS="$(python3 - "$TIMEOUT_SECONDS" <<'PY'+import re+import sys++raw = sys.argv[1]+if not re.fullmatch(r"[1-9][0-9]{0,4}", raw):+ raise SystemExit("timeout must be a positive integer")+value = int(raw)+if value > 86400:+ raise SystemExit("timeout exceeds the one-day limit")+print(value)+PY+)"; then+ log 'refusing an invalid readiness timeout'+ exit 2+fi+deadline=$((SECONDS + TIMEOUT_SECONDS))while :; dodesired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex 27739ea9a..7cd1e40b3 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -43,20 +43,27 @@ runtime_script="$(sed '/^[[:space:]]*#/d' deploy/scripts/provision-gfs-runtime.s|| fail 'post-overlay runtime path does not wait for exact GFSC rollouts'[[ "$runtime_script" != *'rollout restart'* && "$runtime_script" != *'rotate-writer'* ]] \|| fail 'post-overlay runtime path rotates or unconditionally restarts GFS'+reconcile_context_err="$(mktemp)"+if CONTEXT=unverified-context bash deploy/scripts/reconcile-gfs-deploy-credentials.sh 2>"$reconcile_context_err"; then+ rm -f "$reconcile_context_err"+ fail 'credential reconciliation accepted an unverified Kubernetes context'+fi+grep -Fq 'refusing unverified Kubernetes context' "$reconcile_context_err" \+ || fail 'credential reconciliation did not explain its unverified-context rejection'+rm -f "$reconcile_context_err"make_block() {awk -v target="$1" '$0 ~ "^" target ":" {active=1} active && /^\.PHONY:/ {exit} active {print}' Makefile}-for target in minikube-db-reset; do- block="$(make_block "$target")"- reset="$(grep -n 'reset-control-db-storage.sh' <<<"$block" | head -1 | cut -d: -f1)"- recreate="$(grep -n 'apply -k .* -l app=control-postgres' <<<"$block" | head -1 | cut -d: -f1)"- postgres_ready="$(grep -nE 'wait .*control-postgres|wait .*deploy/control-postgres' <<<"$block" | head -1 | cut -d: -f1)"- converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$block" | head -1 | cut -d: -f1)"- success="$(grep -n 'DB reset complete' <<<"$block" | tail -1 | cut -d: -f1)"- [[ -n "$reset" && -n "$recreate" && -n "$postgres_ready" && -n "$converge" && -n "$success" ]] && \- [[ "$reset" -lt "$recreate" && "$recreate" -lt "$postgres_ready" && "$postgres_ready" -lt "$converge" && "$converge" -lt "$success" ]] \- || fail "$target bypasses safe GFS reset recovery"-done+target=minikube-db-reset+block="$(make_block "$target")"+reset="$(grep -n 'reset-control-db-storage.sh' <<<"$block" | head -1 | cut -d: -f1)"+recreate="$(grep -n 'apply -k .* -l app=control-postgres' <<<"$block" | head -1 | cut -d: -f1)"+postgres_ready="$(grep -nE 'wait .*control-postgres|wait .*deploy/control-postgres' <<<"$block" | head -1 | cut -d: -f1)"+converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$block" | head -1 | cut -d: -f1)"+success="$(grep -n 'DB reset complete' <<<"$block" | tail -1 | cut -d: -f1)"+[[ -n "$reset" && -n "$recreate" && -n "$postgres_ready" && -n "$converge" && -n "$success" ]] && \+ [[ "$reset" -lt "$recreate" && "$recreate" -lt "$postgres_ready" && "$postgres_ready" -lt "$converge" && "$converge" -lt "$success" ]] \+ || fail "$target bypasses safe GFS reset recovery"reset_storage="$(cat deploy/scripts/reset-control-db-storage.sh)"for required in preflight-gfs-db-reset.sh 'scale deployment/control-api --replicas=0' 'scale deployment/control-postgres --replicas=0' 'wait_for_pods_gone app=control-api' 'wait_for_pods_gone app=control-postgres' 'preconditions":{"uid":"%s"}' 'persistentvolumeclaims/${PVC}'; do[[ "$reset_storage" == *"$required"* ]] || fail "reset storage boundary omits $required"@@ -113,7 +120,8 @@ assert_minikube_upgrade_classifier() {|| fail "$path does not reconcile ready upgrades, abort unready upgrades, and defer only empty bootstraps"}assert_pre_gate_defers_gfs_reconcile() {- local block="$({ awk -v start="$1" -v end="$2" '+ local block+ block="$({ awk -v start="$1" -v end="$2" 'index($0, start) { active=1 }active { print }active && index($0, end) { exit }diff --git a/scripts/tests/test-minikube-gfs-rollout-shim.sh b/scripts/tests/test-minikube-gfs-rollout-shim.shindex c56580d51..084f78570 100755--- a/scripts/tests/test-minikube-gfs-rollout-shim.sh+++ b/scripts/tests/test-minikube-gfs-rollout-shim.sh@@ -70,6 +70,20 @@ if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\nFalse|\n'fail 'shim readiness wait went green with a live unready reader pod'fi+# Timeout input is untrusted process input; it must be parsed as data rather+# than reaching Bash arithmetic evaluation.+malicious_timeout="1; touch $tmp/timeout-pwned"+if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\n' \+ GFS_READER_WAIT_POLL_SECONDS=0 \+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout="$malicious_timeout" >/dev/null 2>&1; then+ fail 'shim accepted a malformed rollout timeout'+fi+[ ! -e "$tmp/timeout-pwned" ] || fail 'malformed rollout timeout executed shell input'+if PATH="$shim_path" CONTEXT=fake GFS_READER_WAIT_TIMEOUT_SECONDS="$malicious_timeout" \+ bash "$WAIT" >/dev/null 2>&1; then+ fail 'direct readiness helper accepted a malformed timeout'+fi+# 5. A terminating unready pod does not block the wait.: >"$FAKE_KUBECTL_LOG"PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \diff --git a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh b/scripts/tests/test-minikube-settle-gfs-reader-rollout.shindex 98ea7a298..cbbb29f42 100755--- a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh+++ b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh@@ -9,17 +9,51 @@ fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }bash -n "$SCRIPT"run_settle() {- local fake_dir+ local fake_dir repo profile shafake_dir="$(mktemp -d)"+ repo="$fake_dir/repo"+ profile='clerum-settle-gfs-reader'+ mkdir -p "$repo" "$fake_dir/profiles/$profile" "$fake_dir/locks" "$fake_dir/evidence"+ repo="$(cd "$repo" && pwd -P)"+ git init -q -b dev "$repo" >/dev/null+ git -C "$repo" config user.email [email protected]+ git -C "$repo" config user.name settle-test+ printf 'settle fixture\n' >"$repo/README"+ git -C "$repo" add README >/dev/null+ git -C "$repo" commit -q -m fixture >/dev/null+ git -C "$repo" remote add origin https://github.com/evenfire-ai/evenfire.git+ git -C "$repo" switch -c feat/settle-gfs-reader >/dev/null+ git -C "$repo" update-ref refs/remotes/origin/dev HEAD+ sha="$(git -C "$repo" rev-parse --short HEAD)"+ cat >"$fake_dir/profiles/$profile/profile.env" <<EOF+PROFILE=$profile+REPO_DIR=$repo+BRANCH=feat/settle-gfs-reader+SHA_SHORT=$sha+DIRTY=false+EOF+ printf 'PORT_BASE=23457\n' >"$fake_dir/profiles/$profile/ports.env"cat >"$fake_dir/kubectl" <<'STUB'#!/usr/bin/env bashset -euo pipefailprintf '%s\n' "$*" >>"${FAKE_KUBECTL_LOG:?}"case " $* " in+ *' config get-contexts -o name'*)+ printf '%s' "${T2_CONTEXT:?}"; exit 0 ;;+ *' config view '* )+ printf 'https://%s:6443' '127.0.0.1'; exit 0 ;;*' get deployment gfsc-reader '*'-o jsonpath={.spec.replicas}'*)printf '%s' "${FAKE_DESIRED:-1}"; exit 0 ;;*' get deployment gfsc-reader '*'-o jsonpath={.status.readyReplicas}'*)printf '%s' "${FAKE_READY:-1}"; exit 0 ;;+ *' get deployment gfsc-reader '*gfs-template-hash*)+ printf '%s' 'template-42'; exit 0 ;;+ *' get deployment gfsc-reader '*jsonpath={.metadata.generation}*)+ printf '%s' '2'; exit 0 ;;+ *' get deployment gfsc-reader '*jsonpath={.status.observedGeneration}*)+ printf '%s' '2'; exit 0 ;;+ *' get deployment gfsc-reader '*jsonpath={range*)+ printf '%s\n' 'gfs-controller-reader-db'; exit 0 ;;*' get deployment gfsc-reader '*revision*)printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;*' get deployment gfsc-reader'*)@@ -27,8 +61,12 @@ case " $* " inexit 0 ;;*' get rs -l '*)printf '%b' "${FAKE_RS_ROWS:-}"; exit 0 ;;+ *' get rs gfsc-reader-proof-rs '*revision*)+ printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;*' scale rs '*)exit 0 ;;+ *' get pods -l '*gfs-template-hash*)+ printf '%b' "${FAKE_PROOF_POD_ROWS:-gfsc-reader-proof|2026-01-02T00:00:01Z|True||gfsc-reader-proof-rs|}"; exit 0 ;;*' get pods -l '*)printf '%b' "${FAKE_POD_ROWS:-}"; exit 0 ;;*' delete pod '*)@@ -37,19 +75,44 @@ case " $* " inprintf '%s' "${FAKE_STATE:-rollout-running}"; exit 0 ;;*' get secret gfs-controller-reader-db '*gfs-dsn-rotated-at*)printf '%s' "${FAKE_ROTATED:-2026-01-01T00:00:00Z}"; exit 0 ;;+ *' get secret gfs-controller-reader-db '*jsonpath={.metadata.resourceVersion}*)+ printf '%s' '42'; exit 0 ;;+ *' get secret gfs-controller-reader-db '*'-o json'*)+ python3 - <<'PY'+import base64+import json+import os+print(json.dumps({+ "metadata": {"resourceVersion": "42"},+ "data": {"connection-string": base64.b64encode(os.environ["FAKE_DSN"].encode()).decode()},+}))+PY+ exit 0 ;;*' get secret gfs-controller-reader-db'*)[ "${FAKE_SECRET_EXISTS:-1}" = 1 ] || exit 1exit 0 ;;+ *' exec '*)+ input="$(cat)"+ [ "$input" = "${FAKE_DSN:?}" ] || exit 1+ exit 0 ;;*' patch secret gfs-controller-reader-db'*)- cat >/dev/null+ cat >"${FAKE_PATCH_LOG:?}"[ "${FAKE_PATCH_OK:-1}" = 1 ] || exit 1exit 0 ;;esacexit 1STUBchmod +x "$fake_dir/kubectl"- PATH="$fake_dir:$PATH" CONTEXT=fake \+ local status=0+ PATH="$fake_dir:$PATH" CONTEXT="$profile" \+ T2_PROJECT_DIR="$repo" T2_PROFILE="$profile" T2_CONTEXT="$profile" \+ MINIKUBE_PROFILE="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" \+ T2_PROFILE_ROOT="$fake_dir/profiles" T2_PROFILE_ENV="$fake_dir/profiles/$profile/profile.env" \+ T2_PORTS_ENV="$fake_dir/profiles/$profile/ports.env" \+ T2_LOCK_ROOT="$fake_dir/locks" T2_EVIDENCE_ROOT="$fake_dir/evidence" \+ T2_GATE_ID=settle-gfs-reader-test T2_RUN_ID=settle-test-run \FAKE_KUBECTL_LOG="$fake_dir/kubectl.log" \+ FAKE_PATCH_LOG="$fake_dir/patch.json" FAKE_DSN=dsn-value \FAKE_DEPLOY_EXISTS="${1:-1}" \FAKE_DESIRED="${2:-1}" \FAKE_READY="${3:-1}" \@@ -60,13 +123,24 @@ STUBFAKE_RS_ROWS="${8:-}" \FAKE_POD_ROWS="${9:-}" \FAKE_REVISION="${10:-19}" \- bash "$SCRIPT"+ FAKE_PROOF_POD_ROWS="${11:-}" \+ bash "$SCRIPT" || status=$?printf '%s' "$fake_dir"+ return "$status"}dir="$(run_settle 1 1 1 1 rollout-running '2026-01-01T00:00:00Z' 1)"grep -q 'patch secret gfs-controller-reader-db' "$dir/kubectl.log" \|| fail 'Ready leftover claim did not patch the reader Secret to ready'+grep -q 'gfs-dsn-proof-kind' "$dir/patch.json" \+ || fail 'Ready leftover claim did not persist credential-consumption proof'+rm -rf "$dir"++if dir="$(run_settle 1 1 1 1 rollout-running '2026-01-01T00:00:00Z' 1 '' '' 19 'gfsc-reader-old|2025-12-31T23:59:59Z|True||gfsc-reader-old-rs|')"; then+ fail 'a pre-rotation Ready reader incorrectly certified the rollout'+fi+grep -q 'patch secret' "$dir/kubectl.log" \+ && fail 'a pre-rotation Ready reader must not be marked ready'rm -rf "$dir"dir="$(run_settle 1 1 0 1 rollout-running '2026-01-01T00:00:00Z' 1)"diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex 717c11aaf..55f29ebe1 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -25,7 +25,7 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \bash -n "$file"done-required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT"+required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY PROFILE_LOCK_REQUIRED HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED CERTIFICATION_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT"for code in $required_codes; dogrep -Fq "$code" "$COMMON" "$PREFLIGHT" "$T2" "$T1"done@@ -50,12 +50,14 @@ grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"grep -Fq 'postgres:16-alpine' "$T1"-grep -Fq 'db.realPostgresMigration.integration.test.ts' "$T1"-grep -Fq 'gfsReaderRole.realPostgres.integration.test.ts' "$T1"+grep -Fq 'control-api/test/*realPostgres*.test.ts' "$T1"grep -Fq 'start_isolated_postgres' "$T1"grep -Fq 'require_isolated_control_api_files' "$T1"grep -Fq 'run_suite control-api isolated "$ISOLATED_DSN"' "$T1"-grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"+if grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"; then+ echo 'FAIL: control-api Real PostgreSQL suites must not use shared control-postgres' >&2+ exit 1+fiif grep -Fq 'run_suite control-api isolated "$ADMIN_DSN"' "$T1"; thenecho 'FAIL: migration/role-reset lane points at shared control-postgres' >&2exit 1@@ -67,11 +69,8 @@ figrep -Fq 'T1_REDACT_PASSWORD="${PG_PASSWORD}"' "$T1" || grep -Fq 'T1_REDACT_PASSWORD="$PG_PASSWORD"' "$T1"grep -Fq -- '--reporter=json' "$T1"grep -Fq '|| suite_status=$?' "$T1"-grep -Fq 'complete green reporter' "$T1"-if grep -Fq 'Real PostgreSQL suite failed in $package ($lane)' "$T1"; then- echo 'FAIL: T1 still treats npm exit as a suite failure before the JSON reporter' >&2- exit 1-fi+grep -Fq 'Vitest process exited $suite_status' "$T1"+grep -Fq 'complete green JSON reporter' "$T1"grep -Fq 'pending_tests' "$T1"grep -Fq 'numTotalTests' "$T1"grep -Fq 'port-forward' "$T1"@@ -155,8 +154,8 @@ grep -Fq 'run_pre_gate' "$T2"grep -Fq 'run_targeted_sync' "$T2"grep -Fq 'full-reconcile' "$T2"grep -Fq 'already-synced' "$T2" "$COMMON"-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"+grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"+grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"grep -Fq 'T2_PLAN_MODE=false' "$PREFLIGHT"grep -Fq -- '--skip-port-forwards' "$T2"grep -Fq 't2_lane_completed' "$T2"@@ -176,7 +175,8 @@ if grep -Eq 'make minikube-pre-gate-sync GATE=minikube-t2$' "$T2"; thenexit 1fi-grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"+grep -Fq 't2_mutation_lock' "$COMMON"+grep -Fq 't2_lock_validate_inherited' "$COMMON"grep -Fq 'T2_LOCK_ROOT' "$COMMON"grep -Fq 'REPOSITORY=' "$COMMON"grep -Fq 'BRANCH=' "$COMMON"@@ -184,7 +184,10 @@ grep -Fq 'HEAD=' "$COMMON"grep -Fq 'PROFILE=' "$COMMON"grep -Fq 'CONTEXT=' "$COMMON"grep -Fq 'LOCK_KEY=' "$COMMON"+grep -Fq 'TOKEN=' "$COMMON"+grep -Fq 'WORKTREE_ID=' "$COMMON"grep -Fq 'PID=' "$COMMON"+grep -Fq 'PROCESS_START=' "$COMMON"if grep -Eq 'get secret .* -o (yaml|json)' "$COMMON" "$PREFLIGHT"; thenecho 'FAIL: preflight reads Secret values' >&2@@ -212,15 +215,44 @@ bash -c 'parsed="$(t2_get_name control-plane/control-postgres)"test "$parsed" = "control-plane control-postgres"t2_lock_acquire+ t2_lock_release' bash "$COMMON"if [ -e "$tmp/locks/clerum-test-contract.lock" ]; thenecho 'FAIL: lock was not cleaned after clean exit' >&2exit 1fi+# A nested mutator must prove the exact opaque lease token and owner identity;+# T2_SKIP_LOCK=true alone is not an authorization mechanism.+T2_LOCK_ROOT="$tmp/fenced-locks" T2_PROJECT_DIR="$ROOT" \+T2_BRANCH=feat/contract-fence T2_HEAD=0123456789abcdef \+T2_ORIGIN_DEV=0123456789abcdef T2_MERGE_BASE=0123456789abcdef \+MINIKUBE_PROFILE=clerum-fence T2_CONTEXT=clerum-fence \+T2_WORKTREE_ID=contract-worktree \+bash -c '+ common="$1"+ source "$common"+ t2_lock_acquire+ token="$T2_LOCK_TOKEN"+ export T2_PROJECT_DIR T2_BRANCH T2_HEAD T2_ORIGIN_DEV T2_MERGE_BASE+ export MINIKUBE_PROFILE T2_PROFILE T2_CONTEXT T2_WORKTREE_ID T2_LOCK_ROOT+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$token" bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; then+ exit 1+ fi+ if T2_SKIP_LOCK=true T2_LOCK_TOKEN=wrong bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; then+ exit 1+ fi+ t2_lock_release 0+' bash "$COMMON"+if [ -e "$tmp/fenced-locks/clerum-fence.lock" ]; then+ echo 'FAIL: fenced lock survived the owner release' >&2+ exit 1+fi+mkdir -p "$tmp/locks/busy-profile.lock"cat >"$tmp/locks/busy-profile.lock/owner.env" <<EOFPROFILE=busy-profile+TOKEN=lockPID=$$EOFif T2_LOCK_ROOT="$tmp/locks" MINIKUBE_PROFILE=busy-profile T2_CONTEXT=busy-profile \@@ -243,6 +275,103 @@ figrep -Fq 'PROFILE_BUSY' "$tmp/empty.err"grep -Fq 'orphaned' "$tmp/empty.err"+cert_root="$tmp/certifications"+mkdir -p "$cert_root/prior" "$cert_root/current"+CERT_FILE="$cert_root/prior/evidence.json" EXPECTED_REPOSITORY="$ROOT" \+EXPECTED_BRANCH=feat/certification EXPECTED_HEAD=cert-head EXPECTED_ORIGIN_DEV=cert-dev \+EXPECTED_WORKTREE_ID=cert-worktree EXPECTED_PROFILE=clerum-cert EXPECTED_CONTEXT=clerum-cert \+EXPECTED_FINGERPRINT=cert-fingerprint EXPECTED_GATE_ID=cert-gate \+python3 - <<'PY'+import json+import os+from datetime import datetime, timedelta, timezone++now = datetime.now(timezone.utc)+payload = {+ "certificationVersion": 1,+ "attestationStatus": "PASS",+ "runId": "prior-certification-run",+ "attestationStartedAt": now.isoformat().replace("+00:00", "Z"),+ "attestationExpiresAt": (now + timedelta(hours=1)).isoformat().replace("+00:00", "Z"),+ "repository": os.environ["EXPECTED_REPOSITORY"],+ "branch": os.environ["EXPECTED_BRANCH"],+ "head": os.environ["EXPECTED_HEAD"],+ "originDev": os.environ["EXPECTED_ORIGIN_DEV"],+ "worktreeId": os.environ["EXPECTED_WORKTREE_ID"],+ "profile": os.environ["EXPECTED_PROFILE"],+ "context": os.environ["EXPECTED_CONTEXT"],+ "clusterFingerprintRef": os.environ["EXPECTED_FINGERPRINT"],+ "gateId": os.environ["EXPECTED_GATE_ID"],+ "phases": [+ {"name": "T0", "status": "PASS"},+ {"name": "T1", "status": "PASS"},+ {"name": "complete", "status": "PASS"},+ ],+}+with open(os.environ["CERT_FILE"], "w") as handle:+ json.dump(payload, handle)+PY+T2_EVIDENCE_ROOT="$cert_root" T2_PROJECT_DIR="$ROOT" T2_BRANCH=feat/certification \+T2_HEAD=cert-head T2_ORIGIN_DEV=cert-dev T2_MERGE_BASE=cert-dev \+T2_WORKTREE_ID=cert-worktree T2_PROFILE=clerum-cert T2_CONTEXT=clerum-cert \+T2_CLUSTER_FINGERPRINT=cert-fingerprint T2_GATE_ID=cert-gate \+T2_RUN_ID=current-certification-run \+bash -c '+ source "$1"+ T2_PROJECT_DIR="$3"+ T2_BRANCH=feat/certification+ T2_HEAD=cert-head+ T2_ORIGIN_DEV=cert-dev+ T2_MERGE_BASE=cert-dev+ T2_WORKTREE_ID=cert-worktree+ T2_PROFILE=clerum-cert+ T2_CONTEXT=clerum-cert+ T2_CLUSTER_FINGERPRINT=cert-fingerprint+ T2_GATE_ID=cert-gate+ T2_RUN_ID=current-certification-run+ T2_EVIDENCE_DIR="$2/current-run"+ T2_EVIDENCE_FILE="$T2_EVIDENCE_DIR/evidence.json"+ mkdir -p "$T2_EVIDENCE_DIR"+ t2_certification_validate_prior_lanes+ test "$T2_T0_CERTIFIED" = true+ test "$T2_T1_CERTIFIED" = true+ grep -Fq T0_ATTESTED "$T2_EVIDENCE_FILE"+ grep -Fq T1_ATTESTED "$T2_EVIDENCE_FILE"+' bash "$COMMON" "$cert_root" "$ROOT"++python3 - "$cert_root/prior/evidence.json" <<'PY'+import json+import sys++with open(sys.argv[1]) as handle:+ payload = json.load(handle)+payload["attestationStatus"] = "INVALIDATED"+with open(sys.argv[1], "w") as handle:+ json.dump(payload, handle)+PY+if T2_EVIDENCE_ROOT="$cert_root" T2_PROJECT_DIR="$ROOT" T2_BRANCH=feat/certification \+ T2_HEAD=cert-head T2_ORIGIN_DEV=cert-dev T2_MERGE_BASE=cert-dev \+ T2_WORKTREE_ID=cert-worktree T2_PROFILE=clerum-cert T2_CONTEXT=clerum-cert \+ T2_CLUSTER_FINGERPRINT=cert-fingerprint T2_GATE_ID=cert-gate \+ bash -c '+ source "$1"+ T2_PROJECT_DIR="$2"+ T2_BRANCH=feat/certification+ T2_HEAD=cert-head+ T2_ORIGIN_DEV=cert-dev+ T2_MERGE_BASE=cert-dev+ T2_WORKTREE_ID=cert-worktree+ T2_PROFILE=clerum-cert+ T2_CONTEXT=clerum-cert+ T2_CLUSTER_FINGERPRINT=cert-fingerprint+ T2_GATE_ID=cert-gate+ t2_certification_validate_prior_lanes+ ' bash "$COMMON" "$ROOT" 2>"$tmp/certification.err"; then+ echo 'FAIL: invalidated certification was accepted' >&2+ exit 1+fi+grep -Fq CERTIFICATION_REQUIRED "$tmp/certification.err"+redaction_file="$tmp/t1-redaction.log"redaction_secret='super-secret'printf '%s=%s\n' password "$redaction_secret" >"$redaction_file"@@ -255,6 +384,45 @@ if grep -Fq "$redaction_secret" "$redaction_file"; thenexit 1fi+# A complete green reporter is not sufficient evidence when npm/Vitest exits+# non-zero during teardown, worker shutdown, OOM handling, or signal cleanup.+# Exercise the adjudicator with representative failure statuses without Docker+# or Kubernetes; every one must fail the suite despite the green JSON payload.+t1_exit_root="$tmp/t1-exit-adjudicator"+mkdir -p "$t1_exit_root/control-api/test"+printf 'fixture\n' >"$t1_exit_root/control-api/test/fake.realPostgres.test.ts"+for exit_code in 1 2 7 126 137; do+ if EXIT_CODE="$exit_code" T1_EXIT_ROOT="$t1_exit_root" bash -c '+ set -euo pipefail+ source "$1"+ PROJECT_DIR="$T1_EXIT_ROOT"+ T1_TMP_DIR="$(mktemp -d)"+ trap '\''rm -rf "$T1_TMP_DIR"'\'' EXIT+ list_real_pg_files() {+ printf "%s\\n" "$T1_EXIT_ROOT/control-api/test/fake.realPostgres.test.ts"+ }+ is_isolated_control_api_file() { return 0; }+ npm() {+ local output_file=""+ while [ "$#" -gt 0 ]; do+ case "$1" in+ --outputFile=*) output_file="${1#--outputFile=}" ;;+ --outputFile) shift; output_file="$1" ;;+ esac+ shift+ done+ printf "%s\\n" '\''{"success":true,"testResults":[{"status":"passed"}],"numPassedTests":1,"numFailedTests":0,"numPendingTestSuites":0,"numPendingTests":0,"numTotalTests":1}'\'' >"$output_file"+ return "$EXIT_CODE"+ }+ if run_suite control-api isolated postgresql://isolated; then+ exit 1+ fi+ ' bash "$T1"; then+ echo "FAIL: T1 accepted a green reporter after npm exit $exit_code" >&2+ exit 1+ fi+done+bash "$ROOT/scripts/tests/test-minikube-t2-public-boundary.sh"if T2_PUBLIC_BASE_REF=nonexistent-ref-for-contract \bash "$ROOT/scripts/tests/test-minikube-t2-public-boundary.sh"; thendiff --git a/scripts/tests/test-minikube-t2-public-boundary.sh b/scripts/tests/test-minikube-t2-public-boundary.shindex 8813e4a7d..a21cedc3d 100755--- a/scripts/tests/test-minikube-t2-public-boundary.sh+++ b/scripts/tests/test-minikube-t2-public-boundary.sh@@ -40,6 +40,11 @@ import sysdiff = Path(sys.argv[1]).read_text(errors="replace")bad = []current = ""+safe_source_paths = {+ "deploy/scripts/lib/gfs-credential-rollout.sh",+ "deploy/scripts/lib/gfs-credential-secret.sh",+ "deploy/scripts/reconcile-gfs-deploy-credentials.sh",+}for line in diff.splitlines():if line.startswith("+++ b/"):current = line[6:]@@ -49,7 +54,10 @@ for line in diff.splitlines():or path.startswith(".env.")or path.endswith((".pem", ".key", ".p12", ".pfx", ".log"))or path.endswith(("/kubeconfig", "/config"))- or any(token in path for token in ("id_rsa", "id_ed25519", "credential", "wallet", "keystore"))+ or (+ any(token in path for token in ("id_rsa", "id_ed25519", "credential", "wallet", "keystore"))+ and path not in safe_source_paths+ )or "screenshot" in pathor "e2e-artifacts" in path):diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex c4ac168ef..1f3f3e0fb 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -60,6 +60,7 @@ printf 'PORT_BASE=23117\nCONTROL_API_URL=profile-owned-url\n' >"$profile_root/porepo_env=(T2_PROJECT_DIR="$repo"MINIKUBE_PROFILE="$profile"+ T2_CONTEXT="$profile"CONTROL_API_REAL_PG_CONTEXT="$profile"T2_PROFILE_ROOT="$tmp/profiles"T2_PROFILE_ENV="$profile_root/profile.env"@@ -88,7 +89,7 @@ expect_code DEVELOPMENT_SCOPE_REQUIRED shared-profile shared-profile \bash -c 'source "$1"; t2_profile_scope' bash "$COMMON"expect_code DEVELOPMENT_SCOPE_REQUIRED context-mismatch context-mismatch \- env "${repo_env[@]}" CONTROL_API_REAL_PG_CONTEXT=another-context \+ env "${repo_env[@]}" T2_CONTEXT=another-context CONTROL_API_REAL_PG_CONTEXT=another-context \bash -c 'source "$1"; t2_profile_scope' bash "$COMMON"fake_bin="$tmp/fake-bin"@@ -96,20 +97,27 @@ mkdir -p "$fake_bin"cat >"$fake_bin/kubectl" <<'EOF'#!/usr/bin/env bashcase "$*" in- *"config current-context"*) exit 0 ;;+ *"config get-contexts -o name"*)+ for arg in "$@"; do+ case "$arg" in+ --context=*) printf '%s' "${arg#--context=}"; exit 0 ;;+ esac+ done+ printf '%s' "${FAKE_CONTEXT:-}" ;;*"config view"*) printf '%s://%s:6443' https "${FAKE_ENDPOINT:-10.0.0.1}" ;;*) exit 0 ;;esacEOFchmod +x "$fake_bin/kubectl"+context_env=("${repo_env[@]}" FAKE_CONTEXT="$profile")expect_code DEVELOPMENT_SCOPE_REQUIRED remote-context remote-context \- env "${repo_env[@]}" PATH="$fake_bin:$PATH" \+ env "${repo_env[@]}" FAKE_CONTEXT=other-context PATH="$fake_bin:$PATH" \bash -c 'source "$1"; t2_context_check' bash "$COMMON"-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=127.0.0.1 \+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=127.0.0.1 \bash -c 'source "$1"; t2_context_check' bash "$COMMON"-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \bash -c 'source "$1"; t2_context_check' bash "$COMMON"-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \bash -c 'source "$1"; t2_context_check' bash "$COMMON"missing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"@@ -264,9 +272,9 @@ grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$ROOT/scripts/e2e/minikube-real-postggrep -Fq 'ZERO_TESTS_EXECUTED' "$ROOT/scripts/e2e/minikube-real-postgres.sh"grep -Fq 'start_isolated_postgres' "$ROOT/scripts/e2e/minikube-real-postgres.sh"grep -Fq 'ISOLATED_DSN' "$ROOT/scripts/e2e/minikube-real-postgres.sh"-grep -Fq '|| suite_status=$?' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must capture vitest exit without aborting before the reporter'-grep -Fq 'complete green reporter' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must document reporter-authoritative verdict'-grep -Fq 'Real PostgreSQL suite failed in $package ($lane)' "$ROOT/scripts/e2e/minikube-real-postgres.sh" && fail 'T1 still treats npm exit as a suite failure before the JSON reporter'+grep -Fq '|| suite_status=$?' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must capture vitest exit before adjudicating the reporter'+grep -Fq 'complete green JSON reporter' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must require a complete green reporter'+grep -Fq 'Vitest process exited $suite_status' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must fail on a non-zero Vitest exit'grep -Fq 'restore_gfs_runtime_credentials' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must restore branch-profile GFS credentials on exit'grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 GFS restore must opt into the NOLOGIN recovery contract'grep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 GFS restore must resume an interrupted gfsc-reader rollout claim'@@ -284,7 +292,8 @@ grep -Fq 'wait-gfs-reader-ready.sh' "$ROOT/scripts/minikube/pre-gate-sync.sh" ||[[ "$(grep -c 'gfs-rollout-shim' "$ROOT/scripts/minikube/full-setup.sh")" -ge 2 ]] || fail 'full-setup REUSE_DB reconciles must use the HCC-safe reader rollout wait on both calls'[[ "$(grep -c 'sync-auth-key.sh' "$ROOT/scripts/minikube/full-setup.sh")" -ge 3 ]] || fail 'full-setup must re-sync gfs-config.jwt-public-key before both GFS reconciles'grep -Fq 'T2_LOCK_ROOT' "$COMMON"-grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"+grep -Fq 't2_mutation_lock' "$COMMON"+grep -Fq 't2_lock_validate_inherited' "$COMMON"grep -Fq 'PORT_FORWARD_CONFLICT' "$COMMON"grep -Fq 'while read -r uid pid ppid rest' "$COMMON" || fail 't2_process_check must split ps -ef fields'grep -Fq 'IFS= read -r uid pid ppid rest' "$COMMON" && fail 'IFS= read disables PID split and silences PORT_FORWARD_CONFLICT'@@ -314,7 +323,7 @@ grep -Fq 'kubectl -n control-plane port-forward svc/control-api' "$tmp/pf-awk.ougrep -Fq -- "--context ${pf_profile} port-forward svc/control-ui" "$tmp/pf-awk.out" || fail 'awk missed run-e2e real launcher'grep -Fq 'kubectl port-forward --context=' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'grep -Fq '/usr/local/bin/kubectl --context=' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl with flags before port-forward'-read -r uid pid ppid rest <<<"$pf_all"+read -r _ pid _ _ <<<"$pf_all"[ "$pid" = 4242 ] || fail "ps field split left pid='$pid' instead of 4242"pf_root="$tmp/pf-check-profiles"diff --git a/scripts/tests/test-provision-gfs-runtime.sh b/scripts/tests/test-provision-gfs-runtime.shindex 0201a8abe..2740a6431 100755--- a/scripts/tests/test-provision-gfs-runtime.sh+++ b/scripts/tests/test-provision-gfs-runtime.sh@@ -39,13 +39,14 @@ export CALL_LOG="$TMP/calls.log"# through the ALLOWED_CONTEXTS override — the same mechanism a self-hosted dev# pipeline uses for a GKE dev context that the generic patterns do not name.for run in first-upgrade idempotent-rerun; do+ : "$run"ALLOWED_CONTEXTS=clerum-dev-test PATH="$TMP/bin:$PATH" /bin/bash "$ROOT/deploy/scripts/provision-gfs-runtime.sh" \--context clerum-dev-test \--overlay "$TMP/overlay" \--skip-auth-sync >/dev/nulldonePATH="$TMP/bin:$PATH" /bin/bash "$ROOT/deploy/scripts/provision-gfs-runtime.sh" \- --context gke-prod-test \+ --context gke_prod_test \--overlay "$TMP/overlay" \--allow-prod \--skip-auth-sync \