Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: close PR 416 rollout gate findings

Alfredo Lopeze26e24dc4ea9
diff --git a/.gitignore b/.gitignore
index 1fcc0af27..02b789b6d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -95,6 +95,7 @@ docs/analysis/
# Local developer notes (untracked)
.local-notes/
+.ralph/plans/
# Agent session transcripts can contain internal infrastructure details.
session-ses_*.md
diff --git a/deploy/scripts/lib/gfs-credential-rollout.sh b/deploy/scripts/lib/gfs-credential-rollout.sh
index 9b13065c7..1c8bf071e 100755
--- a/deploy/scripts/lib/gfs-credential-rollout.sh
+++ b/deploy/scripts/lib/gfs-credential-rollout.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
# Targeted GFSC rollout transitions after a credential is committed.
+# shellcheck disable=SC2034
deployment_uses_secret() {
local deployment="$1" secret="$2" refs error
@@ -61,6 +62,11 @@ credential_rollout_required() {
credential_adoption_timestamp() {
local deployment="$1" selector="$2" expected_dsn="$3" rows live oldest="" pod created ready
if ! deployment_exists "$deployment"; then
+ GFS_ROLLOUT_PROOF_KIND='deployment-absent'
+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""
+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION='future-pod'
+ GFS_ROLLOUT_PROOF_POD_COUNT=0
+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
date -u '+%Y-%m-%dT%H:%M:%SZ'
return 0
fi
@@ -88,6 +94,104 @@ rollout_exact() {
kc -n "$GFS_NS" rollout status "deployment/${deployment}" --timeout="$ROLLOUT_TIMEOUT"
}
+credential_rollout_proof() {
+ local secret="$1" deployment="$2" rotated_at="$3"
+ local secret_json secret_rv active_dsn template_revision generation observed_revision
+ local selector rows pod_name pod_created pod_ready pod_deleting pod_owner pod_revision
+ local owner_revision
+ local pod_count=0 current_rv
+ if ! deployment_uses_secret "$deployment" "$secret"; then
+ die "$GFS_NS/$deployment does not reference $GFS_NS/$secret; refusing to certify credential consumption"
+ fi
+ if ! secret_json="$(kc -n "$GFS_NS" get secret "$secret" -o json)"; then
+ die "cannot snapshot $GFS_NS/$secret for rollout proof"
+ fi
+ if ! IFS=$'\t' read -r secret_rv active_dsn < <(printf '%s' "$secret_json" | python3 -c 'import base64, json, sys
+obj = json.load(sys.stdin)
+metadata = obj.get("metadata") or {}
+encoded = (obj.get("data") or {}).get("connection-string") or ""
+if not encoded:
+ raise SystemExit(1)
+try:
+ dsn = base64.b64decode(encoded, validate=True).decode()
+except (ValueError, UnicodeDecodeError):
+ raise SystemExit(1)
+print(str(metadata.get("resourceVersion", "")) + "\t" + dsn)'); then
+ die "cannot decode the committed $GFS_NS/$secret credential for rollout proof"
+ fi
+ [ -n "$secret_rv" ] || die "the $GFS_NS/$secret rollout proof has no resourceVersion"
+ # The standard Deployment revision is the portable template identity. A
+ # custom annotation is not guaranteed to be copied to Pods by Kubernetes or
+ # by HCC's reconciler, so it cannot be the sole rollout proof.
+ template_revision="$(kc -n "$GFS_NS" get deployment "$deployment" \
+ -o 'jsonpath={.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
+ generation="$(kc -n "$GFS_NS" get deployment "$deployment" \
+ -o 'jsonpath={.metadata.generation}' 2>/dev/null || true)"
+ observed_revision="$(kc -n "$GFS_NS" get deployment "$deployment" \
+ -o 'jsonpath={.status.observedGeneration}' 2>/dev/null || true)"
+ if [[ ! "$generation" =~ ^[1-9][0-9]*$ ]] ||
+ [[ ! "$observed_revision" =~ ^[1-9][0-9]*$ ]] ||
+ [ "$generation" != "$observed_revision" ]; then
+ die "$GFS_NS/$deployment generation is not observed; refusing a pre-rollout credential proof"
+ fi
+ [ -n "$template_revision" ] ||
+ die "$GFS_NS/$deployment has no observed Deployment revision; refusing an unverifiable credential proof"
+ case "$deployment" in
+ gfsc-reader) selector='app=gfs-controller,clerum.io/gfsc-role=reader' ;;
+ gfsc-writer) selector='app=gfs-controller,clerum.io/gfsc-role=writer' ;;
+ *) die "unsupported GFSC deployment for credential proof: $deployment" ;;
+ esac
+ if ! rows="$(kc -n "$GFS_NS" get pods -l "$selector" -o \
+ 'jsonpath={range .items[*]}{.metadata.name}{"|"}{.metadata.creationTimestamp}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"|"}{.metadata.deletionTimestamp}{"|"}{.metadata.ownerReferences[0].name}{"|"}{.metadata.annotations.clerum\.io/gfs-template-hash}{"\n"}{end}')"; then
+ die "cannot list $GFS_NS/$deployment pods for credential proof"
+ fi
+ while IFS='|' read -r pod_name pod_created pod_ready pod_deleting pod_owner pod_revision; do
+ [ -n "$pod_name" ] || continue
+ [ -z "$pod_deleting" ] || continue
+ if ! python3 - "$pod_created" "$rotated_at" <<'PY'
+from datetime import datetime, timezone
+import sys
+created, rotated = sys.argv[1:]
+def parse(value):
+ return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
+try:
+ ok = parse(created) >= parse(rotated)
+except ValueError:
+ ok = False
+raise SystemExit(0 if ok else 1)
+PY
+ then
+ die "$GFS_NS/$deployment Ready pod predates the committed credential rotation"
+ fi
+ [ "$pod_ready" = True ] || die "$GFS_NS/$deployment has a live pod that is not Ready"
+ owner_revision="$(kc -n "$GFS_NS" get rs "$pod_owner" \
+ -o 'jsonpath={.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
+ [ -n "$owner_revision" ] || owner_revision="$pod_revision"
+ [ "$owner_revision" = "$template_revision" ] ||
+ die "$GFS_NS/$deployment Ready pod is not on the observed template revision"
+ if ! printf '%s' "$active_dsn" |
+ kc -n "$GFS_NS" exec -i "$pod_name" -c gfsc -- node -e '
+let input = "";
+process.stdin.setEncoding("utf8");
+process.stdin.on("data", chunk => input += chunk);
+process.stdin.on("end", () => process.exit(input === process.env.GFS_PG_CONNECTION_STRING ? 0 : 1));' \
+ >/dev/null 2>&1; then
+ die "$GFS_NS/$deployment Ready pod did not consume the committed credential"
+ fi
+ pod_count=$((pod_count + 1))
+ done <<<"$rows"
+ [ "$pod_count" -gt 0 ] || die "no live $GFS_NS/$deployment pod exists for credential proof"
+ current_rv="$(kc -n "$GFS_NS" get secret "$secret" -o \
+ 'jsonpath={.metadata.resourceVersion}' 2>/dev/null || true)"
+ [ "$current_rv" = "$secret_rv" ] ||
+ die "$GFS_NS/$secret changed during rollout proof; refusing stale readiness"
+ GFS_ROLLOUT_PROOF_KIND='pod-env'
+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION="$secret_rv"
+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION="$template_revision"
+ GFS_ROLLOUT_PROOF_POD_COUNT="$pod_count"
+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+}
+
complete_rollout() {
local secret="$1" deployment="$2" rotated state rc
rotated="$(gfs_secret_rotated_at "$secret")"
@@ -106,11 +210,17 @@ complete_rollout() {
esac
if ! deployment_exists "$deployment"; then
log "${GFS_NS}/${deployment} is not present; credential is staged for its future pod"
+ GFS_ROLLOUT_PROOF_KIND='deployment-absent'
+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""
+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION='future-pod'
+ GFS_ROLLOUT_PROOF_POD_COUNT=0
+ GFS_ROLLOUT_PROOF_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
mark_secret_rollout_ready "$secret" "$rotated" rollout-running \
|| die "staged credential state was superseded; refusing a stale completion"
return 0
fi
if rollout_exact "$deployment"; then
+ credential_rollout_proof "$secret" "$deployment" "$rotated"
mark_secret_rollout_ready "$secret" "$rotated" rollout-running \
|| die "${deployment} rolled out but credential state was superseded; refusing a stale completion"
return 0
diff --git a/deploy/scripts/lib/gfs-credential-secret.sh b/deploy/scripts/lib/gfs-credential-secret.sh
index d6cb34291..be130bfff 100755
--- a/deploy/scripts/lib/gfs-credential-secret.sh
+++ b/deploy/scripts/lib/gfs-credential-secret.sh
@@ -3,6 +3,11 @@
# Candidate connection material is accepted only on stdin.
# Snapshot globals are consumed by the provisioning script that sources this file.
# shellcheck disable=SC2034
+GFS_ROLLOUT_PROOF_KIND=""
+GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""
+GFS_ROLLOUT_PROOF_TEMPLATE_REVISION=""
+GFS_ROLLOUT_PROOF_POD_COUNT=""
+GFS_ROLLOUT_PROOF_AT=""
gfs_secret_state() {
kc -n "$GFS_NS" get secret "$1" -o 'jsonpath={.metadata.annotations.clerum\.io/gfs-dsn-state}'
@@ -189,12 +194,37 @@ print(json.dumps([
mark_secret_rollout_ready() {
local secret="$1" expected_rotated_at="$2" expected_state="$3"
+ local proof_kind proof_resource_version proof_template_revision proof_pod_count proof_at
+ proof_kind="$GFS_ROLLOUT_PROOF_KIND"
+ proof_resource_version="$GFS_ROLLOUT_PROOF_RESOURCE_VERSION"
+ proof_template_revision="$GFS_ROLLOUT_PROOF_TEMPLATE_REVISION"
+ proof_pod_count="$GFS_ROLLOUT_PROOF_POD_COUNT"
+ proof_at="$GFS_ROLLOUT_PROOF_AT"
python3 -c 'import json, sys
-state, stamp = sys.argv[1:]
-print(json.dumps([
+state, stamp, proof_kind, proof_resource_version, proof_template_revision, proof_pod_count, proof_at = sys.argv[1:]
+ops = [
{"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": state},
{"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-rotated-at", "value": stamp},
{"op": "replace", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": "ready"},
-]))' "$expected_state" "$expected_rotated_at" \
+]
+if proof_kind:
+ if proof_resource_version:
+ ops.insert(2, {"op": "test", "path": "/metadata/resourceVersion", "value": proof_resource_version})
+ values = {
+ "clerum.io/gfs-dsn-proof-kind": proof_kind,
+ "clerum.io/gfs-dsn-proof-resource-version": proof_resource_version,
+ "clerum.io/gfs-dsn-proof-template-revision": proof_template_revision,
+ "clerum.io/gfs-dsn-proof-pod-count": proof_pod_count,
+ "clerum.io/gfs-dsn-proof-at": proof_at,
+ }
+ for key, value in values.items():
+ ops.append({"op": "add", "path": "/metadata/annotations/" + key.replace("/", "~1"), "value": value})
+print(json.dumps(ops))' "$expected_state" "$expected_rotated_at" \
+ "$proof_kind" "$proof_resource_version" "$proof_template_revision" "$proof_pod_count" "$proof_at" \
| kc -n "$GFS_NS" patch secret "$secret" --type=json --patch-file=/dev/stdin >/dev/null
+ GFS_ROLLOUT_PROOF_KIND=""
+ GFS_ROLLOUT_PROOF_RESOURCE_VERSION=""
+ GFS_ROLLOUT_PROOF_TEMPLATE_REVISION=""
+ GFS_ROLLOUT_PROOF_POD_COUNT=""
+ GFS_ROLLOUT_PROOF_AT=""
}
diff --git a/deploy/scripts/reconcile-gfs-deploy-credentials.sh b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
index 6d821c606..a9a718a21 100755
--- a/deploy/scripts/reconcile-gfs-deploy-credentials.sh
+++ b/deploy/scripts/reconcile-gfs-deploy-credentials.sh
@@ -5,6 +5,49 @@ set -euo pipefail
CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
GFS_NS="${GFS_NS:-gfs}"
+set +u
+T2_PROJECT_DIR="$T2_PROJECT_DIR"
+T2_PROFILE="$T2_PROFILE"
+T2_CONTEXT="$T2_CONTEXT"
+MINIKUBE_PROFILE="$MINIKUBE_PROFILE"
+CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
+T2_LOCK_TOKEN="$T2_LOCK_TOKEN"
+set -u
+if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fi
+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi
+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$CONTEXT"; fi
+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"; fi
+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTEXT"; fi
+if [ -n "$T2_LOCK_TOKEN" ] || [ -n "$MINIKUBE_PROFILE" ] || [[ "$CONTEXT" == clerum-* ]]; then
+ # shellcheck source=scripts/minikube/t2-common.sh
+ source "$ROOT/scripts/minikube/t2-common.sh"
+ if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
+ cleanup_reconcile() {
+ local status=$?
+ t2_lock_release "$status"
+ }
+ trap cleanup_reconcile EXIT INT TERM
+ t2_repo_metadata
+ t2_profile_scope
+ t2_context_check
+ t2_mutation_lock
+ # The verified profile context is authoritative. Do not let a caller keep
+ # a second CONTEXT value after the fence has been acquired.
+ CONTEXT="$T2_CONTEXT"
+else
+ case "$CONTEXT" in
+ gke_*)
+ # GKE deploys are owned by the explicitly selected deploy workflow rather
+ # than the local Minikube lease. Keep this exception narrow: an arbitrary
+ # CONTEXT must never turn this credential mutator into a remote-cluster
+ # command without either the local fence or the deploy-context shape.
+ ;;
+ *)
+ printf '[reconcile-gfs-deploy] ERROR: refusing unverified Kubernetes context: %s\n' "$CONTEXT" >&2
+ exit 1
+ ;;
+ esac
+fi
CONTEXT="$CONTEXT" bash "$ROOT/deploy/scripts/apply-gfs-writer-secret.sh"
writer_dsn_b64="$(kubectl --context="$CONTEXT" -n "$GFS_NS" get secret gfs-controller-db \
diff --git a/scripts/e2e/minikube-real-postgres.sh b/scripts/e2e/minikube-real-postgres.sh
index 66faa8f91..4451df0d1 100755
--- a/scripts/e2e/minikube-real-postgres.sh
+++ b/scripts/e2e/minikube-real-postgres.sh
@@ -37,6 +37,7 @@ T1_PENDING_TESTS=0
# reconcile cluster credentials for a run that never reached the suites
# (e.g. a failed precondition, or this file being sourced by a harness test).
T1_GFS_RESTORE_REQUIRED=false
+T1_CLEANUP_DONE=false
die_t1() {
local code="$1"
@@ -73,11 +74,19 @@ restore_gfs_runtime_credentials() {
# rollout restart it, and let any wait reconcile still needs judge
# readiness via the gfs-rollout-shim instead of the template generation
# HCC's gfsReconciler keeps rewriting.
- if ! CONTEXT="$T2_CONTEXT" \
+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
+ CONTEXT="$T2_CONTEXT" \
bash "$PROJECT_DIR/scripts/minikube/settle-gfs-reader-rollout.sh"; then
printf '[minikube-t1] WARN: could not settle a leftover gfsc-reader rollout claim before restore\n' >&2
fi
if ! PATH="$PROJECT_DIR/scripts/minikube/gfs-rollout-shim:$PATH" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
CONTEXT="$T2_CONTEXT" \
bash "$PROJECT_DIR/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then
@@ -88,6 +97,11 @@ restore_gfs_runtime_credentials() {
cleanup_t1() {
local status=$?
+ if [ "$T1_CLEANUP_DONE" = true ]; then
+ return 0
+ fi
+ T1_CLEANUP_DONE=true
+ trap - EXIT INT TERM
if [ -n "$PORT_FORWARD_PID" ] && kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1; then
local command_line
command_line="$(ps -p "$PORT_FORWARD_PID" -o command= 2>/dev/null || true)"
@@ -101,6 +115,7 @@ cleanup_t1() {
status=1
fi
if [ -n "$T1_TMP_DIR" ] && [ -d "$T1_TMP_DIR" ]; then rm -rf "$T1_TMP_DIR"; fi
+ t2_lock_release "$status" || status=$?
exit "$status"
}
trap cleanup_t1 EXIT INT TERM
@@ -183,7 +198,7 @@ list_real_pg_files() {
is_isolated_control_api_file() {
local path="$1"
case "$path" in
- */db.realPostgresMigration.integration.test.ts|*/gfsReaderRole.realPostgres.integration.test.ts)
+ "$PROJECT_DIR"/control-api/test/*realPostgres*.test.ts)
return 0
;;
*)
@@ -193,20 +208,19 @@ is_isolated_control_api_file() {
}
require_isolated_control_api_files() {
- local relative path
- for relative in \
- test/db.realPostgresMigration.integration.test.ts \
- test/gfsReaderRole.realPostgres.integration.test.ts; do
- path="$PROJECT_DIR/control-api/$relative"
- if [ ! -f "$path" ]; then
- T1_NEXT_COMMAND='restore the isolated Real PostgreSQL suite files, then re-run T1'
- die_t1 ZERO_TESTS_EXECUTED "isolated suite file missing or renamed: $relative"
- fi
+ local path count=0
+ while IFS= read -r path; do
+ [ -n "$path" ] || continue
+ count=$((count + 1))
is_isolated_control_api_file "$path" || {
- T1_NEXT_COMMAND='keep the isolated Real PostgreSQL suite filenames in the T1 harness'
- die_t1 ZERO_TESTS_EXECUTED "isolated suite file is no longer routed away from control-postgres: $relative"
+ T1_NEXT_COMMAND='route every control-api Real PostgreSQL suite through the disposable PostgreSQL instance'
+ die_t1 ZERO_TESTS_EXECUTED "control-api Real PostgreSQL suite is not isolated: $path"
}
- done
+ done < <(list_real_pg_files control-api)
+ if [ "$count" -le 0 ]; then
+ T1_NEXT_COMMAND='restore the control-api Real PostgreSQL suite files, then re-run T1'
+ die_t1 ZERO_TESTS_EXECUTED 'no control-api Real PostgreSQL suites were found'
+ fi
}
start_isolated_postgres() {
@@ -254,8 +268,12 @@ run_suite() {
while IFS= read -r file; do
[ -z "$file" ] && continue
- if [ "$package" = control-api ] && is_isolated_control_api_file "$file"; then
- [ "$lane" = isolated ] || continue
+ if [ "$package" = control-api ]; then
+ if [ "$lane" = isolated ]; then
+ is_isolated_control_api_file "$file" || continue
+ else
+ is_isolated_control_api_file "$file" && continue
+ fi
else
[ "$lane" = shared ] || continue
fi
@@ -277,9 +295,10 @@ run_suite() {
json_file="$T1_TMP_DIR/$(printf '%s' "$package-$lane" | tr / _).json"
printf '[minikube-t1] running %s %s Real PostgreSQL suites (%s files)\n' \
"$package" "$lane" "$expected_files"
- # T1 verdict is the JSON reporter (expected files, executed/passed, zero
- # failures, zero pending). Capture npm's exit but do not treat a leftover
- # process exit after a complete green reporter as a failed suite.
+ # T1 requires both a complete green JSON reporter and a successful process
+ # exit. A reporter can be written before Vitest/npm fails during teardown,
+ # worker shutdown, OOM handling, or signal cleanup; accepting that partial
+ # observation would certify a broken lane.
suite_status=0
(
cd "$PROJECT_DIR/$package"
@@ -332,8 +351,10 @@ PY
die_t1 ZERO_TESTS_EXECUTED "Real PostgreSQL lane reported zero tests or skips in $package ($lane)"
fi
if [ "$suite_status" -ne 0 ]; then
- printf '[minikube-t1] vitest process exited %s after a complete green reporter for %s %s; T1 follows the reporter\n' \
- "$suite_status" "$package" "$lane"
+ cat "$log_file" >&2 || true
+ cat "$json_file" >&2 || true
+ T1_NEXT_COMMAND='repair the failed or incomplete Real PostgreSQL lane, then re-run T1'
+ die_t1 REAL_PG_SUITE_FAILED "Vitest process exited $suite_status for $package ($lane)"
fi
printf '[minikube-t1] PASS %s %s files=%s tests=%s skipped=0\n' \
"$package" "$lane" "$passed_files" "$passed_tests"
@@ -345,6 +366,7 @@ main() {
t2_repo_metadata
t2_profile_scope
t2_context_check
+ t2_mutation_lock
if [ -z "$T2_CONTEXT" ]; then
T1_NEXT_COMMAND='set CONTROL_API_REAL_PG_CONTEXT to the verified Kubernetes context'
die_t1 REAL_PG_REQUIRED_BUT_UNAVAILABLE 'local Real PostgreSQL requires an explicit Minikube context'
@@ -409,7 +431,6 @@ print("postgresql://" + quote(user.decode(), safe="") + ":" + quote(password.dec
start_isolated_postgres
T1_GFS_RESTORE_REQUIRED=true
run_suite control-api isolated "$ISOLATED_DSN"
- run_suite control-api shared "$ADMIN_DSN"
run_suite gfs-controller shared "$ADMIN_DSN"
unset PG_USER PG_PASSWORD PG_DATABASE T1_REDACT_PASSWORD
unset ADMIN_DSN ISOLATED_DSN
diff --git a/scripts/minikube/gfs-rollout-shim/kubectl b/scripts/minikube/gfs-rollout-shim/kubectl
index 133e6dd75..1b6749264 100755
--- a/scripts/minikube/gfs-rollout-shim/kubectl
+++ b/scripts/minikube/gfs-rollout-shim/kubectl
@@ -64,20 +64,32 @@ while [ "$i" -lt "${#args[@]}" ]; do
i=$((i + 1))
done
-timeout_seconds=600
-case "$timeout_raw" in
- '') ;;
- *m) timeout_seconds=$((${timeout_raw%m} * 60)) ;;
- *s) timeout_seconds="${timeout_raw%s}" ;;
- *[!0-9]*) ;;
- *) timeout_seconds="$timeout_raw" ;;
-esac
-# A CrashLoopBackOff retry delay reaches five minutes; a shorter caller
-# timeout cannot observe the credential-restore recovery this wait exists
-# for, so enforce a floor. Overridable only for the stubbed contract test.
-min_timeout="${GFS_ROLLOUT_SHIM_MIN_TIMEOUT_SECONDS:-600}"
-if [ "$timeout_seconds" -lt "$min_timeout" ]; then
- timeout_seconds="$min_timeout"
+if ! timeout_seconds="$(python3 - "$timeout_raw" "${GFS_ROLLOUT_SHIM_MIN_TIMEOUT_SECONDS:-600}" <<'PY'
+import re
+import sys
+
+timeout_raw, minimum_raw = sys.argv[1:]
+
+def parse_duration(raw, default, label):
+ if raw == "":
+ return default
+ match = re.fullmatch(r"[1-9][0-9]{0,4}([sm]?)", raw)
+ if not match:
+ raise SystemExit(f"invalid {label}: expected a positive integer with optional s/m suffix")
+ value = int(raw[:-1] if raw[-1:] in {"s", "m"} else raw)
+ if raw.endswith("m"):
+ value *= 60
+ if value < 1 or value > 86400:
+ raise SystemExit(f"invalid {label}: value must be between 1 and 86400 seconds")
+ return value
+
+timeout = parse_duration(timeout_raw, 600, "rollout timeout")
+minimum = parse_duration(minimum_raw, 600, "minimum rollout timeout")
+print(max(timeout, minimum))
+PY
+)"; then
+ printf '[gfs-rollout-shim] refusing an invalid rollout timeout\n' >&2
+ exit 2
fi
if [ -z "$context" ]; then
diff --git a/scripts/minikube/pf-all-stack.sh b/scripts/minikube/pf-all-stack.sh
index abcec6931..625acefba 100755
--- a/scripts/minikube/pf-all-stack.sh
+++ b/scripts/minikube/pf-all-stack.sh
@@ -7,8 +7,20 @@ PROFILE="${MINIKUBE_PROFILE:-clerum-test}"
KC=(kubectl --context="${PROFILE}")
HOLD=false
PIDS=()
+PIDFILES=()
+PIDFILE_SERVICES=()
SAFE_PROFILE="${PROFILE//[^A-Za-z0-9_.-]/_}"
HAS_PROFILE_OWNED_PORTS=false
+if [[ ! "$PROFILE" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then
+ echo "ERROR: invalid Minikube profile identifier: $PROFILE" >&2
+ exit 1
+fi
+set +u
+PROFILE_PID_ROOT="$CLERUM_PROFILE_CACHE_ROOT"
+set -u
+if [[ -z "$PROFILE_PID_ROOT" ]]; then
+ PROFILE_PID_ROOT="$HOME/.cache/clerum/minikube-profiles"
+fi
load_branch_profile_ports_env() {
local file="$1"
@@ -110,11 +122,37 @@ cleanup() {
if [[ "${HOLD}" != "true" ]]; then
return 0
fi
- for pid in "${PIDS[@]}"; do
- kill "${pid}" 2>/dev/null || true
+ local index
+ for index in "${!PIDFILES[@]}"; do
+ kill_owned_pidfile "${PIDFILES[$index]}" "${PIDFILE_SERVICES[$index]}" || true
done
}
+kill_owned_pidfile() {
+ local pidfile="$1"
+ local service="$2"
+ local pid command_line
+ [[ -f "$pidfile" ]] || return 0
+ pid="$(sed -n '1p' "$pidfile" 2>/dev/null || true)"
+ if [[ ! "$pid" =~ ^[1-9][0-9]*$ ]]; then
+ rm -f -- "$pidfile"
+ return 0
+ fi
+ if ! kill -0 "$pid" 2>/dev/null; then
+ rm -f -- "$pidfile"
+ return 0
+ fi
+ command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)"
+ if [[ "$command_line" != *port-forward* ||
+ "$command_line" != *"svc/$service"* ||
+ "$command_line" != *"$PROFILE"* ]]; then
+ echo "ERROR: refusing to kill PID $pid from $pidfile; it is not the $PROFILE $service port-forward" >&2
+ return 1
+ fi
+ kill "$pid" 2>/dev/null || true
+ rm -f -- "$pidfile"
+}
+
start_pf() {
local name="$1"
local namespace="$2"
@@ -123,23 +161,19 @@ start_pf() {
local health_url="${5:-}"
local log="/tmp/pf-${SAFE_PROFILE}-${name}.log"
local pidfile="/tmp/pf-${SAFE_PROFILE}-${name}.pid"
- local profile_pids_dir="${CLERUM_PROFILE_CACHE_ROOT:-${HOME}/.cache/clerum/minikube-profiles}/${PROFILE}/pids"
+ local profile_pids_dir="$PROFILE_PID_ROOT/$SAFE_PROFILE/pids"
local profile_pidfile="${profile_pids_dir}/${name}.pid"
- mkdir -p "${profile_pids_dir}"
- if [[ -f "${pidfile}" ]]; then
- kill "$(cat "${pidfile}")" 2>/dev/null || true
- rm -f "${pidfile}"
- fi
- if [[ -f "${profile_pidfile}" ]]; then
- kill "$(cat "${profile_pidfile}")" 2>/dev/null || true
- rm -f "${profile_pidfile}"
- fi
+ mkdir -p "$profile_pids_dir"
+ kill_owned_pidfile "$pidfile" "$service"
+ kill_owned_pidfile "$profile_pidfile" "$service"
nohup "${KC[@]}" -n "${namespace}" port-forward --address=127.0.0.1 "svc/${service}" "${ports}" >"${log}" 2>&1 </dev/null &
echo $! >"${pidfile}"
echo $! >"${profile_pidfile}"
PIDS+=("$!")
+ PIDFILES+=("$pidfile" "$profile_pidfile")
+ PIDFILE_SERVICES+=("$service" "$service")
echo " ${name}: pid=$(cat "${pidfile}") ns=${namespace} svc=${service} ports=${ports}"
sleep 0.2
if ! kill -0 "$!" 2>/dev/null; then
diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.sh
index 213956ebf..1b4a45b07 100755
--- a/scripts/minikube/pre-gate-sync.sh
+++ b/scripts/minikube/pre-gate-sync.sh
@@ -2,6 +2,10 @@
# Enforces the "sync to minikube before every gate" rule from the platform E2E plan.
set -euo pipefail
+set +u
+PRE_GATE_SYNC_CONFIG_ONLY="$PRE_GATE_SYNC_CONFIG_ONLY"
+set -u
+if [ -z "$PRE_GATE_SYNC_CONFIG_ONLY" ]; then PRE_GATE_SYNC_CONFIG_ONLY=false; fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
@@ -43,6 +47,23 @@ while [[ $# -gt 0 ]]; do
esac
done
+# The pre-gate entrypoint is also a mutating recovery boundary. It must carry
+# the same branch/profile identity as the T2 orchestrator, or acquire that
+# identity itself when invoked standalone.
+T2_PROJECT_DIR="$PROJECT_DIR"
+T2_PROFILE="$PROFILE"
+T2_CONTEXT="$PROFILE"
+T2_GATE_ID="$GATE_NAME"
+# shellcheck source=scripts/minikube/t2-common.sh
+source "$SCRIPT_DIR/t2-common.sh"
+if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
+if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
+ t2_require_commands
+ t2_repo_metadata
+ t2_profile_scope
+ t2_context_check
+fi
+
log() {
printf '[pre-gate-sync] %s\n' "$*"
}
@@ -154,13 +175,19 @@ restore_control_api() {
}
restore_pre_gate_writers() {
+ local status=$?
# Restore Control API first so it can serve the final readiness checks; WRC
# remains last because it is the consumer that must stay stopped until every
# migration, writer restart, and legacy-policy gate has completed.
- restore_control_api
- restore_workflow_reconciler
+ restore_control_api || status=1
+ restore_workflow_reconciler || status=1
+ t2_lock_release "$status" || status=$?
+ return "$status"
}
+if [ "$PRE_GATE_SYNC_CONFIG_ONLY" != true ]; then
+ t2_mutation_lock
+fi
trap restore_pre_gate_writers EXIT
fingerprint_dir() {
@@ -375,11 +402,19 @@ provision_gfs_serving() {
# still needs a reader rollout, the gfs-rollout-shim PATH prefix makes
# that wait judge readiness instead of the template generation HCC keeps
# rewriting.
- CONTEXT="${PROFILE}" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
+ CONTEXT="$T2_CONTEXT" \
bash "${PROJECT_DIR}/scripts/minikube/settle-gfs-reader-rollout.sh"
if ! PATH="${PROJECT_DIR}/scripts/minikube/gfs-rollout-shim:${PATH}" \
GFS_RESTORE_ACTIVE_NOLOGIN=true GFS_RECOVER_ABANDONED_STATE=true \
- CONTEXT="${PROFILE}" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
+ CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
+ T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
+ CONTEXT="$T2_CONTEXT" \
bash "${PROJECT_DIR}/deploy/scripts/reconcile-gfs-deploy-credentials.sh"; then
log "ERROR: gfs DB provisioning FAILED — gfsc cannot authorize any operation. Aborting ${GATE_NAME} pre-gate sync."
exit 1
diff --git a/scripts/minikube/settle-gfs-reader-rollout.sh b/scripts/minikube/settle-gfs-reader-rollout.sh
index bcdd71141..bb6680103 100755
--- a/scripts/minikube/settle-gfs-reader-rollout.sh
+++ b/scripts/minikube/settle-gfs-reader-rollout.sh
@@ -13,14 +13,50 @@
# times out; not restarting a Ready reader is the only safe path.
set -euo pipefail
-CONTEXT="${CONTEXT:?set CONTEXT to the target kube-context}"
+ROOT="$(cd "$(dirname "$0")/../.." && pwd -P)"
+set +u
+T2_PROJECT_DIR="$T2_PROJECT_DIR"
+T2_PROFILE="$T2_PROFILE"
+T2_CONTEXT="$T2_CONTEXT"
+MINIKUBE_PROFILE="$MINIKUBE_PROFILE"
+CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
+CONTEXT="$CONTEXT"
+set -u
+if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$ROOT"; fi
+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi
+if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$CONTEXT"; fi
+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"; fi
+if [ -z "$T2_CONTEXT" ]; then T2_CONTEXT="$CONTEXT"; fi
+# shellcheck source=scripts/minikube/t2-common.sh
+source "$ROOT/scripts/minikube/t2-common.sh"
+if [ -z "$T2_SKIP_LOCK" ]; then T2_SKIP_LOCK=false; fi
+T2_CONTEXT="$T2_CONTEXT"
+CONTEXT="$T2_CONTEXT"
GFS_NS="${GFS_NS:-gfs}"
DEPLOY="${GFS_READER_DEPLOYMENT:-gfsc-reader}"
SECRET="${GFS_READER_DB_SECRET:-gfs-controller-reader-db}"
SELECTOR="${GFS_READER_SELECTOR:-app=gfs-controller,clerum.io/gfsc-role=reader}"
-kc() { kubectl --context="$CONTEXT" "$@"; }
log() { printf '[settle-gfs-reader-rollout] %s\n' "$*" >&2; }
+die() { log "ERROR: $*"; exit 1; }
+kc() { kubectl --context="$T2_CONTEXT" "$@"; }
+
+cleanup_settle() {
+ local status=$?
+ t2_lock_release "$status"
+}
+trap cleanup_settle EXIT INT TERM
+t2_repo_metadata
+t2_profile_scope
+t2_context_check
+t2_mutation_lock
+
+# The proof library is sourced only after the profile fence is established. It
+# reads the committed DSN in memory and never prints it.
+# shellcheck source=deploy/scripts/lib/gfs-credential-secret.sh
+source "$ROOT/deploy/scripts/lib/gfs-credential-secret.sh"
+# shellcheck source=deploy/scripts/lib/gfs-credential-rollout.sh
+source "$ROOT/deploy/scripts/lib/gfs-credential-rollout.sh"
if ! kc -n "$GFS_NS" get deployment "$DEPLOY" >/dev/null 2>&1; then
exit 0
@@ -77,11 +113,5 @@ if [ -z "$rotated" ]; then
fi
log "gfsc-reader is Ready with an interrupted rollout claim; marking the reader Secret ready without restart"
-python3 -c 'import json, sys
-state, stamp = sys.argv[1:]
-print(json.dumps([
- {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": state},
- {"op": "test", "path": "/metadata/annotations/clerum.io~1gfs-dsn-rotated-at", "value": stamp},
- {"op": "replace", "path": "/metadata/annotations/clerum.io~1gfs-dsn-state", "value": "ready"},
-]))' "$state" "$rotated" \
- | kc -n "$GFS_NS" patch secret "$SECRET" --type=json --patch-file=/dev/stdin >/dev/null
+credential_rollout_proof "$SECRET" "$DEPLOY" "$rotated"
+mark_secret_rollout_ready "$SECRET" "$rotated" rollout-running
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 5ab1a12be..bcce90fa8 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -63,6 +63,16 @@ T2_LOCK_KEY=""
T2_EVIDENCE_DIR=""
T2_EVIDENCE_FILE=""
T2_LOCK_HELD=false
+T2_LOCK_TOKEN="$T2_LOCK_TOKEN"
+T2_RUN_ID="$T2_RUN_ID"
+T2_GATE_ID="$T2_GATE_ID"
+if [ -z "$T2_GATE_ID" ]; then T2_GATE_ID=minikube-t2; fi
+T2_CERTIFICATION_VERSION=1
+T2_T0_CERTIFIED=false
+T2_T1_CERTIFIED=false
+T2_PRIOR_ATTESTATION=""
+T2_PRIOR_T0_ATTESTATION=""
+T2_PRIOR_T1_ATTESTATION=""
T2_REQUIRED_NAMESPACES="$T2_REQUIRED_NAMESPACES"
if [ -z "$T2_REQUIRED_NAMESPACES" ]; then T2_REQUIRED_NAMESPACES="control-plane gfs mcp-host mcp-server profiles rpc-proxy channels sandbox-recipes sandbox-ui webhook-ingress registry"; fi
@@ -220,7 +230,9 @@ t2_profile_scope() {
}
t2_context_check() {
- if ! kubectl --context="$T2_CONTEXT" config current-context >/dev/null 2>&1; then
+ local context_name
+ context_name="$(kubectl config get-contexts -o name 2>/dev/null | awk -v target="$T2_CONTEXT" '$0 == target {print; exit}')"
+ if [ "$context_name" != "$T2_CONTEXT" ]; then
T2_NEXT_COMMAND='select the explicit Kubernetes context generated with the profile'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context is unavailable: $T2_CONTEXT"
fi
@@ -593,60 +605,159 @@ t2_classify_transition() {
fi
}
+t2_lock_owner_value() {
+ local key="$1"
+ [ -f "$T2_LOCK_DIR/owner.env" ] || return 1
+ sed -n "s/^$key=//p" "$T2_LOCK_DIR/owner.env" | head -1
+}
+
+t2_lock_key() {
+ printf '%s\0%s\0%s\0%s\0%s' \
+ "$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_HEAD" "$T2_PROFILE" "$T2_CONTEXT" |
+ shasum | awk '{print $1}'
+}
+
+t2_lock_profile_id_check() {
+ if [[ ! "$T2_PROFILE" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
+ T2_NEXT_COMMAND='use the generated profile name without path separators or shell metacharacters'
+ t2_fail DEVELOPMENT_SCOPE_REQUIRED 'profile lock identifier is not a safe local profile name'
+ fi
+}
+
+t2_lock_process_matches() {
+ local pid="$1" expected_start actual_start state
+ [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1
+ kill -0 "$pid" >/dev/null 2>&1 || return 1
+ state="$(ps -p "$pid" -o state= 2>/dev/null | tr -d '[:space:]' || true)"
+ [ -z "$state" ] || [[ "$state" != Z* ]] || return 1
+ expected_start="$(t2_lock_owner_value PROCESS_START || true)"
+ [ -n "$expected_start" ] || return 1
+ # Some macOS/sandboxed ps implementations hide lstart/state. The opaque
+ # token and live PID still fence the child in that environment; use the
+ # stronger start-time comparison whenever the platform exposes it.
+ [ "$expected_start" = unavailable ] && return 0
+ actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [ -z "$actual_start" ] || [ "$actual_start" = "$expected_start" ]
+}
+
t2_lock_acquire() {
+ local process_start
+ t2_lock_profile_id_check
mkdir -p "$T2_LOCK_ROOT"
T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
- T2_LOCK_KEY="$(printf '%s\0%s\0%s\0%s' "$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_HEAD" "$T2_PROFILE" | shasum | awk '{print $1}')"
- if ! mkdir "$T2_LOCK_DIR" 2>/dev/null; then
- local existing_pid
- existing_pid="$(sed -n 's/^PID=//p' "$T2_LOCK_DIR/owner.env" 2>/dev/null | head -1 || true)"
- if [[ ! "$existing_pid" =~ ^[0-9]+$ ]]; then
+ T2_LOCK_KEY="$(t2_lock_key)"
+ if [ -e "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR" ]; then
+ if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ]; then
+ T2_NEXT_COMMAND='remove the unsafe profile-lock path manually after verifying the profile owner'
+ t2_fail PROFILE_BUSY "profile lock path is not a directory: $T2_LOCK_DIR"
+ fi
+ local existing_pid existing_token existing_start
+ existing_pid="$(t2_lock_owner_value PID || true)"
+ existing_token="$(t2_lock_owner_value TOKEN || true)"
+ existing_start="$(t2_lock_owner_value PROCESS_START || true)"
+ if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; then
T2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
- t2_fail PROFILE_BUSY "profile $T2_PROFILE has a lock without a valid owner PID"
+ t2_fail PROFILE_BUSY "profile $T2_PROFILE has an orphaned lock without a valid owner identity"
fi
- if [ -n "$existing_pid" ] && kill -0 "$existing_pid" >/dev/null 2>&1; then
+ if t2_lock_process_matches "$existing_pid"; then
T2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"
t2_fail PROFILE_BUSY "profile $T2_PROFILE is locked by PID $existing_pid"
fi
- rm -rf "$T2_LOCK_DIR"
- mkdir "$T2_LOCK_DIR" 2>/dev/null || {
- T2_NEXT_COMMAND='retry after the profile lock is released'
- t2_fail PROFILE_BUSY "unable to reclaim the profile lock: $T2_PROFILE"
- }
+ rm -rf -- "$T2_LOCK_DIR"
fi
+ mkdir "$T2_LOCK_DIR" 2>/dev/null || {
+ T2_NEXT_COMMAND='retry after the profile lock is released'
+ t2_fail PROFILE_BUSY "unable to acquire the profile lock: $T2_PROFILE"
+ }
umask 077
+ T2_LOCK_TOKEN="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"
+ process_start="$(ps -p $$ -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
+ [ -n "$process_start" ] || process_start=unavailable
cat >"$T2_LOCK_DIR/owner.env" <<EOF
REPOSITORY=$T2_PROJECT_DIR
BRANCH=$T2_BRANCH
HEAD=$T2_HEAD
PROFILE=$T2_PROFILE
CONTEXT=$T2_CONTEXT
+WORKTREE_ID=$T2_WORKTREE_ID
LOCK_KEY=$T2_LOCK_KEY
+TOKEN=$T2_LOCK_TOKEN
PID=$$
+PROCESS_START=$process_start
STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
EOF
T2_LOCK_HELD=true
- trap t2_lock_release EXIT INT TERM
+}
+
+t2_lock_validate_inherited() {
+ local expected_pid expected_token owner_key owner_value
+ t2_lock_profile_id_check
+ T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
+ T2_LOCK_KEY="$(t2_lock_key)"
+ if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR/owner.env" ]; then
+ T2_NEXT_COMMAND='run the parent T2 command with its profile lock token, then retry this child operation'
+ t2_fail PROFILE_LOCK_REQUIRED "inherited profile lock is missing for $T2_PROFILE"
+ fi
+ expected_token="$T2_LOCK_TOKEN"
+ owner_value="$(t2_lock_owner_value TOKEN || true)"
+ if [ -z "$expected_token" ] || [ "$owner_value" != "$expected_token" ]; then
+ T2_NEXT_COMMAND='pass the opaque T2_LOCK_TOKEN from the owning T2 process; do not bypass the profile lock'
+ t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock token is missing or does not match'
+ fi
+ for owner_key in REPOSITORY BRANCH HEAD PROFILE CONTEXT WORKTREE_ID LOCK_KEY; do
+ case "$owner_key" in
+ REPOSITORY) expected_pid="$T2_PROJECT_DIR" ;;
+ BRANCH) expected_pid="$T2_BRANCH" ;;
+ HEAD) expected_pid="$T2_HEAD" ;;
+ PROFILE) expected_pid="$T2_PROFILE" ;;
+ CONTEXT) expected_pid="$T2_CONTEXT" ;;
+ WORKTREE_ID) expected_pid="$T2_WORKTREE_ID" ;;
+ LOCK_KEY) expected_pid="$T2_LOCK_KEY" ;;
+ esac
+ if [ "$(t2_lock_owner_value "$owner_key" || true)" != "$expected_pid" ]; then
+ T2_NEXT_COMMAND='re-run the operation from the worktree/profile that owns the T2 lock'
+ t2_fail PROFILE_OWNERSHIP_MISMATCH "profile lock owner does not match $owner_key"
+ fi
+ done
+ expected_pid="$(t2_lock_owner_value PID || true)"
+ if ! t2_lock_process_matches "$expected_pid"; then
+ T2_NEXT_COMMAND='restart the parent T2 operation so it can acquire a live profile lock'
+ t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'
+ fi
+ T2_LOCK_HELD=false
+}
+
+t2_mutation_lock() {
+ if [ "$T2_SKIP_LOCK" = true ]; then
+ t2_lock_validate_inherited
+ else
+ t2_lock_acquire
+ fi
}
t2_lock_release() {
- local status=$?
+ local incoming_status=$? status owner_pid owner_token
+ if [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fi
trap - EXIT INT TERM
if [ "$T2_LOCK_HELD" = true ] && [ -n "$T2_LOCK_DIR" ]; then
- local owner_pid
- owner_pid="$(sed -n 's/^PID=//p' "$T2_LOCK_DIR/owner.env" 2>/dev/null | head -1 || true)"
- if [ "$owner_pid" = "$$" ]; then rm -rf "$T2_LOCK_DIR"; fi
+ owner_pid="$(t2_lock_owner_value PID || true)"
+ owner_token="$(t2_lock_owner_value TOKEN || true)"
+ if [ "$owner_pid" = "$$" ] && [ -n "$T2_LOCK_TOKEN" ] && [ "$owner_token" = "$T2_LOCK_TOKEN" ]; then
+ rm -rf -- "$T2_LOCK_DIR"
+ fi
fi
+ T2_LOCK_HELD=false
if [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; then
t2_evidence_write failure FAIL "$T2_ERROR_CODE"
fi
- exit "$status"
+ return "$status"
}
t2_evidence_init() {
local stamp
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
- T2_EVIDENCE_DIR="$T2_EVIDENCE_ROOT/$stamp-$(printf '%s' "$T2_HEAD" | cut -c1-12)"
+ T2_RUN_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"
+ T2_EVIDENCE_DIR="$T2_EVIDENCE_ROOT/$stamp-$(printf '%s' "$T2_HEAD" | cut -c1-12)-$T2_RUN_ID"
T2_EVIDENCE_FILE="$T2_EVIDENCE_DIR/evidence.json"
umask 077
mkdir -p "$T2_EVIDENCE_DIR/logs"
@@ -660,12 +771,14 @@ t2_evidence_write() {
PHASE="$phase" STATUS="$status" DETAIL="$detail" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" T2_BRANCH="$T2_BRANCH" T2_HEAD="$T2_HEAD" \
T2_ORIGIN_DEV="$T2_ORIGIN_DEV" T2_MERGE_BASE="$T2_MERGE_BASE" \
+ T2_WORKTREE_ID="$T2_WORKTREE_ID" T2_RUN_ID="$T2_RUN_ID" T2_GATE_ID="$T2_GATE_ID" \
T2_PROFILE="$T2_PROFILE" T2_CONTEXT="$T2_CONTEXT" T2_CLUSTER_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \
T2_EVIDENCE_DIR="$T2_EVIDENCE_DIR" \
T2_IMAGE_MANIFEST="$T2_IMAGE_MANIFEST" T2_IMAGE_SOURCE="$T2_IMAGE_SOURCE" T2_IMAGE_TAG="$T2_IMAGE_TAG" \
python3 - "$file" "$T2_EVIDENCE_FILE" <<'PY'
import json
import os
+from datetime import datetime, timedelta, timezone
from pathlib import Path
path = Path(os.sys.argv[1])
prior_path = Path(os.sys.argv[2])
@@ -674,6 +787,14 @@ if prior_path.exists():
try: prior = json.loads(prior_path.read_text())
except ValueError: prior = {}
prior.setdefault("evidenceVersion", 1)
+prior["certificationVersion"] = 1
+prior.setdefault("runId", os.environ.get("T2_RUN_ID", ""))
+now = datetime.now(timezone.utc)
+prior.setdefault("attestationStartedAt", now.isoformat().replace("+00:00", "Z"))
+prior.setdefault(
+ "attestationExpiresAt",
+ (now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),
+)
def redact(value):
import re
value = str(value or "")
@@ -689,6 +810,9 @@ prior.update({
"head": os.environ.get("T2_HEAD", ""),
"originDev": os.environ.get("T2_ORIGIN_DEV", ""),
"mergeBase": os.environ.get("T2_MERGE_BASE", ""),
+ "worktreeId": os.environ.get("T2_WORKTREE_ID", ""),
+ "runId": os.environ.get("T2_RUN_ID", ""),
+ "gateId": os.environ.get("T2_GATE_ID", ""),
"worktree": os.environ.get("T2_PROJECT_DIR", ""),
"profile": os.environ.get("T2_PROFILE", ""),
"context": os.environ.get("T2_CONTEXT", ""),
@@ -701,6 +825,14 @@ prior.update({
"status": os.environ.get("STATUS", ""),
"detail": redact(os.environ.get("DETAIL", "")),
})
+phase = os.environ.get("PHASE", "")
+status = os.environ.get("STATUS", "")
+attestation = prior.get("attestationStatus", "IN_PROGRESS")
+if status in {"FAIL", "INVALIDATED"}:
+ attestation = "INVALIDATED"
+elif phase == "complete" and status == "PASS":
+ attestation = "PASS"
+prior["attestationStatus"] = attestation
prior.setdefault("phases", []).append({
"name": os.environ.get("PHASE", ""),
"status": os.environ.get("STATUS", ""),
@@ -711,10 +843,83 @@ PY
mv -- "$file" "$T2_EVIDENCE_FILE"
}
+t2_certification_validate_prior_lanes() {
+ local result
+ if [ -z "$T2_CLUSTER_FINGERPRINT" ]; then
+ T2_NEXT_COMMAND='run the exact-head preflight and regenerate a profile marker with a cluster fingerprint'
+ t2_fail CERTIFICATION_REQUIRED 'current exact-head marker has no cluster fingerprint'
+ fi
+ if ! result="$(
+ CERTIFICATION_ROOT="$T2_EVIDENCE_ROOT" \
+ EXPECTED_REPOSITORY="$T2_PROJECT_DIR" EXPECTED_BRANCH="$T2_BRANCH" \
+ EXPECTED_HEAD="$T2_HEAD" EXPECTED_ORIGIN_DEV="$T2_ORIGIN_DEV" \
+ EXPECTED_WORKTREE_ID="$T2_WORKTREE_ID" EXPECTED_PROFILE="$T2_PROFILE" \
+ EXPECTED_CONTEXT="$T2_CONTEXT" EXPECTED_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \
+ EXPECTED_GATE_ID="$T2_GATE_ID" python3 - <<'PY'
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+root = Path(os.environ["CERTIFICATION_ROOT"])
+expected = {
+ "repository": os.environ["EXPECTED_REPOSITORY"],
+ "branch": os.environ["EXPECTED_BRANCH"],
+ "head": os.environ["EXPECTED_HEAD"],
+ "originDev": os.environ["EXPECTED_ORIGIN_DEV"],
+ "worktreeId": os.environ["EXPECTED_WORKTREE_ID"],
+ "profile": os.environ["EXPECTED_PROFILE"],
+ "context": os.environ["EXPECTED_CONTEXT"],
+ "clusterFingerprintRef": os.environ["EXPECTED_FINGERPRINT"],
+ "gateId": os.environ["EXPECTED_GATE_ID"],
+}
+for candidate in sorted(root.glob("*/evidence.json"), reverse=True):
+ try:
+ data = json.loads(candidate.read_text())
+ except (OSError, ValueError):
+ continue
+ if data.get("certificationVersion") != 1:
+ continue
+ if data.get("attestationStatus") != "PASS":
+ continue
+ if not data.get("runId"):
+ continue
+ try:
+ expiry = datetime.fromisoformat(str(data.get("attestationExpiresAt", "")).replace("Z", "+00:00"))
+ if expiry <= datetime.now(timezone.utc):
+ continue
+ except (TypeError, ValueError):
+ continue
+ if any(data.get(key, "") != value for key, value in expected.items()):
+ continue
+ latest = {}
+ for phase in data.get("phases", []):
+ name = phase.get("name")
+ if name in {"T0", "T0_ATTESTED"}:
+ latest["T0"] = phase.get("status")
+ if name in {"T1", "T1_ATTESTED"}:
+ latest["T1"] = phase.get("status")
+ if latest.get("T0") == "PASS" and latest.get("T1") == "PASS":
+ print("\t".join([str(candidate), "PASS", "PASS"]))
+ raise SystemExit(0)
+raise SystemExit("no valid exact-head T0/T1 attestation was found")
+PY
+ )"; then
+ T2_NEXT_COMMAND='run make minikube-t2 with T0=true and T1=true to create exact-head lane attestations'
+ t2_fail CERTIFICATION_REQUIRED "$result"
+ fi
+ IFS=$'\t' read -r T2_PRIOR_ATTESTATION T2_PRIOR_T0_ATTESTATION T2_PRIOR_T1_ATTESTATION <<< "$result"
+ T2_T0_CERTIFIED=true
+ T2_T1_CERTIFIED=true
+ t2_evidence_write T0_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"
+ t2_evidence_write T1_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"
+}
+
t2_write_plan() {
local plan_file="$1"
[ -n "$plan_file" ] || return 0
- PLAN_STATE="$T2_PLAN_STATE" PLAN_REASON="$T2_PLAN_REASON" python3 - "$plan_file" <<'PY'
+ local tmp_file="$plan_file.tmp.$$"
+ PLAN_STATE="$T2_PLAN_STATE" PLAN_REASON="$T2_PLAN_REASON" python3 - "$tmp_file" <<'PY'
import json
import os
from pathlib import Path
@@ -723,4 +928,5 @@ Path(os.sys.argv[1]).write_text(json.dumps({
"reason": os.environ.get("PLAN_REASON", ""),
}, sort_keys=True) + "\n")
PY
+ mv -- "$tmp_file" "$plan_file"
}
diff --git a/scripts/minikube/t2-preflight.sh b/scripts/minikube/t2-preflight.sh
index 18ee423b0..4a90e54bf 100755
--- a/scripts/minikube/t2-preflight.sh
+++ b/scripts/minikube/t2-preflight.sh
@@ -19,14 +19,18 @@ T2_PLAN_FILE="$T2_PLAN_FILE"
if [ -z "$T2_PLAN_FILE" ]; then T2_PLAN_FILE="$T2_EVIDENCE_ROOT/next-plan.json"; fi
T2_NEXT_COMMAND='re-run the canonical command with the verified profile'
+cleanup_preflight() {
+ local status=$?
+ t2_lock_release "$status"
+}
+trap cleanup_preflight EXIT
+
main() {
t2_require_commands
t2_repo_metadata
t2_profile_scope
t2_context_check
- if [ "$T2_SKIP_LOCK" != true ]; then
- t2_lock_acquire
- fi
+ t2_mutation_lock
t2_evidence_init
# Profile status is read before any resource check. A missing/stopped profile
diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.sh
index 7ff2832dd..b4af7c8b2 100755
--- a/scripts/minikube/t2.sh
+++ b/scripts/minikube/t2.sh
@@ -30,23 +30,26 @@ T2_TMP_ROOT="$T2_TMP_ROOT"
if [ -z "$T2_TMP_ROOT" ]; then T2_TMP_ROOT=/tmp; fi
T2_T1_OUTPUT="$T2_TMP_ROOT/evenfire-t1.$$.out"
T2_FINAL_PREFLIGHT_OUTPUT="$T2_TMP_ROOT/evenfire-t2-final-preflight.$$.out"
+export T2_PROJECT_DIR T2_PROFILE_ROOT T2_PROFILE_ENV T2_PORTS_ENV T2_CONTEXT T2_LOCK_TOKEN
cleanup_plan() {
+ local status=$?
if [ -n "$T2_PLAN_TMP" ] && [ -f "$T2_PLAN_TMP" ]; then rm -f "$T2_PLAN_TMP"; fi
if [ -f "$T2_FINAL_PREFLIGHT_OUTPUT" ]; then rm -f "$T2_FINAL_PREFLIGHT_OUTPUT"; fi
if [ -f "$T2_T1_OUTPUT" ]; then rm -f "$T2_T1_OUTPUT"; fi
+ t2_lock_release "$status"
}
trap cleanup_plan EXIT
t2_lane_completed() {
local status="$1"
- [ "$status" = PASS ] || [ "$status" = SKIPPED ]
+ [ "$status" = PASS ]
}
run_t0() {
if [ "$T2_RUN_T0" != true ]; then
T2_T0_STATUS=SKIPPED
- t2_evidence_write T0 SKIPPED 'T2_RUN_T0=false; static lane already certified on this HEAD'
+ t2_evidence_write T0 SKIPPED 'T2_RUN_T0=false; an exact-head T0 attestation is required before runtime-only certification'
return 0
fi
printf '[minikube-t2] T0: shell syntax and contract checks\n'
@@ -63,7 +66,7 @@ run_t0() {
run_preflight_plan() {
T2_PLAN_TMP="$(mktemp "$T2_TMP_ROOT/evenfire-t2-plan.XXXXXX")"
- if ! T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE="$T2_PLAN_TMP" \
+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE="$T2_PLAN_TMP" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
@@ -115,20 +118,24 @@ run_bootstrap_or_reconcile() {
full-bootstrap)
printf '[minikube-t2] bootstrap: full setup, then PostgreSQL/migrations/roles checks\n'
if [ "$T2_RESET_PVC" = true ]; then
- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
- REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setup
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
+ REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setup
else
- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
REUSE_DB=true make minikube-setup
fi
;;
full-reconcile)
printf '[minikube-t2] full reconcile: infrastructure input changed\n'
if [ "$T2_RESET_PVC" = true ]; then
- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
REUSE_DB=false CONTROL_DB_RESET_PVC_UID="$T2_EXPECTED_PVC_UID" make minikube-setup
else
- MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ MINIKUBE_PROFILE="$T2_PROFILE" IMAGE_SOURCE="$T2_BOOTSTRAP_IMAGE_SOURCE" \
REUSE_DB=true make minikube-setup
fi
;;
@@ -187,17 +194,19 @@ run_pre_gate() {
return 0
fi
printf '[minikube-t2] pre-gate-sync after bootstrap/reconcile\n'
- MINIKUBE_PROFILE="$T2_PROFILE" make minikube-pre-gate-sync GATE=minikube-t2 ARGS='--skip-port-forwards'
+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \
+ MINIKUBE_PROFILE="$T2_PROFILE" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" \
+ make minikube-pre-gate-sync GATE=minikube-t2 ARGS='--skip-port-forwards'
}
run_t1() {
if [ "$T2_RUN_T1" != true ]; then
T2_T1_STATUS=SKIPPED
- t2_evidence_write T1 SKIPPED 'T2_RUN_T1=false; Real PostgreSQL lane already certified on this HEAD'
+ t2_evidence_write T1 SKIPPED 'T2_RUN_T1=false; an exact-head T1 attestation is required before runtime-only certification'
return 0
fi
printf '[minikube-t2] T1: Real PostgreSQL suites\n'
- if T2_SKIP_LOCK=true CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \
+ if T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" MINIKUBE_PROFILE="$T2_PROFILE" \
bash "$T2_PROJECT_DIR/scripts/e2e/minikube-real-postgres.sh" >"$T2_T1_OUTPUT" 2>&1; then
cat "$T2_T1_OUTPUT"
T2_T1_STATUS=PASS
@@ -214,7 +223,7 @@ run_t1() {
run_final_preflight() {
printf '[minikube-t2] final exact-head readiness preflight\n'
# Fail-loud: a stale marker or missing bootstrap must not count as T2.
- if ! T2_SKIP_LOCK=true T2_PLAN_MODE=false T2_PLAN_FILE="$T2_PLAN_TMP" \
+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE="$T2_PLAN_TMP" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" MINIKUBE_PROFILE="$T2_PROFILE" \
CONTROL_API_REAL_PG_CONTEXT="$T2_CONTEXT" T2_PROFILE_ROOT="$T2_PROFILE_ROOT" \
T2_PROFILE_ENV="$T2_PROFILE_ENV" T2_PORTS_ENV="$T2_PORTS_ENV" \
@@ -234,6 +243,7 @@ PY
T2_NEXT_COMMAND='re-run make minikube-t2 so pre-gate-sync updates the marker to this HEAD'
t2_fail HEAD_MARKER_MISMATCH "final T2 preflight selected $T2_PLAN_STATE instead of already-synced"
fi
+ t2_marker_check
T2_T2_STATUS=PASS
t2_evidence_write T2 PASS 'exact marker, image, PostgreSQL, namespace, Service, and deployment readiness passed'
}
@@ -277,11 +287,11 @@ main() {
t2_repo_metadata
t2_profile_scope
t2_context_check
- if [ "$T2_SKIP_LOCK" != true ]; then t2_lock_acquire; fi
+ t2_mutation_lock
t2_evidence_init
run_t0
- if ! t2_lane_completed "$T2_T0_STATUS"; then
+ if [ "$T2_RUN_T0" = true ] && ! t2_lane_completed "$T2_T0_STATUS"; then
T2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T0=true so the static lane is executed'
t2_fail ZERO_TESTS_EXECUTED 'T0 was not executed'
fi
@@ -290,13 +300,21 @@ main() {
T2_NEXT_COMMAND='run make minikube-t2 so bootstrap/reconcile and T0/T1 execute on this HEAD'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2-only mode requires an already-synced exact-head profile'
fi
+ if [ "$T2_RUN_T0" != true ] || [ "$T2_RUN_T1" != true ]; then
+ t2_marker_check
+ t2_certification_validate_prior_lanes
+ fi
run_bootstrap_or_reconcile
run_pre_gate
run_t1
- if ! t2_lane_completed "$T2_T1_STATUS"; then
+ if [ "$T2_RUN_T1" = true ] && ! t2_lane_completed "$T2_T1_STATUS"; then
T2_NEXT_COMMAND='re-run make minikube-t2 with T2_RUN_T1=true so the Real PostgreSQL lane is executed'
t2_fail ZERO_TESTS_EXECUTED 'T1 was not executed'
fi
+ if [ "$T2_RUN_T1" != true ] && [ "$T2_T1_CERTIFIED" != true ]; then
+ T2_NEXT_COMMAND='run make minikube-t2 with T2_RUN_T1=true to create an exact-head T1 attestation'
+ t2_fail CERTIFICATION_REQUIRED 'T1 was skipped without a validated exact-head attestation'
+ fi
run_final_preflight
run_healthcheck_if_requested
run_playwright_if_requested
diff --git a/scripts/minikube/wait-gfs-reader-ready.sh b/scripts/minikube/wait-gfs-reader-ready.sh
index 0a5d652fb..9343cf304 100755
--- a/scripts/minikube/wait-gfs-reader-ready.sh
+++ b/scripts/minikube/wait-gfs-reader-ready.sh
@@ -20,6 +20,23 @@ POLL_SECONDS="${GFS_READER_WAIT_POLL_SECONDS:-5}"
kc() { kubectl --context="$CONTEXT" "$@"; }
log() { printf '[wait-gfs-reader-ready] %s\n' "$*" >&2; }
+if ! TIMEOUT_SECONDS="$(python3 - "$TIMEOUT_SECONDS" <<'PY'
+import re
+import sys
+
+raw = sys.argv[1]
+if not re.fullmatch(r"[1-9][0-9]{0,4}", raw):
+ raise SystemExit("timeout must be a positive integer")
+value = int(raw)
+if value > 86400:
+ raise SystemExit("timeout exceeds the one-day limit")
+print(value)
+PY
+)"; then
+ log 'refusing an invalid readiness timeout'
+ exit 2
+fi
+
deadline=$((SECONDS + TIMEOUT_SECONDS))
while :; do
desired="$(kc -n "$GFS_NS" get deployment "$DEPLOY" \
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index 27739ea9a..7cd1e40b3 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -43,20 +43,27 @@ runtime_script="$(sed '/^[[:space:]]*#/d' deploy/scripts/provision-gfs-runtime.s
|| fail 'post-overlay runtime path does not wait for exact GFSC rollouts'
[[ "$runtime_script" != *'rollout restart'* && "$runtime_script" != *'rotate-writer'* ]] \
|| fail 'post-overlay runtime path rotates or unconditionally restarts GFS'
+reconcile_context_err="$(mktemp)"
+if CONTEXT=unverified-context bash deploy/scripts/reconcile-gfs-deploy-credentials.sh 2>"$reconcile_context_err"; then
+ rm -f "$reconcile_context_err"
+ fail 'credential reconciliation accepted an unverified Kubernetes context'
+fi
+grep -Fq 'refusing unverified Kubernetes context' "$reconcile_context_err" \
+ || fail 'credential reconciliation did not explain its unverified-context rejection'
+rm -f "$reconcile_context_err"
make_block() {
awk -v target="$1" '$0 ~ "^" target ":" {active=1} active && /^\.PHONY:/ {exit} active {print}' Makefile
}
-for target in minikube-db-reset; do
- block="$(make_block "$target")"
- reset="$(grep -n 'reset-control-db-storage.sh' <<<"$block" | head -1 | cut -d: -f1)"
- recreate="$(grep -n 'apply -k .* -l app=control-postgres' <<<"$block" | head -1 | cut -d: -f1)"
- postgres_ready="$(grep -nE 'wait .*control-postgres|wait .*deploy/control-postgres' <<<"$block" | head -1 | cut -d: -f1)"
- converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$block" | head -1 | cut -d: -f1)"
- success="$(grep -n 'DB reset complete' <<<"$block" | tail -1 | cut -d: -f1)"
- [[ -n "$reset" && -n "$recreate" && -n "$postgres_ready" && -n "$converge" && -n "$success" ]] && \
- [[ "$reset" -lt "$recreate" && "$recreate" -lt "$postgres_ready" && "$postgres_ready" -lt "$converge" && "$converge" -lt "$success" ]] \
- || fail "$target bypasses safe GFS reset recovery"
-done
+target=minikube-db-reset
+block="$(make_block "$target")"
+reset="$(grep -n 'reset-control-db-storage.sh' <<<"$block" | head -1 | cut -d: -f1)"
+recreate="$(grep -n 'apply -k .* -l app=control-postgres' <<<"$block" | head -1 | cut -d: -f1)"
+postgres_ready="$(grep -nE 'wait .*control-postgres|wait .*deploy/control-postgres' <<<"$block" | head -1 | cut -d: -f1)"
+converge="$(grep -n 'converge-control-db-after-reset.sh' <<<"$block" | head -1 | cut -d: -f1)"
+success="$(grep -n 'DB reset complete' <<<"$block" | tail -1 | cut -d: -f1)"
+[[ -n "$reset" && -n "$recreate" && -n "$postgres_ready" && -n "$converge" && -n "$success" ]] && \
+ [[ "$reset" -lt "$recreate" && "$recreate" -lt "$postgres_ready" && "$postgres_ready" -lt "$converge" && "$converge" -lt "$success" ]] \
+ || fail "$target bypasses safe GFS reset recovery"
reset_storage="$(cat deploy/scripts/reset-control-db-storage.sh)"
for required in preflight-gfs-db-reset.sh 'scale deployment/control-api --replicas=0' 'scale deployment/control-postgres --replicas=0' 'wait_for_pods_gone app=control-api' 'wait_for_pods_gone app=control-postgres' 'preconditions":{"uid":"%s"}' 'persistentvolumeclaims/${PVC}'; do
[[ "$reset_storage" == *"$required"* ]] || fail "reset storage boundary omits $required"
@@ -113,7 +120,8 @@ assert_minikube_upgrade_classifier() {
|| fail "$path does not reconcile ready upgrades, abort unready upgrades, and defer only empty bootstraps"
}
assert_pre_gate_defers_gfs_reconcile() {
- local block="$({ awk -v start="$1" -v end="$2" '
+ local block
+ block="$({ awk -v start="$1" -v end="$2" '
index($0, start) { active=1 }
active { print }
active && index($0, end) { exit }
diff --git a/scripts/tests/test-minikube-gfs-rollout-shim.sh b/scripts/tests/test-minikube-gfs-rollout-shim.sh
index c56580d51..084f78570 100755
--- a/scripts/tests/test-minikube-gfs-rollout-shim.sh
+++ b/scripts/tests/test-minikube-gfs-rollout-shim.sh
@@ -70,6 +70,20 @@ if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\nFalse|\n'
fail 'shim readiness wait went green with a live unready reader pod'
fi
+# Timeout input is untrusted process input; it must be parsed as data rather
+# than reaching Bash arithmetic evaluation.
+malicious_timeout="1; touch $tmp/timeout-pwned"
+if PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 FAKE_POD_ROWS='True|\n' \
+ GFS_READER_WAIT_POLL_SECONDS=0 \
+ kubectl --context=fake rollout status deployment/gfsc-reader -n gfs --timeout="$malicious_timeout" >/dev/null 2>&1; then
+ fail 'shim accepted a malformed rollout timeout'
+fi
+[ ! -e "$tmp/timeout-pwned" ] || fail 'malformed rollout timeout executed shell input'
+if PATH="$shim_path" CONTEXT=fake GFS_READER_WAIT_TIMEOUT_SECONDS="$malicious_timeout" \
+ bash "$WAIT" >/dev/null 2>&1; then
+ fail 'direct readiness helper accepted a malformed timeout'
+fi
+
# 5. A terminating unready pod does not block the wait.
: >"$FAKE_KUBECTL_LOG"
PATH="$shim_path" FAKE_DESIRED=1 FAKE_READY=1 \
diff --git a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
index 98ea7a298..cbbb29f42 100755
--- a/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
+++ b/scripts/tests/test-minikube-settle-gfs-reader-rollout.sh
@@ -9,17 +9,51 @@ fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
bash -n "$SCRIPT"
run_settle() {
- local fake_dir
+ local fake_dir repo profile sha
fake_dir="$(mktemp -d)"
+ repo="$fake_dir/repo"
+ profile='clerum-settle-gfs-reader'
+ mkdir -p "$repo" "$fake_dir/profiles/$profile" "$fake_dir/locks" "$fake_dir/evidence"
+ repo="$(cd "$repo" && pwd -P)"
+ git init -q -b dev "$repo" >/dev/null
+ git -C "$repo" config user.email [email protected]
+ git -C "$repo" config user.name settle-test
+ printf 'settle fixture\n' >"$repo/README"
+ git -C "$repo" add README >/dev/null
+ git -C "$repo" commit -q -m fixture >/dev/null
+ git -C "$repo" remote add origin https://github.com/evenfire-ai/evenfire.git
+ git -C "$repo" switch -c feat/settle-gfs-reader >/dev/null
+ git -C "$repo" update-ref refs/remotes/origin/dev HEAD
+ sha="$(git -C "$repo" rev-parse --short HEAD)"
+ cat >"$fake_dir/profiles/$profile/profile.env" <<EOF
+PROFILE=$profile
+REPO_DIR=$repo
+BRANCH=feat/settle-gfs-reader
+SHA_SHORT=$sha
+DIRTY=false
+EOF
+ printf 'PORT_BASE=23457\n' >"$fake_dir/profiles/$profile/ports.env"
cat >"$fake_dir/kubectl" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$*" >>"${FAKE_KUBECTL_LOG:?}"
case " $* " in
+ *' config get-contexts -o name'*)
+ printf '%s' "${T2_CONTEXT:?}"; exit 0 ;;
+ *' config view '* )
+ printf 'https://%s:6443' '127.0.0.1'; exit 0 ;;
*' get deployment gfsc-reader '*'-o jsonpath={.spec.replicas}'*)
printf '%s' "${FAKE_DESIRED:-1}"; exit 0 ;;
*' get deployment gfsc-reader '*'-o jsonpath={.status.readyReplicas}'*)
printf '%s' "${FAKE_READY:-1}"; exit 0 ;;
+ *' get deployment gfsc-reader '*gfs-template-hash*)
+ printf '%s' 'template-42'; exit 0 ;;
+ *' get deployment gfsc-reader '*jsonpath={.metadata.generation}*)
+ printf '%s' '2'; exit 0 ;;
+ *' get deployment gfsc-reader '*jsonpath={.status.observedGeneration}*)
+ printf '%s' '2'; exit 0 ;;
+ *' get deployment gfsc-reader '*jsonpath={range*)
+ printf '%s\n' 'gfs-controller-reader-db'; exit 0 ;;
*' get deployment gfsc-reader '*revision*)
printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;
*' get deployment gfsc-reader'*)
@@ -27,8 +61,12 @@ case " $* " in
exit 0 ;;
*' get rs -l '*)
printf '%b' "${FAKE_RS_ROWS:-}"; exit 0 ;;
+ *' get rs gfsc-reader-proof-rs '*revision*)
+ printf '%s' "${FAKE_REVISION:-19}"; exit 0 ;;
*' scale rs '*)
exit 0 ;;
+ *' get pods -l '*gfs-template-hash*)
+ printf '%b' "${FAKE_PROOF_POD_ROWS:-gfsc-reader-proof|2026-01-02T00:00:01Z|True||gfsc-reader-proof-rs|}"; exit 0 ;;
*' get pods -l '*)
printf '%b' "${FAKE_POD_ROWS:-}"; exit 0 ;;
*' delete pod '*)
@@ -37,19 +75,44 @@ case " $* " in
printf '%s' "${FAKE_STATE:-rollout-running}"; exit 0 ;;
*' get secret gfs-controller-reader-db '*gfs-dsn-rotated-at*)
printf '%s' "${FAKE_ROTATED:-2026-01-01T00:00:00Z}"; exit 0 ;;
+ *' get secret gfs-controller-reader-db '*jsonpath={.metadata.resourceVersion}*)
+ printf '%s' '42'; exit 0 ;;
+ *' get secret gfs-controller-reader-db '*'-o json'*)
+ python3 - <<'PY'
+import base64
+import json
+import os
+print(json.dumps({
+ "metadata": {"resourceVersion": "42"},
+ "data": {"connection-string": base64.b64encode(os.environ["FAKE_DSN"].encode()).decode()},
+}))
+PY
+ exit 0 ;;
*' get secret gfs-controller-reader-db'*)
[ "${FAKE_SECRET_EXISTS:-1}" = 1 ] || exit 1
exit 0 ;;
+ *' exec '*)
+ input="$(cat)"
+ [ "$input" = "${FAKE_DSN:?}" ] || exit 1
+ exit 0 ;;
*' patch secret gfs-controller-reader-db'*)
- cat >/dev/null
+ cat >"${FAKE_PATCH_LOG:?}"
[ "${FAKE_PATCH_OK:-1}" = 1 ] || exit 1
exit 0 ;;
esac
exit 1
STUB
chmod +x "$fake_dir/kubectl"
- PATH="$fake_dir:$PATH" CONTEXT=fake \
+ local status=0
+ PATH="$fake_dir:$PATH" CONTEXT="$profile" \
+ T2_PROJECT_DIR="$repo" T2_PROFILE="$profile" T2_CONTEXT="$profile" \
+ MINIKUBE_PROFILE="$profile" CONTROL_API_REAL_PG_CONTEXT="$profile" \
+ T2_PROFILE_ROOT="$fake_dir/profiles" T2_PROFILE_ENV="$fake_dir/profiles/$profile/profile.env" \
+ T2_PORTS_ENV="$fake_dir/profiles/$profile/ports.env" \
+ T2_LOCK_ROOT="$fake_dir/locks" T2_EVIDENCE_ROOT="$fake_dir/evidence" \
+ T2_GATE_ID=settle-gfs-reader-test T2_RUN_ID=settle-test-run \
FAKE_KUBECTL_LOG="$fake_dir/kubectl.log" \
+ FAKE_PATCH_LOG="$fake_dir/patch.json" FAKE_DSN=dsn-value \
FAKE_DEPLOY_EXISTS="${1:-1}" \
FAKE_DESIRED="${2:-1}" \
FAKE_READY="${3:-1}" \
@@ -60,13 +123,24 @@ STUB
FAKE_RS_ROWS="${8:-}" \
FAKE_POD_ROWS="${9:-}" \
FAKE_REVISION="${10:-19}" \
- bash "$SCRIPT"
+ FAKE_PROOF_POD_ROWS="${11:-}" \
+ bash "$SCRIPT" || status=$?
printf '%s' "$fake_dir"
+ return "$status"
}
dir="$(run_settle 1 1 1 1 rollout-running '2026-01-01T00:00:00Z' 1)"
grep -q 'patch secret gfs-controller-reader-db' "$dir/kubectl.log" \
|| fail 'Ready leftover claim did not patch the reader Secret to ready'
+grep -q 'gfs-dsn-proof-kind' "$dir/patch.json" \
+ || fail 'Ready leftover claim did not persist credential-consumption proof'
+rm -rf "$dir"
+
+if dir="$(run_settle 1 1 1 1 rollout-running '2026-01-01T00:00:00Z' 1 '' '' 19 'gfsc-reader-old|2025-12-31T23:59:59Z|True||gfsc-reader-old-rs|')"; then
+ fail 'a pre-rotation Ready reader incorrectly certified the rollout'
+fi
+grep -q 'patch secret' "$dir/kubectl.log" \
+ && fail 'a pre-rotation Ready reader must not be marked ready'
rm -rf "$dir"
dir="$(run_settle 1 1 0 1 rollout-running '2026-01-01T00:00:00Z' 1)"
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 717c11aaf..55f29ebe1 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -25,7 +25,7 @@ for file in "$COMMON" "$PREFLIGHT" "$T2" "$T1" \
bash -n "$file"
done
-required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT"
+required_codes="DEVELOPMENT_SCOPE_REQUIRED PROFILE_OWNERSHIP_MISMATCH PROFILE_BUSY PROFILE_LOCK_REQUIRED HEAD_MARKER_MISMATCH IMAGE_MANIFEST_MISMATCH BOOTSTRAP_REQUIRED CERTIFICATION_REQUIRED SECRET_MISSING CONFIGMAP_MISSING POSTGRES_NOT_READY REAL_PG_REQUIRED_BUT_UNAVAILABLE REAL_PG_SUITE_FAILED ZERO_TESTS_EXECUTED PORT_FORWARD_CONFLICT"
for code in $required_codes; do
grep -Fq "$code" "$COMMON" "$PREFLIGHT" "$T2" "$T1"
done
@@ -50,12 +50,14 @@ grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"
grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"
grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"
grep -Fq 'postgres:16-alpine' "$T1"
-grep -Fq 'db.realPostgresMigration.integration.test.ts' "$T1"
-grep -Fq 'gfsReaderRole.realPostgres.integration.test.ts' "$T1"
+grep -Fq 'control-api/test/*realPostgres*.test.ts' "$T1"
grep -Fq 'start_isolated_postgres' "$T1"
grep -Fq 'require_isolated_control_api_files' "$T1"
grep -Fq 'run_suite control-api isolated "$ISOLATED_DSN"' "$T1"
-grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"
+if grep -Fq 'run_suite control-api shared "$ADMIN_DSN"' "$T1"; then
+ echo 'FAIL: control-api Real PostgreSQL suites must not use shared control-postgres' >&2
+ exit 1
+fi
if grep -Fq 'run_suite control-api isolated "$ADMIN_DSN"' "$T1"; then
echo 'FAIL: migration/role-reset lane points at shared control-postgres' >&2
exit 1
@@ -67,11 +69,8 @@ fi
grep -Fq 'T1_REDACT_PASSWORD="${PG_PASSWORD}"' "$T1" || grep -Fq 'T1_REDACT_PASSWORD="$PG_PASSWORD"' "$T1"
grep -Fq -- '--reporter=json' "$T1"
grep -Fq '|| suite_status=$?' "$T1"
-grep -Fq 'complete green reporter' "$T1"
-if grep -Fq 'Real PostgreSQL suite failed in $package ($lane)' "$T1"; then
- echo 'FAIL: T1 still treats npm exit as a suite failure before the JSON reporter' >&2
- exit 1
-fi
+grep -Fq 'Vitest process exited $suite_status' "$T1"
+grep -Fq 'complete green JSON reporter' "$T1"
grep -Fq 'pending_tests' "$T1"
grep -Fq 'numTotalTests' "$T1"
grep -Fq 'port-forward' "$T1"
@@ -155,8 +154,8 @@ grep -Fq 'run_pre_gate' "$T2"
grep -Fq 'run_targeted_sync' "$T2"
grep -Fq 'full-reconcile' "$T2"
grep -Fq 'already-synced' "$T2" "$COMMON"
-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
-grep -Fq 'T2_SKIP_LOCK=true T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"
+grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=true T2_PLAN_FILE' "$T2"
+grep -Fq 'T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" T2_PLAN_MODE=false T2_PLAN_FILE' "$T2"
grep -Fq 'T2_PLAN_MODE=false' "$PREFLIGHT"
grep -Fq -- '--skip-port-forwards' "$T2"
grep -Fq 't2_lane_completed' "$T2"
@@ -176,7 +175,8 @@ if grep -Eq 'make minikube-pre-gate-sync GATE=minikube-t2$' "$T2"; then
exit 1
fi
-grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"
+grep -Fq 't2_mutation_lock' "$COMMON"
+grep -Fq 't2_lock_validate_inherited' "$COMMON"
grep -Fq 'T2_LOCK_ROOT' "$COMMON"
grep -Fq 'REPOSITORY=' "$COMMON"
grep -Fq 'BRANCH=' "$COMMON"
@@ -184,7 +184,10 @@ grep -Fq 'HEAD=' "$COMMON"
grep -Fq 'PROFILE=' "$COMMON"
grep -Fq 'CONTEXT=' "$COMMON"
grep -Fq 'LOCK_KEY=' "$COMMON"
+grep -Fq 'TOKEN=' "$COMMON"
+grep -Fq 'WORKTREE_ID=' "$COMMON"
grep -Fq 'PID=' "$COMMON"
+grep -Fq 'PROCESS_START=' "$COMMON"
if grep -Eq 'get secret .* -o (yaml|json)' "$COMMON" "$PREFLIGHT"; then
echo 'FAIL: preflight reads Secret values' >&2
@@ -212,15 +215,44 @@ bash -c '
parsed="$(t2_get_name control-plane/control-postgres)"
test "$parsed" = "control-plane control-postgres"
t2_lock_acquire
+ t2_lock_release
' bash "$COMMON"
if [ -e "$tmp/locks/clerum-test-contract.lock" ]; then
echo 'FAIL: lock was not cleaned after clean exit' >&2
exit 1
fi
+# A nested mutator must prove the exact opaque lease token and owner identity;
+# T2_SKIP_LOCK=true alone is not an authorization mechanism.
+T2_LOCK_ROOT="$tmp/fenced-locks" T2_PROJECT_DIR="$ROOT" \
+T2_BRANCH=feat/contract-fence T2_HEAD=0123456789abcdef \
+T2_ORIGIN_DEV=0123456789abcdef T2_MERGE_BASE=0123456789abcdef \
+MINIKUBE_PROFILE=clerum-fence T2_CONTEXT=clerum-fence \
+T2_WORKTREE_ID=contract-worktree \
+bash -c '
+ common="$1"
+ source "$common"
+ t2_lock_acquire
+ token="$T2_LOCK_TOKEN"
+ export T2_PROJECT_DIR T2_BRANCH T2_HEAD T2_ORIGIN_DEV T2_MERGE_BASE
+ export MINIKUBE_PROFILE T2_PROFILE T2_CONTEXT T2_WORKTREE_ID T2_LOCK_ROOT
+ if ! T2_SKIP_LOCK=true T2_LOCK_TOKEN="$token" bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; then
+ exit 1
+ fi
+ if T2_SKIP_LOCK=true T2_LOCK_TOKEN=wrong bash -c '\''source "$1"; t2_lock_validate_inherited'\'' bash "$common"; then
+ exit 1
+ fi
+ t2_lock_release 0
+' bash "$COMMON"
+if [ -e "$tmp/fenced-locks/clerum-fence.lock" ]; then
+ echo 'FAIL: fenced lock survived the owner release' >&2
+ exit 1
+fi
+
mkdir -p "$tmp/locks/busy-profile.lock"
cat >"$tmp/locks/busy-profile.lock/owner.env" <<EOF
PROFILE=busy-profile
+TOKEN=lock
PID=$$
EOF
if T2_LOCK_ROOT="$tmp/locks" MINIKUBE_PROFILE=busy-profile T2_CONTEXT=busy-profile \
@@ -243,6 +275,103 @@ fi
grep -Fq 'PROFILE_BUSY' "$tmp/empty.err"
grep -Fq 'orphaned' "$tmp/empty.err"
+cert_root="$tmp/certifications"
+mkdir -p "$cert_root/prior" "$cert_root/current"
+CERT_FILE="$cert_root/prior/evidence.json" EXPECTED_REPOSITORY="$ROOT" \
+EXPECTED_BRANCH=feat/certification EXPECTED_HEAD=cert-head EXPECTED_ORIGIN_DEV=cert-dev \
+EXPECTED_WORKTREE_ID=cert-worktree EXPECTED_PROFILE=clerum-cert EXPECTED_CONTEXT=clerum-cert \
+EXPECTED_FINGERPRINT=cert-fingerprint EXPECTED_GATE_ID=cert-gate \
+python3 - <<'PY'
+import json
+import os
+from datetime import datetime, timedelta, timezone
+
+now = datetime.now(timezone.utc)
+payload = {
+ "certificationVersion": 1,
+ "attestationStatus": "PASS",
+ "runId": "prior-certification-run",
+ "attestationStartedAt": now.isoformat().replace("+00:00", "Z"),
+ "attestationExpiresAt": (now + timedelta(hours=1)).isoformat().replace("+00:00", "Z"),
+ "repository": os.environ["EXPECTED_REPOSITORY"],
+ "branch": os.environ["EXPECTED_BRANCH"],
+ "head": os.environ["EXPECTED_HEAD"],
+ "originDev": os.environ["EXPECTED_ORIGIN_DEV"],
+ "worktreeId": os.environ["EXPECTED_WORKTREE_ID"],
+ "profile": os.environ["EXPECTED_PROFILE"],
+ "context": os.environ["EXPECTED_CONTEXT"],
+ "clusterFingerprintRef": os.environ["EXPECTED_FINGERPRINT"],
+ "gateId": os.environ["EXPECTED_GATE_ID"],
+ "phases": [
+ {"name": "T0", "status": "PASS"},
+ {"name": "T1", "status": "PASS"},
+ {"name": "complete", "status": "PASS"},
+ ],
+}
+with open(os.environ["CERT_FILE"], "w") as handle:
+ json.dump(payload, handle)
+PY
+T2_EVIDENCE_ROOT="$cert_root" T2_PROJECT_DIR="$ROOT" T2_BRANCH=feat/certification \
+T2_HEAD=cert-head T2_ORIGIN_DEV=cert-dev T2_MERGE_BASE=cert-dev \
+T2_WORKTREE_ID=cert-worktree T2_PROFILE=clerum-cert T2_CONTEXT=clerum-cert \
+T2_CLUSTER_FINGERPRINT=cert-fingerprint T2_GATE_ID=cert-gate \
+T2_RUN_ID=current-certification-run \
+bash -c '
+ source "$1"
+ T2_PROJECT_DIR="$3"
+ T2_BRANCH=feat/certification
+ T2_HEAD=cert-head
+ T2_ORIGIN_DEV=cert-dev
+ T2_MERGE_BASE=cert-dev
+ T2_WORKTREE_ID=cert-worktree
+ T2_PROFILE=clerum-cert
+ T2_CONTEXT=clerum-cert
+ T2_CLUSTER_FINGERPRINT=cert-fingerprint
+ T2_GATE_ID=cert-gate
+ T2_RUN_ID=current-certification-run
+ T2_EVIDENCE_DIR="$2/current-run"
+ T2_EVIDENCE_FILE="$T2_EVIDENCE_DIR/evidence.json"
+ mkdir -p "$T2_EVIDENCE_DIR"
+ t2_certification_validate_prior_lanes
+ test "$T2_T0_CERTIFIED" = true
+ test "$T2_T1_CERTIFIED" = true
+ grep -Fq T0_ATTESTED "$T2_EVIDENCE_FILE"
+ grep -Fq T1_ATTESTED "$T2_EVIDENCE_FILE"
+' bash "$COMMON" "$cert_root" "$ROOT"
+
+python3 - "$cert_root/prior/evidence.json" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1]) as handle:
+ payload = json.load(handle)
+payload["attestationStatus"] = "INVALIDATED"
+with open(sys.argv[1], "w") as handle:
+ json.dump(payload, handle)
+PY
+if T2_EVIDENCE_ROOT="$cert_root" T2_PROJECT_DIR="$ROOT" T2_BRANCH=feat/certification \
+ T2_HEAD=cert-head T2_ORIGIN_DEV=cert-dev T2_MERGE_BASE=cert-dev \
+ T2_WORKTREE_ID=cert-worktree T2_PROFILE=clerum-cert T2_CONTEXT=clerum-cert \
+ T2_CLUSTER_FINGERPRINT=cert-fingerprint T2_GATE_ID=cert-gate \
+ bash -c '
+ source "$1"
+ T2_PROJECT_DIR="$2"
+ T2_BRANCH=feat/certification
+ T2_HEAD=cert-head
+ T2_ORIGIN_DEV=cert-dev
+ T2_MERGE_BASE=cert-dev
+ T2_WORKTREE_ID=cert-worktree
+ T2_PROFILE=clerum-cert
+ T2_CONTEXT=clerum-cert
+ T2_CLUSTER_FINGERPRINT=cert-fingerprint
+ T2_GATE_ID=cert-gate
+ t2_certification_validate_prior_lanes
+ ' bash "$COMMON" "$ROOT" 2>"$tmp/certification.err"; then
+ echo 'FAIL: invalidated certification was accepted' >&2
+ exit 1
+fi
+grep -Fq CERTIFICATION_REQUIRED "$tmp/certification.err"
+
redaction_file="$tmp/t1-redaction.log"
redaction_secret='super-secret'
printf '%s=%s\n' password "$redaction_secret" >"$redaction_file"
@@ -255,6 +384,45 @@ if grep -Fq "$redaction_secret" "$redaction_file"; then
exit 1
fi
+# A complete green reporter is not sufficient evidence when npm/Vitest exits
+# non-zero during teardown, worker shutdown, OOM handling, or signal cleanup.
+# Exercise the adjudicator with representative failure statuses without Docker
+# or Kubernetes; every one must fail the suite despite the green JSON payload.
+t1_exit_root="$tmp/t1-exit-adjudicator"
+mkdir -p "$t1_exit_root/control-api/test"
+printf 'fixture\n' >"$t1_exit_root/control-api/test/fake.realPostgres.test.ts"
+for exit_code in 1 2 7 126 137; do
+ if EXIT_CODE="$exit_code" T1_EXIT_ROOT="$t1_exit_root" bash -c '
+ set -euo pipefail
+ source "$1"
+ PROJECT_DIR="$T1_EXIT_ROOT"
+ T1_TMP_DIR="$(mktemp -d)"
+ trap '\''rm -rf "$T1_TMP_DIR"'\'' EXIT
+ list_real_pg_files() {
+ printf "%s\\n" "$T1_EXIT_ROOT/control-api/test/fake.realPostgres.test.ts"
+ }
+ is_isolated_control_api_file() { return 0; }
+ npm() {
+ local output_file=""
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --outputFile=*) output_file="${1#--outputFile=}" ;;
+ --outputFile) shift; output_file="$1" ;;
+ esac
+ shift
+ done
+ printf "%s\\n" '\''{"success":true,"testResults":[{"status":"passed"}],"numPassedTests":1,"numFailedTests":0,"numPendingTestSuites":0,"numPendingTests":0,"numTotalTests":1}'\'' >"$output_file"
+ return "$EXIT_CODE"
+ }
+ if run_suite control-api isolated postgresql://isolated; then
+ exit 1
+ fi
+ ' bash "$T1"; then
+ echo "FAIL: T1 accepted a green reporter after npm exit $exit_code" >&2
+ exit 1
+ fi
+done
+
bash "$ROOT/scripts/tests/test-minikube-t2-public-boundary.sh"
if T2_PUBLIC_BASE_REF=nonexistent-ref-for-contract \
bash "$ROOT/scripts/tests/test-minikube-t2-public-boundary.sh"; then
diff --git a/scripts/tests/test-minikube-t2-public-boundary.sh b/scripts/tests/test-minikube-t2-public-boundary.sh
index 8813e4a7d..a21cedc3d 100755
--- a/scripts/tests/test-minikube-t2-public-boundary.sh
+++ b/scripts/tests/test-minikube-t2-public-boundary.sh
@@ -40,6 +40,11 @@ import sys
diff = Path(sys.argv[1]).read_text(errors="replace")
bad = []
current = ""
+safe_source_paths = {
+ "deploy/scripts/lib/gfs-credential-rollout.sh",
+ "deploy/scripts/lib/gfs-credential-secret.sh",
+ "deploy/scripts/reconcile-gfs-deploy-credentials.sh",
+}
for line in diff.splitlines():
if line.startswith("+++ b/"):
current = line[6:]
@@ -49,7 +54,10 @@ for line in diff.splitlines():
or path.startswith(".env.")
or path.endswith((".pem", ".key", ".p12", ".pfx", ".log"))
or path.endswith(("/kubeconfig", "/config"))
- or any(token in path for token in ("id_rsa", "id_ed25519", "credential", "wallet", "keystore"))
+ or (
+ any(token in path for token in ("id_rsa", "id_ed25519", "credential", "wallet", "keystore"))
+ and path not in safe_source_paths
+ )
or "screenshot" in path
or "e2e-artifacts" in path
):
diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.sh
index c4ac168ef..1f3f3e0fb 100755
--- a/scripts/tests/test-minikube-t2-scenarios.sh
+++ b/scripts/tests/test-minikube-t2-scenarios.sh
@@ -60,6 +60,7 @@ printf 'PORT_BASE=23117\nCONTROL_API_URL=profile-owned-url\n' >"$profile_root/po
repo_env=(
T2_PROJECT_DIR="$repo"
MINIKUBE_PROFILE="$profile"
+ T2_CONTEXT="$profile"
CONTROL_API_REAL_PG_CONTEXT="$profile"
T2_PROFILE_ROOT="$tmp/profiles"
T2_PROFILE_ENV="$profile_root/profile.env"
@@ -88,7 +89,7 @@ expect_code DEVELOPMENT_SCOPE_REQUIRED shared-profile shared-profile \
bash -c 'source "$1"; t2_profile_scope' bash "$COMMON"
expect_code DEVELOPMENT_SCOPE_REQUIRED context-mismatch context-mismatch \
- env "${repo_env[@]}" CONTROL_API_REAL_PG_CONTEXT=another-context \
+ env "${repo_env[@]}" T2_CONTEXT=another-context CONTROL_API_REAL_PG_CONTEXT=another-context \
bash -c 'source "$1"; t2_profile_scope' bash "$COMMON"
fake_bin="$tmp/fake-bin"
@@ -96,20 +97,27 @@ mkdir -p "$fake_bin"
cat >"$fake_bin/kubectl" <<'EOF'
#!/usr/bin/env bash
case "$*" in
- *"config current-context"*) exit 0 ;;
+ *"config get-contexts -o name"*)
+ for arg in "$@"; do
+ case "$arg" in
+ --context=*) printf '%s' "${arg#--context=}"; exit 0 ;;
+ esac
+ done
+ printf '%s' "${FAKE_CONTEXT:-}" ;;
*"config view"*) printf '%s://%s:6443' https "${FAKE_ENDPOINT:-10.0.0.1}" ;;
*) exit 0 ;;
esac
EOF
chmod +x "$fake_bin/kubectl"
+context_env=("${repo_env[@]}" FAKE_CONTEXT="$profile")
expect_code DEVELOPMENT_SCOPE_REQUIRED remote-context remote-context \
- env "${repo_env[@]}" PATH="$fake_bin:$PATH" \
+ env "${repo_env[@]}" FAKE_CONTEXT=other-context PATH="$fake_bin:$PATH" \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=127.0.0.1 \
+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=127.0.0.1 \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \
+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT='[::1]' \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
-env "${repo_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \
+env "${context_env[@]}" PATH="$fake_bin:$PATH" FAKE_ENDPOINT=localhost \
bash -c 'source "$1"; t2_context_check' bash "$COMMON"
missing_profile_state="$(env "${repo_env[@]}" bash -c 'source "$1"; t2_mk(){ printf "%s" "Profile not found"; }; t2_profile_status; printf "%s" "$T2_PLAN_STATE"' bash "$COMMON")"
@@ -264,9 +272,9 @@ grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$ROOT/scripts/e2e/minikube-real-postg
grep -Fq 'ZERO_TESTS_EXECUTED' "$ROOT/scripts/e2e/minikube-real-postgres.sh"
grep -Fq 'start_isolated_postgres' "$ROOT/scripts/e2e/minikube-real-postgres.sh"
grep -Fq 'ISOLATED_DSN' "$ROOT/scripts/e2e/minikube-real-postgres.sh"
-grep -Fq '|| suite_status=$?' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must capture vitest exit without aborting before the reporter'
-grep -Fq 'complete green reporter' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must document reporter-authoritative verdict'
-grep -Fq 'Real PostgreSQL suite failed in $package ($lane)' "$ROOT/scripts/e2e/minikube-real-postgres.sh" && fail 'T1 still treats npm exit as a suite failure before the JSON reporter'
+grep -Fq '|| suite_status=$?' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must capture vitest exit before adjudicating the reporter'
+grep -Fq 'complete green JSON reporter' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must require a complete green reporter'
+grep -Fq 'Vitest process exited $suite_status' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must fail on a non-zero Vitest exit'
grep -Fq 'restore_gfs_runtime_credentials' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 must restore branch-profile GFS credentials on exit'
grep -Fq 'GFS_RESTORE_ACTIVE_NOLOGIN=true' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 GFS restore must opt into the NOLOGIN recovery contract'
grep -Fq 'GFS_RECOVER_ABANDONED_STATE=true' "$ROOT/scripts/e2e/minikube-real-postgres.sh" || fail 'T1 GFS restore must resume an interrupted gfsc-reader rollout claim'
@@ -284,7 +292,8 @@ grep -Fq 'wait-gfs-reader-ready.sh' "$ROOT/scripts/minikube/pre-gate-sync.sh" ||
[[ "$(grep -c 'gfs-rollout-shim' "$ROOT/scripts/minikube/full-setup.sh")" -ge 2 ]] || fail 'full-setup REUSE_DB reconciles must use the HCC-safe reader rollout wait on both calls'
[[ "$(grep -c 'sync-auth-key.sh' "$ROOT/scripts/minikube/full-setup.sh")" -ge 3 ]] || fail 'full-setup must re-sync gfs-config.jwt-public-key before both GFS reconciles'
grep -Fq 'T2_LOCK_ROOT' "$COMMON"
-grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"
+grep -Fq 't2_mutation_lock' "$COMMON"
+grep -Fq 't2_lock_validate_inherited' "$COMMON"
grep -Fq 'PORT_FORWARD_CONFLICT' "$COMMON"
grep -Fq 'while read -r uid pid ppid rest' "$COMMON" || fail 't2_process_check must split ps -ef fields'
grep -Fq 'IFS= read -r uid pid ppid rest' "$COMMON" && fail 'IFS= read disables PID split and silences PORT_FORWARD_CONFLICT'
@@ -314,7 +323,7 @@ grep -Fq 'kubectl -n control-plane port-forward svc/control-api' "$tmp/pf-awk.ou
grep -Fq -- "--context ${pf_profile} port-forward svc/control-ui" "$tmp/pf-awk.out" || fail 'awk missed run-e2e real launcher'
grep -Fq 'kubectl port-forward --context=' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'
grep -Fq '/usr/local/bin/kubectl --context=' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl with flags before port-forward'
-read -r uid pid ppid rest <<<"$pf_all"
+read -r _ pid _ _ <<<"$pf_all"
[ "$pid" = 4242 ] || fail "ps field split left pid='$pid' instead of 4242"
pf_root="$tmp/pf-check-profiles"
diff --git a/scripts/tests/test-provision-gfs-runtime.sh b/scripts/tests/test-provision-gfs-runtime.sh
index 0201a8abe..2740a6431 100755
--- a/scripts/tests/test-provision-gfs-runtime.sh
+++ b/scripts/tests/test-provision-gfs-runtime.sh
@@ -39,13 +39,14 @@ export CALL_LOG="$TMP/calls.log"
# through the ALLOWED_CONTEXTS override — the same mechanism a self-hosted dev
# pipeline uses for a GKE dev context that the generic patterns do not name.
for run in first-upgrade idempotent-rerun; do
+ : "$run"
ALLOWED_CONTEXTS=clerum-dev-test PATH="$TMP/bin:$PATH" /bin/bash "$ROOT/deploy/scripts/provision-gfs-runtime.sh" \
--context clerum-dev-test \
--overlay "$TMP/overlay" \
--skip-auth-sync >/dev/null
done
PATH="$TMP/bin:$PATH" /bin/bash "$ROOT/deploy/scripts/provision-gfs-runtime.sh" \
- --context gke-prod-test \
+ --context gke_prod_test \
--overlay "$TMP/overlay" \
--allow-prod \
--skip-auth-sync \