Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# Secret-safe helpers for the local Evenfire Minikube T0/T1/T2 contract.
# shellcheck disable=SC2034,SC2269
set -eo pipefail
set +x
set +u
T2_SCRIPT_DIR="$T2_SCRIPT_DIR"
if [ -z "$T2_SCRIPT_DIR" ]; then T2_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"; fi
# shellcheck source=t2-worktree-id.sh
. "$T2_SCRIPT_DIR/t2-worktree-id.sh"
# shellcheck source=profile-owner.sh
. "$T2_SCRIPT_DIR/profile-owner.sh"
# shellcheck source=port-forward-owner.sh
. "$T2_SCRIPT_DIR/port-forward-owner.sh"
# shellcheck source=profile-readiness.sh
. "$T2_SCRIPT_DIR/profile-readiness.sh"
# shellcheck source=pre-gate-marker.sh
. "$T2_SCRIPT_DIR/pre-gate-marker.sh"
# shellcheck source=context-identity.sh
. "$T2_SCRIPT_DIR/context-identity.sh"
T2_PROJECT_DIR="$T2_PROJECT_DIR"
if [ -z "$T2_PROJECT_DIR" ]; then T2_PROJECT_DIR="$(cd -- "$T2_SCRIPT_DIR/../.." && pwd -P)"; fi
T2_PROFILE="$T2_PROFILE"
if [ -z "$T2_PROFILE" ]; then T2_PROFILE="$MINIKUBE_PROFILE"; fi
T2_CONTEXT="$T2_CONTEXT"
T2_CONTEXT_SOURCE=explicit-t2-context
if [ -z "$T2_CONTEXT" ] && [ -n "$CONTROL_API_REAL_PG_CONTEXT" ]; then
T2_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
T2_CONTEXT_SOURCE=explicit-control-api-real-pg-context
fi
if [ -z "$T2_CONTEXT" ] && [ -n "$K8S_CONTEXT" ]; then
T2_CONTEXT="$K8S_CONTEXT"
T2_CONTEXT_SOURCE=explicit-k8s-context
fi
if [ -z "$T2_CONTEXT" ] && [ -n "$KUBECONTEXT" ]; then
T2_CONTEXT="$KUBECONTEXT"
T2_CONTEXT_SOURCE=explicit-kubecontext
fi
if [ -z "$T2_CONTEXT" ]; then
T2_CONTEXT="$T2_PROFILE"
T2_CONTEXT_SOURCE=implicit-profile-fallback
fi
T2_TIMEOUT_SECONDS="$T2_TIMEOUT_SECONDS"
if [ -z "$T2_TIMEOUT_SECONDS" ]; then T2_TIMEOUT_SECONDS="$T2_TIMEOUT"; fi
if [ -z "$T2_TIMEOUT_SECONDS" ]; then T2_TIMEOUT_SECONDS=180; fi
T2_RUNTIME_TIMEOUT_SECONDS="$T2_RUNTIME_TIMEOUT_SECONDS"
if [ -z "$T2_RUNTIME_TIMEOUT_SECONDS" ]; then T2_RUNTIME_TIMEOUT_SECONDS=60; fi
T2_RUNTIME_KILL_GRACE_SECONDS="$T2_RUNTIME_KILL_GRACE_SECONDS"
if [ -z "$T2_RUNTIME_KILL_GRACE_SECONDS" ]; then T2_RUNTIME_KILL_GRACE_SECONDS=5; fi
T2_DEADLINE_RUNNER="$T2_DEADLINE_RUNNER"
if [ -z "$T2_DEADLINE_RUNNER" ]; then T2_DEADLINE_RUNNER="$T2_SCRIPT_DIR/run-with-deadline.mjs"; fi
T2_PROFILE_ROOT="$T2_PROFILE_ROOT"
if [ -z "$T2_PROFILE_ROOT" ]; then T2_PROFILE_ROOT="$CLERUM_PROFILE_CACHE_ROOT"; fi
if [ -z "$T2_PROFILE_ROOT" ]; then T2_PROFILE_ROOT="$HOME/.cache/clerum/minikube-profiles"; fi
T2_PROFILE_ENV="$T2_PROFILE_ENV"
if [ -z "$T2_PROFILE_ENV" ]; then T2_PROFILE_ENV="$T2_PROFILE_ROOT/$T2_PROFILE/profile.env"; fi
T2_PORTS_ENV="$T2_PORTS_ENV"
if [ -z "$T2_PORTS_ENV" ]; then T2_PORTS_ENV="$CLERUM_PROFILE_PORTS_ENV"; fi
if [ -z "$T2_PORTS_ENV" ]; then T2_PORTS_ENV="$T2_PROFILE_ROOT/$T2_PROFILE/ports.env"; fi
T2_LOCK_ROOT="$T2_LOCK_ROOT"
if [ -z "$T2_LOCK_ROOT" ]; then T2_LOCK_ROOT="$HOME/.cache/evenfire/minikube-t2-locks"; fi
T2_EVIDENCE_ROOT="$T2_EVIDENCE_ROOT"
if [ -z "$T2_EVIDENCE_ROOT" ]; then T2_EVIDENCE_ROOT="$T2_PROJECT_DIR/.local-notes/infra/runs"; fi
T2_MARKER_NAME="$T2_MARKER_NAME"
if [ -z "$T2_MARKER_NAME" ]; then T2_MARKER_NAME="$CLERUM_PRE_GATE_SYNC_CONFIGMAP"; fi
if [ -z "$T2_MARKER_NAME" ]; then T2_MARKER_NAME=clerum-pre-gate-sync-state; fi
T2_CONTROL_NAMESPACE="$T2_CONTROL_NAMESPACE"
if [ -z "$T2_CONTROL_NAMESPACE" ]; then T2_CONTROL_NAMESPACE=control-plane; fi
T2_IMAGE_MANIFEST="$T2_IMAGE_MANIFEST"
if [ -z "$T2_IMAGE_MANIFEST" ]; then T2_IMAGE_MANIFEST="$T2_PROJECT_DIR/deploy/minikube/.image-manifest.json"; fi
T2_ERROR_CODE=""
T2_NEXT_COMMAND='re-run the canonical command with the verified profile'
T2_PLAN_STATE=""
T2_PLAN_REASON=""
T2_PLAN_MODE="$T2_PLAN_MODE"
if [ -z "$T2_PLAN_MODE" ]; then T2_PLAN_MODE=false; fi
T2_BOOTSTRAP_REQUIRED=false
T2_PROFILE_HEALTHY=false
T2_PROFILE_STATUS=unknown
T2_CONTEXT_IDENTITY_VERIFIED=false
T2_UNREADY_DEPLOYMENTS=""
T2_MARKER_MATCHES_HEAD=false
T2_MARKER_JSON=""
T2_IMAGE_SOURCE=""
T2_IMAGE_TAG=""
T2_MARKER_IMAGES_GENERATED_AT=""
T2_MANIFEST_GENERATED_AT=""
T2_CLUSTER_FINGERPRINT=""
T2_WORKTREE_ID=""
T2_HEAD=""
T2_BRANCH=""
T2_ORIGIN_DEV=""
T2_MERGE_BASE=""
T2_LOCK_DIR=""
T2_LOCK_KEY=""
T2_EVIDENCE_DIR=""
T2_EVIDENCE_FILE=""
T2_EVIDENCE_KIND="$T2_EVIDENCE_KIND"
if [ -z "$T2_EVIDENCE_KIND" ]; then T2_EVIDENCE_KIND=certification; fi
T2_HEALTHCHECK_PENDING="$T2_HEALTHCHECK_PENDING"
if [ -z "$T2_HEALTHCHECK_PENDING" ]; then T2_HEALTHCHECK_PENDING=false; fi
T2_LOCK_HELD=false
T2_LOCK_RELEASED=false
T2_LOCK_TOKEN="$T2_LOCK_TOKEN"
T2_RUN_ID="$T2_RUN_ID"
T2_GATE_ID="$T2_GATE_ID"
if [ -z "$T2_GATE_ID" ]; then T2_GATE_ID=minikube-t2; fi
T2_CERTIFICATION_VERSION=1
T2_T0_CERTIFIED=false
T2_T1_CERTIFIED=false
T2_PRIOR_ATTESTATION=""
T2_PRIOR_T0_ATTESTATION=""
T2_PRIOR_T1_ATTESTATION=""
T2_REQUIRED_NAMESPACES="$T2_REQUIRED_NAMESPACES"
if [ -z "$T2_REQUIRED_NAMESPACES" ]; then T2_REQUIRED_NAMESPACES="control-plane gfs mcp-host mcp-server profiles rpc-proxy channels sandbox-recipes sandbox-ui webhook-ingress registry"; fi
T2_REQUIRED_SERVICES="$T2_REQUIRED_SERVICES"
if [ -z "$T2_REQUIRED_SERVICES" ]; then T2_REQUIRED_SERVICES="control-plane/control-postgres control-plane/control-api control-plane/control-ui control-plane/host-context-controller control-plane/workflow-recipes profiles/external-rest-api profiles/profile-ui rpc-proxy/rpc-proxy mcp-server/mcp-proxy"; fi
# An empty or unset scope uses the production contract: every core Deployment
# is mandatory and every additional Deployment in the inventory is evaluated.
# Tests may provide a non-empty scope for isolated fixtures; that explicit
# scope remains narrowed to the listed deployments.
T2_REQUIRED_DEPLOYMENTS="$T2_REQUIRED_DEPLOYMENTS"
T2_REQUIRED_DEPLOYMENTS_INCLUDE_ADDITIONAL=false
if [ -z "${T2_REQUIRED_DEPLOYMENTS//[[:space:]]/}" ]; then
T2_REQUIRED_DEPLOYMENTS="control-plane/control-api control-plane/host-context-controller profiles/external-rest-api rpc-proxy/rpc-proxy mcp-host/chatllm"
T2_REQUIRED_DEPLOYMENTS_INCLUDE_ADDITIONAL=true
fi
T2_REQUIRED_SECRETS="$T2_REQUIRED_SECRETS"
if [ -z "$T2_REQUIRED_SECRETS" ]; then T2_REQUIRED_SECRETS="control-plane/control-postgres control-plane/control-api-secrets control-plane/control-ui-secrets control-plane/inter-service-tokens profiles/external-rest-api-secrets rpc-proxy/rpc-proxy-secrets mcp-host/chatllm-api-keys gfs/gfs-controller-db gfs/gfs-controller-reader-db"; fi
T2_REQUIRED_CONFIGMAPS="$T2_REQUIRED_CONFIGMAPS"
if [ -z "$T2_REQUIRED_CONFIGMAPS" ]; then T2_REQUIRED_CONFIGMAPS="control-plane/$T2_MARKER_NAME control-plane/control-api-config control-plane/control-api-public-key mcp-host/mcp-host-config mcp-host/clerum-model-secret-mapping profiles/external-rest-api-config rpc-proxy/rpc-proxy-config gfs/gfs-config"; fi
T2_REQUIRED_PVC="$T2_REQUIRED_PVC"
if [ -z "$T2_REQUIRED_PVC" ]; then T2_REQUIRED_PVC="control-plane/control-postgres-data"; fi
T2_PLAN_MODE="$T2_PLAN_MODE"
T2_SKIP_LOCK="$T2_SKIP_LOCK"
T2_PLAN_FILE="$T2_PLAN_FILE"
T2_RUN_T0="$T2_RUN_T0"
T2_RUN_T1="$T2_RUN_T1"
T2_REQUIRE_PLAYWRIGHT="$T2_REQUIRE_PLAYWRIGHT"
T2_T0_COMMAND="$T2_T0_COMMAND"
T2_PLAYWRIGHT_COMMAND="$T2_PLAYWRIGHT_COMMAND"
T2_HEALTHCHECK_COMMAND="$T2_HEALTHCHECK_COMMAND"
T2_PORT_FORWARD_COMMAND="$T2_PORT_FORWARD_COMMAND"
T2_RESET_PVC="$T2_RESET_PVC"
T2_EXPECTED_PVC_UID="$T2_EXPECTED_PVC_UID"
T2_TMP_ROOT="$TMPDIR"
CONTROL_API_REAL_PG_CONTEXT="$CONTROL_API_REAL_PG_CONTEXT"
MINIKUBE_PROFILE="$MINIKUBE_PROFILE"
set -u
t2_fail() {
T2_ERROR_CODE="$1"
shift
printf '%s: %s\n' "$T2_ERROR_CODE" "$*" >&2
printf 'next: %s\n' "$T2_NEXT_COMMAND" >&2
return 1
}
t2_validate_runtime_deadline() {
if ! [[ "$T2_RUNTIME_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]] ||
[ "$T2_RUNTIME_TIMEOUT_SECONDS" -gt 300 ]; then
T2_NEXT_COMMAND='set T2_RUNTIME_TIMEOUT_SECONDS to an integer from 1 to 300, then re-run T2'
t2_fail DEVELOPMENT_SCOPE_REQUIRED \
'T2_RUNTIME_TIMEOUT_SECONDS must be an integer from 1 to 300'
return 1
fi
if ! [[ "$T2_RUNTIME_KILL_GRACE_SECONDS" =~ ^[1-9][0-9]*$ ]] ||
[ "$T2_RUNTIME_KILL_GRACE_SECONDS" -gt 60 ]; then
T2_NEXT_COMMAND='set T2_RUNTIME_KILL_GRACE_SECONDS to an integer from 1 to 60, then re-run T2'
t2_fail DEVELOPMENT_SCOPE_REQUIRED \
'T2_RUNTIME_KILL_GRACE_SECONDS must be an integer from 1 to 60'
return 1
fi
if [ ! -f "$T2_DEADLINE_RUNNER" ]; then
T2_NEXT_COMMAND='restore scripts/minikube/run-with-deadline.mjs, then re-run T2'
t2_fail LOCAL_DEPENDENCY_MISSING \
"bounded Minikube/Kubernetes runtime helper is missing: $T2_DEADLINE_RUNNER"
return 1
fi
}
t2_bounded_command() {
local label="$1" timeout_seconds="$2"
shift 2
node "$T2_DEADLINE_RUNNER" \
--timeout-seconds "$timeout_seconds" \
--heartbeat-seconds 20 \
--kill-grace-seconds "$T2_RUNTIME_KILL_GRACE_SECONDS" \
--label "$label" -- "$@"
}
t2_require_commands() {
local command_name
for command_name in git kubectl minikube node python3 shasum awk sed find ps; do
if ! command -v "$command_name" >/dev/null 2>&1; then
T2_NEXT_COMMAND="install or enable $command_name, then re-run make minikube-t2-preflight"
t2_fail LOCAL_DEPENDENCY_MISSING "required local dependency is unavailable: $command_name"
fi
done
t2_validate_runtime_deadline
}
t2_kc() {
t2_bounded_command t2-kubectl "$T2_RUNTIME_TIMEOUT_SECONDS" \
kubectl --context="$T2_CONTEXT" "$@"
}
t2_mk() {
t2_bounded_command t2-minikube "$T2_RUNTIME_TIMEOUT_SECONDS" \
minikube -p "$T2_PROFILE" "$@"
}
t2_require_explicit_context() {
if [ "$T2_CONTEXT_SOURCE" = implicit-profile-fallback ]; then
T2_NEXT_COMMAND='set CONTROL_API_REAL_PG_CONTEXT (or T2_CONTEXT) to the generated branch-owned context'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'Kubernetes context must be supplied explicitly; profile-name fallback is not accepted'
fi
}
t2_canonical_path() {
(cd -- "$1" 2>/dev/null && pwd -P)
}
# refs/remotes/origin/dev is shared by every worktree of this repository, so a
# fetch from any of them moves it while a lane runs. The lease owner reads it
# once (t2_lock_acquire exports T2_PINNED_ORIGIN_DEV) and every child of that
# lease validates against the pin. The pin is honored only under an inherited
# lease: an exported pin cannot certify a branch against an older dev, and a
# lease child that lost the pin fails instead of re-reading the moved ref.
#
# A child the lease owner starts without handing it the lease (T0 in t2.sh
# run_t0) must not inherit the pin either, or it is refused as an unleased pin.
# t2_run_outside_lease starts it without T2_PINNED_ORIGIN_DEV; the arguments
# are those of env(1), so NAME=VALUE assignments may precede the command.
t2_run_outside_lease() {
env -u T2_PINNED_ORIGIN_DEV "$@"
}
t2_resolve_origin_dev() {
local pin="${T2_PINNED_ORIGIN_DEV:-}" current
current="$(git -C "$T2_PROJECT_DIR" rev-parse --verify origin/dev 2>/dev/null || true)"
if [ "${T2_SKIP_LOCK:-}" != true ]; then
if [ -n "$pin" ]; then
T2_NEXT_COMMAND='unset T2_PINNED_ORIGIN_DEV; the lease owner pins origin/dev for its children'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2_PINNED_ORIGIN_DEV is only honored under an inherited T2 lease'
return 1
fi
T2_ORIGIN_DEV="$current"
return 0
fi
if [ -z "$pin" ]; then
T2_NEXT_COMMAND='run this child from the T2 lease owner, which exports T2_PINNED_ORIGIN_DEV'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'an inherited T2 lease carries no pinned origin/dev'
return 1
fi
if [[ ! "$pin" =~ ^[0-9a-f]{40}$ ]]; then
T2_NEXT_COMMAND='run this child from the T2 lease owner, which exports the full origin/dev SHA'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2_PINNED_ORIGIN_DEV is not a full commit SHA'
return 1
fi
if ! git -C "$T2_PROJECT_DIR" cat-file -e "$pin^{commit}" 2>/dev/null; then
T2_NEXT_COMMAND='run this child from the T2 lease owner of this repository'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'T2_PINNED_ORIGIN_DEV is not a commit in this repository'
return 1
fi
if [ -z "$current" ]; then
T2_NEXT_COMMAND='fetch origin/dev and run from a named development branch'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'branch, HEAD, or origin/dev could not be resolved'
return 1
fi
if ! git -C "$T2_PROJECT_DIR" merge-base --is-ancestor "$pin" "$current"; then
T2_NEXT_COMMAND='origin/dev was rewritten during the lane; merge the current origin/dev, then re-run T2'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'the pinned origin/dev is not an ancestor of the current origin/dev'
return 1
fi
if [ "$current" != "$pin" ]; then
printf '[minikube-t2] origin/dev moved during the lane: pinned %s, local ref now %s; validating against the pin\n' \
"$pin" "$current" >&2
fi
T2_ORIGIN_DEV="$pin"
}
t2_repo_metadata() {
local actual_root remote_url
actual_root="$(git -C "$T2_PROJECT_DIR" rev-parse --show-toplevel 2>/dev/null || true)"
if [ -z "$actual_root" ]; then
T2_NEXT_COMMAND='run from the canonical Evenfire checkout'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'the working directory is not a Git repository'
fi
actual_root="$(t2_canonical_path "$actual_root")"
if [ "$actual_root" != "$T2_PROJECT_DIR" ]; then
T2_NEXT_COMMAND='run from the Git worktree that owns the branch profile'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "repository root does not match the active worktree"
fi
remote_url="$(git -C "$T2_PROJECT_DIR" remote get-url origin 2>/dev/null || true)"
if [[ "$remote_url" != *evenfire* ]]; then
T2_NEXT_COMMAND='select a checkout whose origin remote is the Evenfire repository'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'origin is not the Evenfire repository'
fi
T2_BRANCH="$(git -C "$T2_PROJECT_DIR" branch --show-current 2>/dev/null || true)"
T2_HEAD="$(git -C "$T2_PROJECT_DIR" rev-parse --verify HEAD 2>/dev/null || true)"
t2_resolve_origin_dev || return 1
if [ -z "$T2_BRANCH" ] || [ -z "$T2_HEAD" ] || [ -z "$T2_ORIGIN_DEV" ]; then
T2_NEXT_COMMAND='fetch origin/dev and run from a named development branch'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'branch, HEAD, or origin/dev could not be resolved'
fi
case "$T2_BRANCH" in
main|master|dev|origin/dev|production|staging|release/*)
T2_NEXT_COMMAND='create a new development branch from the current origin/dev'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "protected branch is not allowed: $T2_BRANCH" ;;
esac
if [[ "$T2_BRANCH" == detached/* ]]; then
T2_NEXT_COMMAND='create a named development branch from origin/dev'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'detached HEADs are not allowed'
fi
if ! git -C "$T2_PROJECT_DIR" merge-base --is-ancestor "$T2_ORIGIN_DEV" "$T2_HEAD"; then
T2_NEXT_COMMAND='rebase or merge the current origin/dev, then re-run the preflight'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'origin/dev is not an ancestor of HEAD'
fi
T2_MERGE_BASE="$(git -C "$T2_PROJECT_DIR" merge-base "$T2_ORIGIN_DEV" "$T2_HEAD" 2>/dev/null || true)"
if [ "$T2_MERGE_BASE" != "$T2_ORIGIN_DEV" ]; then
T2_NEXT_COMMAND='rebase or merge the current origin/dev, then re-run the preflight'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'merge-base is not the current origin/dev'
fi
if [ -n "$(git -C "$T2_PROJECT_DIR" status --porcelain=v1)" ]; then
T2_NEXT_COMMAND='commit or restore the worktree before invoking Minikube validation'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'worktree is dirty'
fi
T2_WORKTREE_ID="$(t2_worktree_id "$T2_PROJECT_DIR")"
}
t2_profile_scope() {
t2_require_explicit_context
if [ -z "$T2_PROFILE" ] || [ -z "$T2_CONTEXT" ] || [ "$T2_PROFILE" != "$T2_CONTEXT" ]; then
T2_NEXT_COMMAND='set MINIKUBE_PROFILE and CONTROL_API_REAL_PG_CONTEXT to the same generated profile'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'an explicit branch-owned profile and matching Kubernetes context are required'
fi
case "$T2_PROFILE" in
*gke*|*prod*|*staging*|clerum-test|default|minikube)
T2_NEXT_COMMAND='select the generated branch-owned Minikube profile for this worktree'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "profile is shared, non-local, or protected: $T2_PROFILE" ;;
esac
if [[ ! "$T2_PROFILE" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
T2_NEXT_COMMAND='use the generated profile name without shell metacharacters'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'profile name is not a valid local Minikube identifier'
fi
if ! profile_owner_validate_selection "$T2_PROFILE_ENV" "$T2_PORTS_ENV" \
"$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_PROFILE"; then
case "$PROFILE_OWNER_ERROR_CODE" in
PROFILE_METADATA_MISSING)
# A profile that was never created has no directory to restore (#1001).
if [ ! -e "$(dirname -- "$T2_PROFILE_ENV")" ]; then
T2_NEXT_COMMAND='the profile was never created; run make -f scripts/minikube-profiles/branch.mk branch-profile-start, then retry'
else
T2_NEXT_COMMAND='restore profile.env from a backup or stop and ask; the branch profile helper never regenerates metadata for an existing profile directory'
fi ;;
PROFILE_PORTS_MISSING)
T2_NEXT_COMMAND='restore the persisted profile-owned ports.env; never regenerate adopted ports' ;;
PROFILE_PORTS_INVALID)
T2_NEXT_COMMAND='repair the selected profile metadata through the branch profile helper; do not invent ports' ;;
*)
T2_NEXT_COMMAND='resolve the unique profile owned by this canonical worktree and branch, then retry' ;;
esac
t2_fail PROFILE_OWNERSHIP_MISMATCH "$PROFILE_OWNER_ERROR_MESSAGE"
fi
# CREATED_HEAD/SHA_SHORT is historical creation metadata only. Exact runtime
# freshness remains the marker's gitHead + worktreeId + clusterFingerprint.
}
t2_context_check() {
local context_name
context_name="$(kubectl --context="$T2_CONTEXT" config get-contexts -o name 2>/dev/null | awk -v target="$T2_CONTEXT" '$0 == target {print; exit}')"
if [ "$context_name" != "$T2_CONTEXT" ]; then
T2_NEXT_COMMAND='select the explicit Kubernetes context generated with the profile'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context is unavailable: $T2_CONTEXT"
fi
local endpoint
endpoint="$(t2_kc config view --raw --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null || true)"
if [ -z "$endpoint" ]; then
T2_NEXT_COMMAND='select a Kubernetes context with a resolvable local cluster endpoint'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is unavailable: $T2_CONTEXT"
fi
if ! kube_endpoint_is_local "$endpoint"; then
T2_NEXT_COMMAND='select the generated branch-owned Minikube context, not a remote cluster context'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context endpoint is not local: $(kube_endpoint_host "$endpoint")"
return 1
fi
}
t2_profile_context_identity_check() {
# A localhost API endpoint is necessary but not sufficient: kubeconfig can
# point at another local Minikube profile, and a broad RFC1918 allowlist can
# point at an unrelated LAN cluster. Bind the context to the exact profile
# identity and node address reported by the selected Minikube instance.
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
return 0
fi
local expected_ip nodes_json
expected_ip="$(t2_mk ip 2>/dev/null || true)"
if [ -z "$expected_ip" ]; then
T2_NEXT_COMMAND='start the verified branch-owned Minikube profile, then retry the gate'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Minikube did not report an IP for profile $T2_PROFILE"
return 1
fi
nodes_json="$(t2_kc get nodes -o json 2>/dev/null || true)"
if ! minikube_nodes_identify_profile "$nodes_json" "$T2_PROFILE" "$expected_ip"; then
T2_NEXT_COMMAND='select the kube-context generated for this exact branch-owned Minikube profile'
t2_fail DEVELOPMENT_SCOPE_REQUIRED "Kubernetes context does not identify Minikube profile $T2_PROFILE at $expected_ip"
return 1
fi
T2_CONTEXT_IDENTITY_VERIFIED=true
}
t2_profile_status() {
local status_text status_rc=0
# Keep stderr so a missing profile remains distinguishable from a deadline,
# transport, or permission failure. An empty result from a failed bounded
# probe is not evidence that bootstrap is safe.
status_text="$(t2_mk status 2>&1)" || status_rc=$?
if [ -n "$status_text" ] && minikube_profile_status_is_missing_or_stopped "$status_text"; then
if [[ "$status_text" == *Nonexistent* ||
"$status_text" == *'does not exist'* ||
"$status_text" == *'not found'* ]]; then
T2_PROFILE_STATUS=missing
T2_PLAN_REASON='profile is missing or uninitialized'
else
T2_PROFILE_STATUS=stopped
T2_PLAN_REASON='profile is missing or stopped'
fi
T2_BOOTSTRAP_REQUIRED=true
T2_PROFILE_HEALTHY=false
T2_PLAN_STATE=full-bootstrap
return 0
fi
if [ "$status_rc" -ne 0 ]; then
T2_PROFILE_STATUS=unhealthy
T2_PROFILE_HEALTHY=false
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"
t2_fail PROFILE_UNHEALTHY "Minikube status probe failed with exit $status_rc: $T2_PROFILE"
return 1
fi
if [ -z "$status_text" ] || ! minikube_profile_status_is_healthy "$status_text"; then
T2_PROFILE_STATUS=unhealthy
T2_PROFILE_HEALTHY=false
if [ -z "$status_text" ]; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-status"
t2_fail PROFILE_UNHEALTHY "Minikube status probe returned no output: $T2_PROFILE"
else
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-start"
t2_fail PROFILE_UNHEALTHY "Minikube profile is not healthy: $T2_PROFILE"
fi
return 1
fi
T2_PROFILE_STATUS=healthy
T2_PROFILE_HEALTHY=true
}
t2_marker_check() {
local marker_probe marker_status=0
# A missing or stopped profile has no Kubernetes context to query yet.
[ "$T2_BOOTSTRAP_REQUIRED" != true ] || return 0
# `--ignore-not-found` makes an absent marker an explicit empty-object
# result. Preserve every other kubectl failure: a timeout, RBAC denial, or
# transport error must never be reclassified as bootstrap permission.
marker_probe="$(t2_kc -n "$T2_CONTROL_NAMESPACE" get configmap "$T2_MARKER_NAME" \
-o json --ignore-not-found)" || marker_status=$?
if [ "$marker_status" -ne 0 ]; then
T2_NEXT_COMMAND='verify the explicit branch-owned Kubernetes context and retry the read-only preflight'
t2_fail PROFILE_UNHEALTHY \
"unable to read pre-gate marker (kubectl status ${marker_status}): ${marker_probe}"
return 1
fi
T2_MARKER_JSON="$marker_probe"
if [ -z "$T2_MARKER_JSON" ] || [ "$T2_MARKER_JSON" = '{}' ]; then
T2_BOOTSTRAP_REQUIRED=true
[ -n "$T2_PLAN_STATE" ] || T2_PLAN_STATE=full-bootstrap
[ -n "$T2_PLAN_REASON" ] || T2_PLAN_REASON='pre-gate marker is missing'
return 0
fi
local marker_values
if ! marker_values="$(python3 - "$T2_MARKER_JSON" "$T2_WORKTREE_ID" "$T2_HEAD" 2>&1 <<'PY'
import json
import sys
data = (json.loads(sys.argv[1]).get("data") or {})
for key in ("clusterFingerprint", "gitHead", "worktreeId"):
if not data.get(key):
raise SystemExit("missing:" + key)
if data.get("worktreeId") != sys.argv[2]:
raise SystemExit("ownership")
if data.get("gitHead") != sys.argv[3]:
raise SystemExit("head")
if not data.get("imagesGeneratedAt"):
raise SystemExit("missing:imagesGeneratedAt")
# \x1f, not a tab: a local build has an empty imageTag, and `read` collapses
# consecutive IFS whitespace, which would shift the next field into its slot.
print("\x1f".join([
data.get("clusterFingerprint", ""),
data.get("imageSource", ""),
data.get("imageTag", ""),
data.get("imagesGeneratedAt", ""),
]))
PY
)"; then
case "$marker_values" in
*ownership*) T2_NEXT_COMMAND='run pre-gate-sync for this worktree/profile, then retry'; t2_fail PROFILE_OWNERSHIP_MISMATCH 'pre-gate marker belongs to another worktree' ;;
*head*)
# Standalone preflight is fail-loud. The orchestrator planner
# (T2_PLAN_MODE=true) must keep going so it can select targeted-sync
# or full-reconcile and update the marker.
if [ "$T2_PLAN_MODE" = true ]; then
T2_MARKER_MATCHES_HEAD=false
T2_PLAN_REASON='pre-gate marker does not match current HEAD'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2"
t2_fail HEAD_MARKER_MISMATCH 'pre-gate marker does not match current HEAD'
;;
*) T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"; t2_fail BOOTSTRAP_REQUIRED 'pre-gate marker is incomplete' ;;
esac
# Explicit so a caller in an `||` context (errexit suspended) cannot fall
# through to the fingerprint check with the error text as marker values.
return 1
fi
local marker_fingerprint source_fingerprint
IFS=$'\x1f' read -r marker_fingerprint _ <<< "$marker_values"
# A marker for this HEAD is current only while the working tree still hashes
# to the digest it was stamped with; NP-08 recomputes the same digest.
if ! source_fingerprint="$(pre_gate_marker_cluster_fingerprint "$T2_PROJECT_DIR")"; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2"
t2_fail HEAD_MARKER_MISMATCH 'source fingerprint could not be computed'
return 1
fi
if [ "$source_fingerprint" != "$marker_fingerprint" ]; then
if [ "$T2_PLAN_MODE" = true ]; then
T2_MARKER_MATCHES_HEAD=false
T2_PLAN_REASON='source fingerprint changed since the pre-gate marker'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2"
t2_fail HEAD_MARKER_MISMATCH 'source fingerprint changed since the pre-gate marker'
return 1
fi
T2_MARKER_MATCHES_HEAD=true
IFS=$'\x1f' read -r T2_CLUSTER_FINGERPRINT T2_IMAGE_SOURCE T2_IMAGE_TAG \
T2_MARKER_IMAGES_GENERATED_AT <<< "$marker_values"
}
t2_image_check() {
if [ ! -f "$T2_IMAGE_MANIFEST" ]; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON='image manifest is absent'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail IMAGE_MANIFEST_MISMATCH "image manifest is missing: $T2_IMAGE_MANIFEST"
fi
local manifest_values
if ! manifest_values="$(python3 - "$T2_IMAGE_MANIFEST" <<'PY'
import json
import sys
from pathlib import Path
try:
payload = json.loads(Path(sys.argv[1]).read_text())
except (OSError, ValueError):
raise SystemExit("invalid")
source = payload.get("imageSource") or payload.get("source") or payload.get("mode") or ""
tag = payload.get("imageTag") or payload.get("tag") or ""
generated = payload.get("generated")
# Local builds are identified by the per-image digests in the manifest and
# intentionally have no registry tag. GHCR manifests still require a tag.
images = payload.get("images")
if not isinstance(generated, str) or not generated:
raise SystemExit("generated")
if source not in {"local", "ghcr"} or (source == "ghcr" and not tag):
raise SystemExit("missing")
if source == "local":
import re
if not isinstance(images, dict) or not images:
raise SystemExit("local-images")
digest = re.compile(r"^sha256:[0-9a-fA-F]{64}$")
if any(not isinstance(name, str) or not name or not isinstance(value, str) or not digest.fullmatch(value) for name, value in images.items()):
raise SystemExit("local-digests")
# \x1f for the same reason as the marker values: tag is empty for local builds.
print(source + "\x1f" + tag + "\x1f" + generated)
PY
)"; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON='image manifest is invalid or has no source/tag; bootstrap will replace it'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail IMAGE_MANIFEST_MISMATCH 'image manifest is invalid or incomplete for its image source'
fi
local manifest_source manifest_tag manifest_generated
IFS=$'\x1f' read -r manifest_source manifest_tag manifest_generated <<< "$manifest_values"
if { [ -n "$T2_IMAGE_SOURCE" ] && [ "$manifest_source" != "$T2_IMAGE_SOURCE" ]; } ||
{ [ -n "$T2_IMAGE_TAG" ] && [ "$manifest_tag" != "$T2_IMAGE_TAG" ]; }; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail IMAGE_MANIFEST_MISMATCH 'image manifest does not match the deployed marker'
fi
T2_IMAGE_SOURCE="$manifest_source"
T2_IMAGE_TAG="$manifest_tag"
T2_MANIFEST_GENERATED_AT="$manifest_generated"
if [ "$T2_MARKER_MATCHES_HEAD" = true ] &&
[ "$T2_MARKER_IMAGES_GENERATED_AT" != "$T2_MANIFEST_GENERATED_AT" ]; then
# A new image acquisition can keep the same Git HEAD while replacing the
# release/shadow digest on the cluster. Refuse to call that already-synced
# and let the orchestrator select a fresh targeted/full transition.
T2_MARKER_MATCHES_HEAD=false
T2_PLAN_REASON='image acquisition changed since the pre-gate marker'
fi
}
t2_get_name() {
printf '%s\t%s\n' "$(printf '%s' "$1" | cut -d/ -f1)" "$(printf '%s' "$1" | cut -d/ -f2-)"
}
t2_resource_checks() {
local item namespace name
for namespace in $T2_REQUIRED_NAMESPACES; do
if ! t2_kc get namespace "$namespace" >/dev/null 2>&1; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="namespace $namespace is not present"
continue
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail PROFILE_UNHEALTHY "required namespace is missing: $namespace"
fi
done
for item in $T2_REQUIRED_SERVICES; do
IFS=$'\t' read -r namespace name <<< "$(t2_get_name "$item")"
if ! t2_kc -n "$namespace" get service "$name" >/dev/null 2>&1; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="service $item is not present"
continue
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail PROFILE_UNHEALTHY "required Service is missing: $item"
fi
done
for item in $T2_REQUIRED_SECRETS; do
IFS=$'\t' read -r namespace name <<< "$(t2_get_name "$item")"
if ! t2_kc -n "$namespace" get secret "$name" >/dev/null 2>&1; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="Secret $item is not present"
continue
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail SECRET_MISSING "required Secret is missing: $namespace/$name"
fi
done
for item in $T2_REQUIRED_CONFIGMAPS; do
IFS=$'\t' read -r namespace name <<< "$(t2_get_name "$item")"
if ! t2_kc -n "$namespace" get configmap "$name" >/dev/null 2>&1; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="ConfigMap $item is not present"
continue
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail CONFIGMAP_MISSING "required ConfigMap is missing: $namespace/$name"
fi
done
}
t2_postgres_check() {
local namespace name pvc_json phase
IFS=$'\t' read -r namespace name <<< "$(t2_get_name "$T2_REQUIRED_PVC")"
pvc_json="$(t2_kc -n "$namespace" get pvc "$name" -o json 2>/dev/null || true)"
if [ -z "$pvc_json" ]; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="PVC $T2_REQUIRED_PVC is not present"
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail POSTGRES_NOT_READY "required PVC is missing: $T2_REQUIRED_PVC"
fi
phase="$(python3 - "$pvc_json" <<'PY'
import json
import sys
print((json.loads(sys.argv[1]).get("status") or {}).get("phase", ""))
PY
)"
if [ "$phase" != Bound ]; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="PVC $T2_REQUIRED_PVC is not Bound"
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail POSTGRES_NOT_READY "PostgreSQL PVC is not Bound: $T2_REQUIRED_PVC ($phase)"
fi
if ! t2_kc -n "$namespace" get service control-postgres >/dev/null 2>&1; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail POSTGRES_NOT_READY 'control-postgres Service is missing'
fi
if ! t2_kc -n "$namespace" rollout status deployment/control-postgres --timeout="${T2_TIMEOUT_SECONDS}s" >/dev/null 2>&1; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail POSTGRES_NOT_READY "control-postgres did not become Ready within $T2_TIMEOUT_SECONDS seconds"
fi
}
# Readiness alone does not prove that a journey restored its production images.
# The planner may reconcile; certifying preflight requires the live baseline.
t2_restored_runtime_check() {
[ "$T2_PLAN_MODE" != true ] && [ "$T2_BOOTSTRAP_REQUIRED" != true ] || return 0
local deployments="$1" service="$2" failure_code="$3" fixture_flags="$4" marker_file="$5" fixture_mounts="$6"
local pods inventory pod_names pod_name environment_check
if ! pods="$(t2_kc -n "$T2_CONTROL_NAMESPACE" get pods -l app="$service" -o json)"; then
t2_fail "$failure_code" "unable to observe running $service identity"
return 1
fi
if ! inventory="$(t2_mk image ls --format=json)"; then
t2_fail "$failure_code" 'unable to observe profile image inventory'
return 1
fi
if ! pod_names="$(python3 - "$T2_IMAGE_MANIFEST" "$T2_PROFILE" "$T2_CONTROL_NAMESPACE" "$deployments" "$pods" "$inventory" "$service" "$fixture_flags" "$fixture_mounts" <<'PY_RUNTIME'
import json
import re
import sys
from pathlib import Path
service = sys.argv[7]
fixture_flags = set(sys.argv[8].split())
fixture_mounts = set(sys.argv[9].split())
def require(condition):
if not condition:
raise ValueError("unproven production service")
def completed_job_pod(pod):
status = pod.get("status", {})
if status.get("phase") != "Succeeded":
return False
owners = pod.get("metadata", {}).get("ownerReferences") or []
if not any(owner.get("apiVersion") == "batch/v1" and owner.get("kind") == "Job" and
owner.get("controller") is True for owner in owners):
return False
containers = status.get("containerStatuses") or []
return bool(containers) and all(
set(container.get("state", {})) == {"terminated"} and
container["state"]["terminated"].get("exitCode") == 0
for container in containers + (status.get("initContainerStatuses") or [])
)
def clean(template):
require("evenfire.ai/codex-tools-fixture-run" not in (template.get("metadata", {}).get("annotations") or {}))
containers = [c for c in template["spec"]["containers"] if c.get("name") == service]
require(len(containers) == 1)
container = containers[0]
for entry in container.get("env", []):
require(entry.get("name") not in fixture_flags)
require(entry.get("name") != "NODE_ENV" or entry.get("value") != "test")
# A fresh fixture emptyDir has no marker file, so the storage itself must
# be absent from the restored workload.
for volume in template["spec"].get("volumes") or []:
require(volume.get("name") != "approved-tools-oauth-tmp")
for mount in container.get("volumeMounts") or []:
require(mount.get("name") != "approved-tools-oauth-tmp")
require(mount.get("mountPath") not in fixture_mounts)
return container
try:
manifest = json.loads(Path(sys.argv[1]).read_text())
require(manifest.get("profile") == sys.argv[2])
images = manifest["images"]
# Both acquisition modes record this production alias and Docker config ID.
baseline = images[f"clerum/{service}:test"]
require(isinstance(baseline, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", baseline))
inventory = json.loads(sys.argv[6])
require(isinstance(inventory, list))
observed = {}
for item in inventory:
identity = item.get("id", "")
if not identity.startswith("sha256:"):
identity = "sha256:" + identity
require(re.fullmatch(r"sha256:[0-9a-f]{64}", identity))
for ref in (item.get("repoTags") or []) + (item.get("repoDigests") or []):
require(ref not in observed or observed[ref] == identity)
observed[ref] = identity
deployments = [d for d in json.loads(sys.argv[4])["items"] if
d.get("metadata", {}).get("namespace") == sys.argv[3] and
d.get("metadata", {}).get("name") == service]
require(len(deployments) == 1)
deployment = deployments[0]
desired = deployment["spec"].get("replicas", 1)
require(isinstance(desired, int) and desired > 0)
target = clean(deployment["spec"]["template"])
require(images.get(target["image"]) == baseline)
ref = target["image"]
require(re.fullmatch(r"(?:clerum/|ghcr\.io/evenfire-ai/)" + re.escape(service) +
r"(?::[A-Za-z0-9._-]+|@sha256:[0-9a-f]{64})", ref))
canonical = "docker.io/" + ref if ref.startswith("clerum/") else ref
require(observed.get(canonical) == baseline)
pods = json.loads(sys.argv[5])["items"]
# Migration Jobs share the service label but completed Job pods do not
# serve requests. Exclude only proven successful Job completions before
# checking the required serving count; all other pods remain mandatory.
pods = [pod for pod in pods if not completed_job_pod(pod)]
require(len(pods) >= desired)
names = []
for pod in pods:
require(not pod["metadata"].get("deletionTimestamp"))
require(pod["metadata"].get("namespace") == sys.argv[3])
container = clean(pod)
require(container["image"] == target["image"])
statuses = [c for c in pod["status"]["containerStatuses"] if c.get("name") == service]
require(len(statuses) == 1 and statuses[0].get("ready") is True)
require(bool(statuses[0].get("state", {}).get("running")))
# Resolve a repository digest through the owned profile inventory;
# config IDs and repository manifest digests are different identities.
image_id = statuses[0].get("imageID", "")
if image_id.startswith("docker://"):
image_id = image_id[len("docker://"):]
elif image_id.startswith("docker-pullable://"):
image_id = image_id[len("docker-pullable://"):]
if "@sha256:" in image_id:
image_id = observed.get(image_id)
require(image_id == baseline)
name = pod["metadata"]["name"]
require(re.fullmatch(r"[a-z0-9][a-z0-9.-]*", name))
names.append(name)
print("\n".join(names))
except (OSError, ValueError, KeyError, TypeError, AttributeError):
raise SystemExit("production service identity, fixture markers, or baseline evidence did not match")
PY_RUNTIME
)"; then
T2_NEXT_COMMAND="restore production $service and retry MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2-runtime"
t2_fail "$failure_code" "live $service does not prove the recorded production baseline"
return 1
fi
# Read only fixture predicates, including envFrom/image-level variables;
# never emit the environment or secret-bearing objects as evidence.
while IFS= read -r pod_name; do
if ! environment_check="$(t2_kc -n "$T2_CONTROL_NAMESPACE" exec "$pod_name" -c "$service" -- node -e \
'const [flags, marker] = process.argv.slice(1); process.stdout.write(String(flags.split(" ").every(name => !(name in process.env)) && process.env.NODE_ENV !== "test" && (!marker || !require("node:fs").existsSync(marker))))' "$fixture_flags" "$marker_file")" ||
[ "$environment_check" != true ]; then
t2_fail "$failure_code" "running $service environment is unknown or retains fixture configuration"
return 1
fi
done <<< "$pod_names"
}
# Keep callers explicit about which production workload they certify.
t2_proxy_runtime_check() {
t2_restored_runtime_check "$1" codex-llm-proxy PROXY_RUNTIME_MISMATCH \
'CODEX_APPROVED_TOOLS_TEST_ONLY CODEX_APPROVED_TOOLS_MINIKUBE_PROFILE' '' ''
}
t2_control_api_runtime_check() {
t2_restored_runtime_check "$1" control-api CONTROL_API_RUNTIME_MISMATCH \
'EVENFIRE_APPROVED_TOOLS_OAUTH_FIXTURE APPROVED_TOOLS_RUN_ID' \
'/tmp/approved-tools-oauth-active.json' '/tmp'
}
# An optional image-input fixture must never survive into a runtime verdict.
# Runs without that opt-in image keep the existing probe set unchanged.
t2_image_capability_fixture_check() {
# The planner and bootstrap lanes never certify the live Host baseline, so
# they never probe it: an absent manifest or an unready Host there is a plan,
# not residue. The strict final preflight is T2_PLAN_MODE=false on a
# bootstrapped profile.
[ "$T2_PLAN_MODE" != true ] && [ "$T2_BOOTSTRAP_REQUIRED" != true ] || return 0
local acquired configuration restored verdict
if [ ! -f "$T2_IMAGE_MANIFEST" ]; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail IMAGE_MANIFEST_MISMATCH "image manifest is missing: $T2_IMAGE_MANIFEST"
return 1
fi
if ! acquired="$(python3 - "$T2_IMAGE_MANIFEST" 2>&1 <<'PY_IMAGE_FIXTURE'
import json, sys
with open(sys.argv[1]) as source:
images = json.load(source)["images"]
if not isinstance(images, dict):
raise SystemExit("images")
print("yes" if any(ref.removeprefix("docker.io/") == "clerum/image-capabilities-mcp-host:test" for ref in images) else "no")
PY_IMAGE_FIXTURE
)"; then
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
# The last line of a Python traceback names the error without echoing data.
t2_fail IMAGE_MANIFEST_MISMATCH "image manifest is invalid or incomplete: $T2_IMAGE_MANIFEST${acquired:+: ${acquired##*$'\n'}}"
return 1
fi
# stderr is merged into the value, so anything other than an exact verdict
# (for example an interpreter warning before "yes") must fail, not skip.
case "$acquired" in
yes) ;;
no) return 0 ;;
*)
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail IMAGE_MANIFEST_MISMATCH "image manifest check returned an unexpected verdict: ${acquired##*$'\n'}"
return 1
;;
esac
if ! configuration="$(t2_kc -n mcp-host get configmap mcp-host-config -o json)"; then
t2_fail HOST_RUNTIME_MISMATCH 'unable to observe the mcp-host image capability configuration'
return 1
fi
# The check prints a verdict so that a crash (malformed input) is reported as
# a crash instead of being read as residue.
if ! verdict="$(python3 - "$1" "$configuration" 2>&1 <<'PY_IMAGE_FIXTURE'
import json, sys
deployments, config = (json.loads(value) for value in sys.argv[1:])
flag = "evenfire.ai/image-capabilities-run"
def clean(data):
return not any(key.startswith("IMAGE_CAPABILITIES_") or key == "EVENFIRE_IMAGE_CAPABILITIES_FIXTURE" for key in data)
def residue():
print("residue")
raise SystemExit(0)
if flag in config.get("metadata", {}).get("annotations", {}) or not clean(config.get("data", {})) or config.get("data", {}).get("NODE_ENV") == "test":
residue()
for deployment in deployments["items"]:
if flag in deployment.get("metadata", {}).get("annotations", {}):
residue()
for container in deployment["spec"]["template"]["spec"]["containers"]:
if "image-capabilities-mcp-host" in container.get("image", ""):
residue()
for entry in container.get("env", []):
if not clean([entry["name"]]) or "image-capabilities-mcp-host" in entry.get("value", ""):
residue()
print("clean")
PY_IMAGE_FIXTURE
)"; then
t2_fail HOST_RUNTIME_MISMATCH "image capability residue check crashed: ${verdict##*$'\n'}"
return 1
fi
case "$verdict" in
clean) ;;
residue)
t2_fail HOST_RUNTIME_MISMATCH 'image capability fixture configuration remains installed'
return 1
;;
*)
t2_fail HOST_RUNTIME_MISMATCH "image capability residue check returned an unexpected verdict: ${verdict##*$'\n'}"
return 1
;;
esac
if ! restored="$(t2_kc -n mcp-host exec deployment/chatllm -- node -e \
'process.stdout.write(String(process.env.NODE_ENV !== "test" && !Object.keys(process.env).some(key => key.startsWith("IMAGE_CAPABILITIES_") || key === "EVENFIRE_IMAGE_CAPABILITIES_FIXTURE") && !require("node:fs").existsSync("/tmp/image-capabilities-evidence.json")))')"; then
t2_fail HOST_RUNTIME_MISMATCH 'unable to observe the running Host image capability environment'
return 1
fi
if [ "$restored" != true ]; then
t2_fail HOST_RUNTIME_MISMATCH 'image capability fixture remains in the running Host'
return 1
fi
}
t2_deployment_check() {
local deployment_json unready
deployment_json="$(t2_kc get deployments -A -o json 2>/dev/null || true)"
T2_DEPLOYMENT_JSON="$deployment_json"
if [ -z "$deployment_json" ]; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON='deployment readiness inventory is unavailable'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-setup-local"
t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is unavailable'
fi
if ! unready="$(python3 - "$deployment_json" "$T2_REQUIRED_DEPLOYMENTS" "$T2_REQUIRED_DEPLOYMENTS_INCLUDE_ADDITIONAL" 2>/dev/null <<'PY'
import json
import sys
payload = json.loads(sys.argv[1])
scope_refs = {}
for ref in sys.argv[2].split():
namespace, separator, name = ref.partition("/")
if not separator or not namespace or not name:
raise ValueError("invalid required deployment reference")
scope_refs[(namespace, name)] = None
include_additional = sys.argv[3] == "true"
inventory = {}
inventory_order = []
for item in payload.get("items", []):
metadata = item.get("metadata") or {}
identity = (metadata.get("namespace"), metadata.get("name"))
if identity not in inventory:
inventory_order.append(identity)
inventory[identity] = item
selected = []
if scope_refs:
for identity in scope_refs:
selected.append((identity, inventory.get(identity)))
if include_additional:
selected.extend((identity, inventory[identity]) for identity in inventory_order if identity not in scope_refs)
else:
selected = [(identity, inventory[identity]) for identity in inventory_order]
bad = []
for (namespace, name), item in selected:
if item is None:
bad.append(f"{namespace}/{name} missing")
continue
# The first loop leaves `metadata` bound to the last inventory item. Read
# metadata from the deployment being evaluated so each resource is
# compared with its own observed generation.
metadata = item.get("metadata") or {}
spec = item.get("spec") or {}
status = item.get("status") or {}
def integer(value, field):
if value in (None, ""):
return 0
if isinstance(value, bool):
raise ValueError(field)
return int(value)
desired = integer(spec.get("replicas"), "spec.replicas")
# Additional deployments may be intentionally suspended locally, but a
# deployment in the required scope is part of the certification contract
# and must not pass readiness at replicas=0.
if desired == 0:
if (namespace, name) in scope_refs:
bad.append(f"{namespace}/{name} scaled to zero")
continue
ready = integer(status.get("readyReplicas"), "status.readyReplicas")
available = integer(status.get("availableReplicas"), "status.availableReplicas")
updated = integer(status.get("updatedReplicas"), "status.updatedReplicas")
unavailable = integer(status.get("unavailableReplicas"), "status.unavailableReplicas")
generation = integer(metadata.get("generation"), "metadata.generation")
observed = integer(status.get("observedGeneration"), "status.observedGeneration")
if generation <= 0 or observed != generation or updated < desired or ready < desired or available < desired or unavailable != 0:
bad.append("%s/%s ready=%s/%s updated=%s observed=%s/%s unavailable=%s" % (
namespace, name, ready, desired,
updated, observed, generation, unavailable))
print("; ".join(bad))
PY
)"; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON='deployment readiness inventory is invalid'
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2"
t2_fail PROFILE_UNHEALTHY 'deployment readiness inventory is invalid'
fi
if [ -n "$unready" ]; then
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
T2_PLAN_REASON="one or more deployments are not Ready: $unready"
return 0
fi
if [ "$T2_PLAN_MODE" = true ]; then
# Orchestrator planner: a bootstrapped profile with an unready required
# deployment is repaired by a full reconcile inside the same run.
# Failing PROFILE_UNHEALTHY here would abort before any transition is
# selected and force manual repair scripts. The standalone preflight and
# the final exact-head check run with T2_PLAN_MODE=false and stay
# fail-loud below.
T2_UNREADY_DEPLOYMENTS="$unready"
return 0
fi
T2_NEXT_COMMAND="MINIKUBE_PROFILE=$T2_PROFILE make minikube-t2"
t2_fail PROFILE_UNHEALTHY "one or more deployments are not Ready: $unready"
fi
}
t2_port_forward_targets_context() {
local command_line="$1" index context_matches=false port_forward=false
local -a argv=()
read -r -a argv <<<"$command_line"
[ "${#argv[@]}" -gt 0 ] || return 1
[ "${argv[0]##*/}" = kubectl ] || return 1
for ((index = 1; index < ${#argv[@]}; index += 1)); do
case "${argv[$index]}" in
port-forward) port_forward=true ;;
"--context=$T2_CONTEXT") context_matches=true ;;
--context)
if [ $((index + 1)) -lt "${#argv[@]}" ] &&
[ "${argv[$((index + 1))]}" = "$T2_CONTEXT" ]; then
context_matches=true
fi
;;
esac
done
[ "$port_forward" = true ] && [ "$context_matches" = true ]
}
t2_port_forward_uses_profile_port() {
local command_line="$1" index arg local_port key port
local -a argv=()
read -r -a argv <<<"$command_line"
[ "${#argv[@]}" -gt 0 ] || return 1
for ((index = 1; index < ${#argv[@]}; index += 1)); do
[ "${argv[$index]}" = port-forward ] || continue
for ((index += 1; index < ${#argv[@]}; index += 1)); do
arg="${argv[$index]}"
if [[ "$arg" =~ ^([0-9]+):[0-9]+$ ]]; then
local_port="${BASH_REMATCH[1]}"
elif [[ "$arg" =~ ^[0-9]+$ ]]; then
local_port="$arg"
else
continue
fi
while IFS='=' read -r key port; do
[ "$port" = "$local_port" ] && return 0
done < <(awk -F= '/^(CONTROL_UI_PORT|PROFILE_UI_PORT|MCP_HOST_PORT|REGISTRY_API_PORT|CONTROL_API_PORT|EXTERNAL_REST_API_PORT|MEMBER_REGISTRATION_SERVICE_PORT|RPC_PROXY_PORT|WORKFLOW_APPROVAL_READER_PORT)=[0-9]+$/ { print $1 "=" $2 }' "$T2_PORTS_ENV" 2>/dev/null || true)
done
return 1
done
return 1
}
t2_pid_file_matches_process() {
local pid_file="$1" pid="$2"
pf_owner_read_record "$pid_file" || return 1
[ "$PF_OWNER_RECORD_FIRST_PID" = "$pid" ] || return 1
pf_owner_record_process_matches "$pid_file" "$T2_PROFILE" "$T2_CONTEXT" \
"$T2_PROJECT_DIR" "$PF_OWNER_RECORD_NAMESPACE" "$PF_OWNER_RECORD_SERVICE" \
"$PF_OWNER_RECORD_LOCAL_PORT" "$PF_OWNER_RECORD_REMOTE_PORT"
}
t2_process_check() {
local uid pid ppid rest command_line comm pid_file recorded_pid
local matching_records invalid_record
# A dead or reused registered forward is still an ownership failure even
# when ps(1) no longer lists a kubectl process. Scan the profile's durable
# records first so a stale pidfile cannot disappear from the verdict.
for pid_file in "$T2_PROFILE_ROOT/$T2_PROFILE"/pids/*.pid; do
[ -f "$pid_file" ] || continue
recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
if ! pf_owner_validate_pid "$recorded_pid" ||
! t2_pid_file_matches_process "$pid_file" "$recorded_pid"; then
T2_NEXT_COMMAND='repair the exact profile-owned port-forward record before rerunning T2'
t2_fail PORT_FORWARD_CONFLICT "registered port-forward ownership is stale or invalid: $pid_file"
return 1
fi
done
# Only real kubectl port-forward processes. A wrapper whose argv merely
# mentions those words is not a port-forward (rejected by exact argv0/comm).
# Default IFS so UID/PID/PPID split. `IFS=` left pid empty and skipped every line.
# awk is a loose pre-filter: kubectl as argv0/path token AND a later
# standalone port-forward token. Flags may sit between those tokens.
while read -r uid pid ppid rest; do
[ -n "$pid" ] || continue
command_line="$(pf_owner_process_command "$pid" 2>/dev/null || true)"
[ -n "$command_line" ] || continue
comm="$(ps -p "$pid" -o comm= 2>/dev/null || true)"
case "${comm##*/}" in
kubectl) ;;
*) continue ;;
esac
# Context-less or differently pinned developer forwards are not
# attributable to this profile. A forward on one of this profile's
# persisted local ports is still a conflict, even without a usable
# context, because it could make the user-facing health probe hit another
# cluster. Unrelated profile ports remain eligible for parallel lanes.
if ! t2_port_forward_targets_context "$command_line"; then
if t2_port_forward_uses_profile_port "$command_line"; then
T2_NEXT_COMMAND='stop the contextless or foreign port-forward occupying a persisted profile port'
t2_fail PORT_FORWARD_CONFLICT 'a contextless or foreign port-forward occupies a port owned by this profile'
return 1
fi
continue
fi
matching_records=0
invalid_record=false
for pid_file in "$T2_PROFILE_ROOT/$T2_PROFILE"/pids/*.pid; do
[ -f "$pid_file" ] || continue
recorded_pid="$(sed -n '1p' "$pid_file" 2>/dev/null || true)"
[ "$recorded_pid" = "$pid" ] || continue
if t2_pid_file_matches_process "$pid_file" "$pid"; then
matching_records=$((matching_records + 1))
else
invalid_record=true
fi
done
if [ "$invalid_record" = true ] || [ "$matching_records" -ne 1 ]; then
T2_NEXT_COMMAND='stop the unrelated profile port-forward or select the owner worktree; do not share it'
t2_fail PORT_FORWARD_CONFLICT 'a port-forward for this profile lacks one exact ownership record'
return 1
fi
done < <(ps -ef 2>/dev/null | awk '/([^[:space:]]*\/)?kubectl([[:space:]]|$)/ && /[[:space:]]port-forward([[:space:]]|$)/ {print}' || true)
}
t2_classify_transition() {
if [ "$T2_BOOTSTRAP_REQUIRED" = true ]; then
T2_PLAN_STATE=full-bootstrap
[ -n "$T2_PLAN_REASON" ] || T2_PLAN_REASON='profile is not bootstrapped'
return 0
fi
if [ -n "$T2_UNREADY_DEPLOYMENTS" ]; then
# Recorded by t2_deployment_check in planner mode only. Reconcile the
# running profile in place; an unready deployment must never certify as
# already-synced nor stop the orchestrator before a transition exists.
T2_PLAN_STATE=full-reconcile
T2_PLAN_REASON="deployment not Ready: $T2_UNREADY_DEPLOYMENTS"
return 0
fi
if [ "$T2_MARKER_MATCHES_HEAD" = true ]; then
T2_PLAN_STATE=already-synced
T2_PLAN_REASON='pre-gate marker already matches HEAD'
return 0
fi
local changed infra=false path
changed="$(git -C "$T2_PROJECT_DIR" diff --name-only "$T2_ORIGIN_DEV...$T2_HEAD")"
if [ -z "$changed" ]; then
T2_PLAN_STATE=targeted-sync
T2_PLAN_REASON='no source changes after exact marker'
return 0
fi
while IFS= read -r path; do
[ -z "$path" ] && continue
case "$path" in
deploy/*|charts/*)
infra=true
break
;;
esac
done <<< "$changed"
if [ "$infra" = true ]; then
T2_PLAN_STATE=full-reconcile
T2_PLAN_REASON='infrastructure, manifest, CRD, policy, storage, or overlay input changed'
else
T2_PLAN_STATE=targeted-sync
T2_PLAN_REASON='changes are limited to service, package, harness, or documentation inputs'
fi
}
t2_lock_owner_value() {
local key="$1"
[ -f "$T2_LOCK_DIR/owner.env" ] || return 1
sed -n "s/^$key=//p" "$T2_LOCK_DIR/owner.env" | head -1
}
t2_lock_key() {
printf '%s\0%s\0%s\0%s\0%s' \
"$T2_PROJECT_DIR" "$T2_BRANCH" "$T2_HEAD" "$T2_PROFILE" "$T2_CONTEXT" |
shasum | awk '{print $1}'
}
t2_lock_profile_id_check() {
if [[ ! "$T2_PROFILE" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
T2_NEXT_COMMAND='use the generated profile name without path separators or shell metacharacters'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'profile lock identifier is not a safe local profile name'
return 1
fi
}
t2_lock_process_matches() {
local pid="$1" expected_start actual_start state
[[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1
kill -0 "$pid" >/dev/null 2>&1 || return 1
state="$(ps -p "$pid" -o state= 2>/dev/null | tr -d '[:space:]' || true)"
[ -z "$state" ] || [[ "$state" != Z* ]] || return 1
expected_start="$(t2_lock_owner_value PROCESS_START || true)"
[ -n "$expected_start" ] || return 1
# Some macOS/sandboxed ps implementations hide lstart/state. The opaque
# token and live PID still fence the child in that environment; use the
# stronger start-time comparison whenever the platform exposes it.
[ "$expected_start" = unavailable ] && return 0
actual_start="$(ps -p "$pid" -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
[ -z "$actual_start" ] || [ "$actual_start" = "$expected_start" ]
}
t2_lock_acquire() {
local process_start reclaim_dir=""
t2_lock_profile_id_check || return 1
if [ -z "${T2_ORIGIN_DEV:-}" ]; then
T2_NEXT_COMMAND='resolve repository metadata (t2_repo_metadata) before acquiring the profile lock'
t2_fail DEVELOPMENT_SCOPE_REQUIRED 'the lease owner has no origin/dev to pin'
return 1
fi
mkdir -p "$T2_LOCK_ROOT"
T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
T2_LOCK_KEY="$(t2_lock_key)"
if [ -e "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR" ]; then
if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ]; then
T2_NEXT_COMMAND='remove the unsafe profile-lock path manually after verifying the profile owner'
t2_fail PROFILE_BUSY "profile lock path is not a directory: $T2_LOCK_DIR"
return 1
fi
local existing_pid existing_token existing_start
existing_pid="$(t2_lock_owner_value PID || true)"
existing_token="$(t2_lock_owner_value TOKEN || true)"
existing_start="$(t2_lock_owner_value PROCESS_START || true)"
if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; then
T2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
t2_fail PROFILE_BUSY "profile $T2_PROFILE has an orphaned lock without a valid owner identity"
return 1
fi
if t2_lock_process_matches "$existing_pid"; then
T2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"
t2_fail PROFILE_BUSY "profile $T2_PROFILE is locked by PID $existing_pid"
return 1
fi
# Reclaim ownership is itself acquired atomically beside the stale lock.
# The claim must not live inside T2_LOCK_DIR: the winner removes that
# directory before creating the replacement lock, and a losing reclaimer
# could otherwise recreate `.reclaim` inside the winner's live lock.
# If a reclaimer is killed while holding this sibling claim, the marker
# deliberately remains and requires the documented orphan-lock recovery
# rather than permitting an unsafe retry.
reclaim_dir="$T2_LOCK_ROOT/$T2_PROFILE.reclaim"
if ! mkdir "$reclaim_dir" 2>/dev/null; then
T2_NEXT_COMMAND='wait for the stale-lock reclaimer to finish; if it died, follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
t2_fail PROFILE_BUSY "profile $T2_PROFILE stale lock is already being reclaimed"
return 1
fi
# Revalidate after winning the claim. A changed or live owner means the
# original stale observation is no longer authoritative.
existing_pid="$(t2_lock_owner_value PID || true)"
existing_token="$(t2_lock_owner_value TOKEN || true)"
existing_start="$(t2_lock_owner_value PROCESS_START || true)"
if [[ ! "$existing_pid" =~ ^[0-9]+$ ]] || [ -z "$existing_token" ] || [ -z "$existing_start" ]; then
T2_NEXT_COMMAND='verify no T2 process owns the lock, then follow the orphaned-lock recovery steps in docs/testing/minikube-t2-runbook.md'
t2_fail PROFILE_BUSY "profile $T2_PROFILE owner changed while its stale lock was being reclaimed"
return 1
fi
if t2_lock_process_matches "$existing_pid"; then
rmdir "$reclaim_dir" 2>/dev/null || true
T2_NEXT_COMMAND="wait for PID $existing_pid to finish, then retry the same profile"
t2_fail PROFILE_BUSY "profile $T2_PROFILE became live while its stale lock was being reclaimed"
return 1
fi
rm -rf -- "$T2_LOCK_DIR"
fi
mkdir "$T2_LOCK_DIR" 2>/dev/null || {
T2_NEXT_COMMAND='retry after the profile lock is released'
t2_fail PROFILE_BUSY "unable to acquire the profile lock: $T2_PROFILE"
return 1
}
umask 077
T2_LOCK_TOKEN="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"
process_start="$(ps -p $$ -o lstart= 2>/dev/null | sed 's/^ *//' || true)"
[ -n "$process_start" ] || process_start=unavailable
cat >"$T2_LOCK_DIR/owner.env" <<EOF
REPOSITORY=$T2_PROJECT_DIR
BRANCH=$T2_BRANCH
HEAD=$T2_HEAD
PROFILE=$T2_PROFILE
CONTEXT=$T2_CONTEXT
WORKTREE_ID=$T2_WORKTREE_ID
LOCK_KEY=$T2_LOCK_KEY
ORIGIN_DEV=$T2_ORIGIN_DEV
TOKEN=$T2_LOCK_TOKEN
PID=$$
PROCESS_START=$process_start
STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
EOF
if [ -n "$reclaim_dir" ]; then
rmdir "$reclaim_dir" 2>/dev/null || true
fi
# Children of this lease validate against the origin/dev read here, not
# against the shared remote-tracking ref (see t2_resolve_origin_dev).
export T2_PINNED_ORIGIN_DEV="$T2_ORIGIN_DEV"
T2_LOCK_HELD=true
T2_LOCK_RELEASED=false
}
t2_lock_validate_inherited() {
local expected_pid expected_token owner_key owner_value
t2_lock_profile_id_check || return 1
T2_LOCK_DIR="$T2_LOCK_ROOT/$T2_PROFILE.lock"
T2_LOCK_KEY="$(t2_lock_key)"
if [ -L "$T2_LOCK_DIR" ] || [ ! -d "$T2_LOCK_DIR" ] || [ -L "$T2_LOCK_DIR/owner.env" ]; then
T2_NEXT_COMMAND='run the parent T2 command with its profile lock token, then retry this child operation'
t2_fail PROFILE_LOCK_REQUIRED "inherited profile lock is missing for $T2_PROFILE"
return 1
fi
expected_token="$T2_LOCK_TOKEN"
owner_value="$(t2_lock_owner_value TOKEN || true)"
if [ -z "$expected_token" ] || [ "$owner_value" != "$expected_token" ]; then
T2_NEXT_COMMAND='pass the opaque T2_LOCK_TOKEN from the owning T2 process; do not bypass the profile lock'
t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock token is missing or does not match'
return 1
fi
for owner_key in REPOSITORY BRANCH HEAD PROFILE CONTEXT WORKTREE_ID LOCK_KEY ORIGIN_DEV; do
case "$owner_key" in
REPOSITORY) expected_pid="$T2_PROJECT_DIR" ;;
BRANCH) expected_pid="$T2_BRANCH" ;;
HEAD) expected_pid="$T2_HEAD" ;;
PROFILE) expected_pid="$T2_PROFILE" ;;
CONTEXT) expected_pid="$T2_CONTEXT" ;;
WORKTREE_ID) expected_pid="$T2_WORKTREE_ID" ;;
LOCK_KEY) expected_pid="$T2_LOCK_KEY" ;;
ORIGIN_DEV) expected_pid="${T2_PINNED_ORIGIN_DEV:-}" ;;
esac
# An empty pin never matches, even against a lock written before the
# owner recorded ORIGIN_DEV.
if [ -z "$expected_pid" ] && [ "$owner_key" = ORIGIN_DEV ] ||
[ "$(t2_lock_owner_value "$owner_key" || true)" != "$expected_pid" ]; then
T2_NEXT_COMMAND='re-run the operation from the worktree/profile that owns the T2 lock'
t2_fail PROFILE_OWNERSHIP_MISMATCH "profile lock owner does not match $owner_key"
return 1
fi
done
expected_pid="$(t2_lock_owner_value PID || true)"
if ! t2_lock_process_matches "$expected_pid"; then
T2_NEXT_COMMAND='restart the parent T2 operation so it can acquire a live profile lock'
t2_fail PROFILE_LOCK_REQUIRED 'inherited profile lock owner is not a live process'
return 1
fi
T2_LOCK_HELD=false
T2_LOCK_RELEASED=false
}
t2_mutation_lock() {
if [ "$T2_SKIP_LOCK" = true ]; then
t2_lock_validate_inherited
else
t2_lock_acquire
fi
# A disruptive local gate can outlive its process (SIGKILL/API outage). The
# profile lease is transient; its durable recovery obligation is not. Never
# certify or reconcile over an unfinished DNS intervention.
if ! node "$T2_SCRIPT_DIR/../e2e/_lib/wrc-egress-lifecycle.cjs" "$T2_PROJECT_DIR" "$T2_PROFILE"; then
t2_lock_release 0 || true
T2_NEXT_COMMAND='run the owned test-e2e-wrc-egress-recover target, then retry the original operation'
t2_fail WRC_EGRESS_RECOVERY_REQUIRED 'an unfinished WRC fault-injection journal prevents profile mutation or certification'
return 1
fi
}
t2_lock_release() {
local incoming_status=$? status owner_pid owner_token cleanup_status=0
if [ "$#" -gt 0 ]; then status="$1"; else status="$incoming_status"; fi
if [ "$T2_LOCK_RELEASED" = true ]; then
return "$status"
fi
T2_LOCK_RELEASED=true
if [ "$T2_LOCK_HELD" = true ] && [ -n "$T2_LOCK_DIR" ]; then
owner_pid="$(t2_lock_owner_value PID || true)"
owner_token="$(t2_lock_owner_value TOKEN || true)"
if [ "$owner_pid" = "$$" ] && [ -n "$T2_LOCK_TOKEN" ] && [ "$owner_token" = "$T2_LOCK_TOKEN" ]; then
rm -rf -- "$T2_LOCK_DIR" || cleanup_status=1
fi
fi
T2_LOCK_HELD=false
if [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ] && [ -n "$T2_ERROR_CODE" ]; then
t2_evidence_write failure FAIL "$T2_ERROR_CODE"
fi
if [ "$cleanup_status" -ne 0 ] && [ -n "$T2_EVIDENCE_FILE" ] && [ -f "$T2_EVIDENCE_FILE" ]; then
# A PASS written before EXIT is not a valid attestation if the profile lock
# could not be released. Invalidate it explicitly and keep the detail
# secret-free so a later runtime-only lane cannot reuse unsafe evidence.
t2_evidence_write lock-cleanup INVALIDATED 'profile lock cleanup failed; prior attestation is invalid' || true
fi
if [ "$status" -eq 0 ] && [ "$cleanup_status" -ne 0 ]; then
status="$cleanup_status"
fi
return "$status"
}
t2_evidence_init() {
local stamp
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
T2_RUN_ID="$(python3 -c 'import uuid; print(uuid.uuid4().hex)')"
T2_EVIDENCE_DIR="$T2_EVIDENCE_ROOT/$stamp-$(printf '%s' "$T2_HEAD" | cut -c1-12)-$T2_RUN_ID"
T2_EVIDENCE_FILE="$T2_EVIDENCE_DIR/evidence.json"
umask 077
mkdir -p "$T2_EVIDENCE_DIR/logs"
t2_evidence_write preflight RUNNING ''
}
t2_prior_targeted_health_pending() {
local pending
pending="$(
CERTIFICATION_ROOT="$T2_EVIDENCE_ROOT" \
EXPECTED_REPOSITORY="$T2_PROJECT_DIR" EXPECTED_BRANCH="$T2_BRANCH" \
EXPECTED_HEAD="$T2_HEAD" EXPECTED_WORKTREE_ID="$T2_WORKTREE_ID" \
EXPECTED_PROFILE="$T2_PROFILE" EXPECTED_CONTEXT="$T2_CONTEXT" \
EXPECTED_GATE_ID="$T2_GATE_ID" python3 - <<'PY'
import json
import os
from pathlib import Path
root = Path(os.environ["CERTIFICATION_ROOT"])
expected = {
"repository": os.environ["EXPECTED_REPOSITORY"],
"branch": os.environ["EXPECTED_BRANCH"],
"head": os.environ["EXPECTED_HEAD"],
"worktreeId": os.environ["EXPECTED_WORKTREE_ID"],
"profile": os.environ["EXPECTED_PROFILE"],
"context": os.environ["EXPECTED_CONTEXT"],
"gateId": os.environ["EXPECTED_GATE_ID"],
}
candidates = []
for candidate in root.glob("*/evidence.json"):
try:
data = json.loads(candidate.read_text())
stamp = candidate.stat().st_mtime_ns
except (OSError, ValueError):
continue
if data.get("certificationVersion") != 1 or data.get("evidenceKind") != "certification":
continue
if all(data.get(key, "") == value for key, value in expected.items()):
candidates.append((stamp, str(candidate), data))
if not candidates:
print("false")
else:
_, _, latest = max(candidates)
print("true" if latest.get("targetedHealthPending") is True else "false")
PY
)" || return 1
printf '%s' "$pending"
}
t2_evidence_write() {
local phase="$1" status="$2" detail="$3" file
[ -n "$T2_EVIDENCE_FILE" ] || return 0
file="$T2_EVIDENCE_FILE.tmp"
PHASE="$phase" STATUS="$status" DETAIL="$detail" \
T2_PROJECT_DIR="$T2_PROJECT_DIR" T2_BRANCH="$T2_BRANCH" T2_HEAD="$T2_HEAD" \
T2_ORIGIN_DEV="$T2_ORIGIN_DEV" T2_MERGE_BASE="$T2_MERGE_BASE" \
T2_WORKTREE_ID="$T2_WORKTREE_ID" T2_RUN_ID="$T2_RUN_ID" T2_GATE_ID="$T2_GATE_ID" \
T2_PROFILE="$T2_PROFILE" T2_CONTEXT="$T2_CONTEXT" T2_CLUSTER_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \
T2_HEALTHCHECK_PENDING="$T2_HEALTHCHECK_PENDING" \
T2_PROFILE_STATUS="$T2_PROFILE_STATUS" T2_PROFILE_HEALTHY="$T2_PROFILE_HEALTHY" \
T2_EVIDENCE_DIR="$T2_EVIDENCE_DIR" T2_EVIDENCE_KIND="$T2_EVIDENCE_KIND" \
T2_IMAGE_MANIFEST="$T2_IMAGE_MANIFEST" T2_IMAGE_SOURCE="$T2_IMAGE_SOURCE" T2_IMAGE_TAG="$T2_IMAGE_TAG" \
python3 - "$file" "$T2_EVIDENCE_FILE" <<'PY'
import json
import os
from datetime import datetime, timedelta, timezone
from pathlib import Path
path = Path(os.sys.argv[1])
prior_path = Path(os.sys.argv[2])
prior = {}
if prior_path.exists():
try: prior = json.loads(prior_path.read_text())
except ValueError: prior = {}
prior.setdefault("evidenceVersion", 1)
evidence_kind = os.environ.get("T2_EVIDENCE_KIND", "certification")
if evidence_kind not in {"certification", "planner"}:
raise SystemExit(f"unsupported T2 evidence kind: {evidence_kind}")
prior["evidenceKind"] = evidence_kind
if evidence_kind == "certification":
prior["certificationVersion"] = 1
else:
prior.pop("certificationVersion", None)
prior.setdefault("runId", os.environ.get("T2_RUN_ID", ""))
now = datetime.now(timezone.utc)
if evidence_kind == "certification":
prior.setdefault("attestationStartedAt", now.isoformat().replace("+00:00", "Z"))
prior.setdefault(
"attestationExpiresAt",
(now + timedelta(hours=24)).isoformat().replace("+00:00", "Z"),
)
else:
prior.setdefault("plannerStartedAt", now.isoformat().replace("+00:00", "Z"))
def redact(value):
import re
value = str(value or "")
value = re.sub(r"postgres(?:ql)?://[^\s\"'<>]+", "<postgres-dsn-redacted>", value)
value = re.sub(r"(?i)(bearer\s+)[^\s]+", r"\1<token-redacted>", value)
value = re.sub(r"-----BEGIN [^-]+-----.*?-----END [^-]+-----", "<private-key-redacted>", value, flags=re.S)
value = re.sub(r"(?i)(?:password|token|secret|api[_-]?key|private[_-]?key)\s*[:=]\s*[^\s,;]+", "<secret-assignment-redacted>", value)
return value
detail = redact(os.environ.get("DETAIL", ""))
prior.update({
"repository": os.environ.get("T2_PROJECT_DIR", ""),
"branch": os.environ.get("T2_BRANCH", ""),
"head": os.environ.get("T2_HEAD", ""),
"originDev": os.environ.get("T2_ORIGIN_DEV", ""),
"mergeBase": os.environ.get("T2_MERGE_BASE", ""),
"worktreeId": os.environ.get("T2_WORKTREE_ID", ""),
"runId": os.environ.get("T2_RUN_ID", ""),
"gateId": os.environ.get("T2_GATE_ID", ""),
"worktree": os.environ.get("T2_PROJECT_DIR", ""),
"profile": os.environ.get("T2_PROFILE", ""),
"context": os.environ.get("T2_CONTEXT", ""),
"profileStatus": os.environ.get("T2_PROFILE_STATUS", ""),
"profileHealthy": os.environ.get("T2_PROFILE_HEALTHY", "false"),
"clusterFingerprintRef": os.environ.get("T2_CLUSTER_FINGERPRINT", ""),
"imageManifestRef": os.environ.get("T2_IMAGE_MANIFEST", ""),
"localLogDirectory": os.path.join(os.environ.get("T2_EVIDENCE_DIR", ""), "logs"),
"imageSource": os.environ.get("T2_IMAGE_SOURCE", ""),
"imageTag": os.environ.get("T2_IMAGE_TAG", ""),
"targetedHealthPending": os.environ.get("T2_HEALTHCHECK_PENDING", "false") == "true",
"phase": os.environ.get("PHASE", ""),
"status": os.environ.get("STATUS", ""),
"detail": detail,
})
phase = os.environ.get("PHASE", "")
status = os.environ.get("STATUS", "")
if evidence_kind == "planner":
planner_status = prior.get("plannerStatus", "IN_PROGRESS")
if status in {"FAIL", "INVALIDATED"}:
planner_status = "FAIL"
elif phase == "preflight" and status == "PASS":
planner_status = "PASS"
prior["plannerStatus"] = planner_status
prior["attestationStatus"] = "NOT_APPLICABLE"
prior["laneAttestationStatus"] = "NOT_APPLICABLE"
else:
attestation = prior.get("attestationStatus", "IN_PROGRESS")
if status in {"FAIL", "INVALIDATED"}:
attestation = "INVALIDATED"
elif phase == "complete" and status == "PASS":
attestation = "PASS"
prior["attestationStatus"] = attestation
lane_attestation = prior.get("laneAttestationStatus", "IN_PROGRESS")
if phase == "lanes" and status == "PASS":
lane_attestation = "PASS"
elif phase == "lock-cleanup" and status == "INVALIDATED":
lane_attestation = "INVALIDATED"
prior["laneAttestationStatus"] = lane_attestation
phase_entry = {
"name": os.environ.get("PHASE", ""),
"status": os.environ.get("STATUS", ""),
"detail": detail,
"timestamp": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat().replace("+00:00", "Z"),
}
import re
duration_match = re.search(r"(?:^|[;\s])duration(?:Seconds)?=(\d+(?:\.\d+)?)(ms|s)?(?:$|[;\s])", detail)
if duration_match:
duration = float(duration_match.group(1))
if duration_match.group(2) == "ms":
duration /= 1000
phase_entry["durationSeconds"] = duration
prior.setdefault("phases", []).append(phase_entry)
path.write_text(json.dumps(prior, indent=2, sort_keys=True) + "\n")
PY
mv -- "$file" "$T2_EVIDENCE_FILE"
}
t2_certification_validate_prior_lanes() {
local result
if [ -z "$T2_CLUSTER_FINGERPRINT" ]; then
T2_NEXT_COMMAND='run the exact-head preflight and regenerate a profile marker with a cluster fingerprint'
t2_fail CERTIFICATION_REQUIRED 'current exact-head marker has no cluster fingerprint'
fi
if ! result="$(
CERTIFICATION_ROOT="$T2_EVIDENCE_ROOT" \
EXPECTED_REPOSITORY="$T2_PROJECT_DIR" EXPECTED_BRANCH="$T2_BRANCH" \
EXPECTED_HEAD="$T2_HEAD" EXPECTED_ORIGIN_DEV="$T2_ORIGIN_DEV" \
EXPECTED_WORKTREE_ID="$T2_WORKTREE_ID" EXPECTED_PROFILE="$T2_PROFILE" \
EXPECTED_CONTEXT="$T2_CONTEXT" EXPECTED_FINGERPRINT="$T2_CLUSTER_FINGERPRINT" \
EXPECTED_GATE_ID="$T2_GATE_ID" python3 - <<'PY'
import json
import os
from datetime import datetime, timezone
from pathlib import Path
root = Path(os.environ["CERTIFICATION_ROOT"])
expected = {
"repository": os.environ["EXPECTED_REPOSITORY"],
"branch": os.environ["EXPECTED_BRANCH"],
"head": os.environ["EXPECTED_HEAD"],
"originDev": os.environ["EXPECTED_ORIGIN_DEV"],
"worktreeId": os.environ["EXPECTED_WORKTREE_ID"],
"profile": os.environ["EXPECTED_PROFILE"],
"context": os.environ["EXPECTED_CONTEXT"],
"clusterFingerprintRef": os.environ["EXPECTED_FINGERPRINT"],
"gateId": os.environ["EXPECTED_GATE_ID"],
}
for candidate in sorted(root.glob("*/evidence.json"), reverse=True):
try:
data = json.loads(candidate.read_text())
except (OSError, ValueError):
continue
if data.get("certificationVersion") != 1:
continue
if data.get("attestationStatus") != "PASS" and data.get("laneAttestationStatus") != "PASS":
continue
if not data.get("runId"):
continue
try:
expiry = datetime.fromisoformat(str(data.get("attestationExpiresAt", "")).replace("Z", "+00:00"))
if expiry <= datetime.now(timezone.utc):
continue
except (TypeError, ValueError):
continue
if any(data.get(key, "") != value for key, value in expected.items()):
continue
latest = {}
for phase in data.get("phases", []):
name = phase.get("name")
if name in {"T0", "T0_ATTESTED"}:
latest["T0"] = phase.get("status")
if name in {"T1", "T1_ATTESTED"}:
latest["T1"] = phase.get("status")
if latest.get("T0") == "PASS" and latest.get("T1") == "PASS":
print("\t".join([str(candidate), "PASS", "PASS"]))
raise SystemExit(0)
raise SystemExit("no valid exact-head T0/T1 attestation was found")
PY
)"; then
T2_NEXT_COMMAND='run make minikube-t2 with T0=true and T1=true to create exact-head lane attestations'
t2_fail CERTIFICATION_REQUIRED "$result"
fi
IFS=$'\t' read -r T2_PRIOR_ATTESTATION T2_PRIOR_T0_ATTESTATION T2_PRIOR_T1_ATTESTATION <<< "$result"
T2_T0_CERTIFIED=true
T2_T1_CERTIFIED=true
t2_evidence_write T0_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"
t2_evidence_write T1_ATTESTED PASS "reused exact-head attestation from $T2_PRIOR_ATTESTATION"
}
t2_write_plan() {
local plan_file="$1"
[ -n "$plan_file" ] || return 0
local tmp_file="$plan_file.tmp.$$"
PLAN_STATE="$T2_PLAN_STATE" PLAN_REASON="$T2_PLAN_REASON" \
PLAN_PROFILE_STATUS="$T2_PROFILE_STATUS" PLAN_PROFILE_HEALTHY="$T2_PROFILE_HEALTHY" \
python3 - "$tmp_file" <<'PY'
import json
import os
from pathlib import Path
Path(os.sys.argv[1]).write_text(json.dumps({
"state": os.environ.get("PLAN_STATE", ""),
"reason": os.environ.get("PLAN_REASON", ""),
"profileStatus": os.environ.get("PLAN_PROFILE_STATUS", ""),
"profileHealthy": os.environ.get("PLAN_PROFILE_HEALTHY", "false"),
}, sort_keys=True) + "\n")
PY
mv -- "$tmp_file" "$plan_file"
}