Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# gfs E2E runtime gate (plan P5-S05) — single-node RWO.
#
# Flow (fail-loud at every step; never skip, never mock-to-green):
# 1. GlobalFileSystem reaches Ready (gfsc writer+reader, PVC, Service).
# 2. Operator bootstrap seeded the root directories.
# 3. A folder owner delegates Layer-2 write (no operator) — grant accepted.
# 4. An agent (host token) writes to a scratch folder; the write is audited,
# and its DESTRUCTIVE delete is default-denied (403).
# 5. The written resource resolves by its gfs:// URI.
# 6. The resource downloads by its gfs:// URI (bytes match).
#
# This is NOT a CI unit test — it needs a live cluster. Run ONLY after a deploy
# sync on a branch profile. Any not-Ready / unreachable / unexpected status FAILS
# the gate with the concrete reason (pod/service/HTTP state), never a silent skip.
#
# Auth surfaces exercised (real, no mocks):
# - operator session → E2E_ADMIN_TOKEN when provided, otherwise seeded admin login
# (POST /gfs/token, PUT /gfs/grants, by-path)
# - provisioner → InternalControl JWT (iss=hcc) signed IN the control-api
# pod, minting a host gfs token + seeding roots
# - gfsc broker → the gfs token, called inside the gfsc pod (localhost)
#
# Usage:
# CONTEXT=clerum-codex-gfs-<sha> scripts/e2e/gfs-e2e-gate.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=scripts/e2e/load-dotenv.sh
source "${SCRIPT_DIR}/load-dotenv.sh"
# shellcheck source=scripts/e2e/admin-credentials.sh
source "${SCRIPT_DIR}/admin-credentials.sh"
# CONTEXT is mandatory and explicit — never rely on the current kube-context.
CONTEXT="${CONTEXT:?set CONTEXT to an allowed branch/clerum-test profile (never a prod context)}"
GFS_NS="${GFS_NS:-gfs}"
CONTROL_NS="${CONTROL_NS:-control-plane}"
GFS_NAME="${GFS_NAME:-gfs}"
GFSC_PORT="${GFSC_PORT:-8087}"
CONTROL_API_PORT="${CONTROL_API_PORT:-8090}"
DRIVE="${DRIVE:-main}"
TIMEOUT="${TIMEOUT:-180}"
# A unique scratch dir + host name per run so reruns never collide on the
# sibling-uniqueness index. Caller may pin GFS_E2E_TAG for a deterministic run.
TAG="${GFS_E2E_TAG:-$(kubectl --context="$CONTEXT" -n "$CONTROL_NS" get ns "$CONTROL_NS" -o jsonpath='{.metadata.uid}' 2>/dev/null | cut -c1-8 || echo run)}"
SCRATCH_NAME="scratch-e2e-${TAG}"
HOST_LEAF="gfs-e2e-${TAG}"
# A per-RUN nonce so the gate is re-runnable against the same scratch dir without
# colliding on the sibling-uniqueness index (the scratch dir is stable per
# cluster; the created files are unique per run).
RUN_NONCE="${GFS_E2E_NONCE:-$(date +%s)}"
OP_FILE="op-note-${RUN_NONCE}.txt"
AGENT_FILE="agent-note-${RUN_NONCE}.txt"
HOST_NS="${HOST_ISSUANCE_NAMESPACE:-mcp-host}"
HOST_SUBJECT="host:1st:${HOST_NS}/${HOST_LEAF}"
kc() { kubectl --context="$CONTEXT" "$@"; }
pass=0
fail=0
ok() { echo " PASS: $1"; pass=$((pass + 1)); }
bad() { echo " FAIL: $1"; fail=$((fail + 1)); }
die() {
echo "FATAL: $1" >&2
echo "--- gfs namespace pods ---" >&2
kc -n "$GFS_NS" get pods -o wide 2>&1 | sed 's/^/ /' >&2 || true
echo "--- GlobalFileSystem status ---" >&2
kc -n "$GFS_NS" get globalfilesystem "$GFS_NAME" -o yaml 2>&1 | sed 's/^/ /' >&2 || true
exit 1
}
wait_ready() {
local ns="$1" sel="$2" deadline=$((SECONDS + TIMEOUT))
while ((SECONDS < deadline)); do
if kc -n "$ns" get pods -l "$sel" \
-o 'jsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null \
| grep -q 'True'; then
return 0
fi
sleep 3
done
die "no Ready pod for '$sel' in ns '$ns' within ${TIMEOUT}s"
}
# ready_pod NS SELECTOR — first Running+Ready pod name, or empty.
ready_pod() {
kc -n "$1" get pods -l "$2" --field-selector=status.phase=Running \
-o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null \
| awk -F'\t' '$2=="True"{print $1; exit}'
}
echo "== gfs E2E runtime gate (context=$CONTEXT) =="
# Guard: refuse to run against anything that is not an allowed local/branch
# profile. Accept `clerum-test` and any generated branch-scoped profile
# (`clerum-<topic>-<8hex>`, e.g. clerum-feat-gfs-open-core-b64f4c58 /
# clerum-codex-* / clerum-detached-*) — never a prod context.
case "$CONTEXT" in
*prod* | *production*) die "CONTEXT '$CONTEXT' looks like production — refusing" ;;
esac
if [[ "$CONTEXT" == "clerum-test" || "$CONTEXT" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]; then
:
else
die "CONTEXT '$CONTEXT' is not an allowed local/branch profile for this gate"
fi
# ─── 1. GlobalFileSystem Ready ───────────────────────────────────────────────
kc -n "$GFS_NS" get globalfilesystem "$GFS_NAME" >/dev/null 2>&1 \
|| die "GlobalFileSystem/$GFS_NAME not found in ns $GFS_NS (apply the CRD instance first)"
phase="$(kc -n "$GFS_NS" get globalfilesystem "$GFS_NAME" -o 'jsonpath={.status.phase}' 2>/dev/null || true)"
[[ "$phase" == "Ready" ]] && ok "GlobalFileSystem phase=Ready" || bad "GlobalFileSystem phase='$phase' (want Ready)"
wait_ready "$GFS_NS" "app=gfs-controller" && ok "gfsc pod Ready"
# ─── 2. Service reachable (the brokered access point) ────────────────────────
kc -n "$GFS_NS" get svc gfsc >/dev/null 2>&1 && ok "gfsc Service present" \
|| bad "gfsc Service missing (reconciler did not create it)"
# Route all broker calls through the WRITER pod: it mounts the drive RW so it
# serves both reads and writes (a reader replica mounts RO and rejects writes by
# design). Read-replica fan-out is a separate scaling concern, not this gate.
GFSC_POD="$(ready_pod "$GFS_NS" "app=gfs-controller,clerum.io/gfsc-role=writer")"
[[ -n "$GFSC_POD" ]] || die "no Ready gfsc WRITER pod to exec into"
CONTROL_POD="$(ready_pod "$CONTROL_NS" "app=control-api")"
[[ -n "$CONTROL_POD" ]] || CONTROL_POD="$(ready_pod "$CONTROL_NS" "app.kubernetes.io/name=control-api")"
[[ -n "$CONTROL_POD" ]] || die "no Ready control-api pod to exec into"
# ─── helpers: in-pod node fetch + psql ───────────────────────────────────────
# All HTTP is issued from INSIDE a pod (localhost) so the gate needs no
# port-forward and crosses no NetworkPolicy hop it should not.
# admin_http METHOD PATH [BODY] → "STATUS\tBODY" (operator session cookie).
# The Control-UI admin endpoints (incl. /gfs/token, /gfs/grants, /gfs/by-path)
# authenticate via the httpOnly session COOKIE, not a Bearer header — so
# E2E_ADMIN_TOKEN is sent as the control_ui_admin_session cookie value.
admin_http() {
kc -n "$CONTROL_NS" exec "$CONTROL_POD" -- node -e '
const [method, path, token, body] = process.argv.slice(1)
fetch("http://localhost:'"$CONTROL_API_PORT"'" + path, {
method,
headers: { Cookie: "control_ui_admin_session=" + token, "Content-Type": "application/json" },
body: body || undefined,
}).then(async r => { process.stdout.write(r.status + "\t" + await r.text()) })
.catch(e => { process.stderr.write(e.message); process.exit(1) })
' "$1" "$2" "$E2E_ADMIN_TOKEN" "${3:-}"
}
# ic_http METHOD PATH [BODY] → "STATUS\tBODY". Signs an InternalControl JWT
# (iss=hcc) in-pod with the HCC HMAC secret, then calls control-api as a
# provisioner. Fails loud if the secret is not present in the pod env.
ic_http() {
kc -n "$CONTROL_NS" exec "$CONTROL_POD" -- node -e '
const jwt = require("jsonwebtoken")
const secret = process.env.INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET
if (!secret) { process.stderr.write("INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET not set in control-api pod"); process.exit(2) }
const [method, path, body] = process.argv.slice(1)
const ic = jwt.sign({ sub: "gfs-e2e", jti: "gfs-e2e-" + process.pid + "-" + process.hrtime.bigint() }, secret,
{ algorithm: "HS256", issuer: "hcc", audience: "control-api", expiresIn: 300 })
fetch("http://localhost:'"$CONTROL_API_PORT"'" + path, {
method,
headers: { Authorization: "Bearer " + ic, "Content-Type": "application/json" },
body: body || undefined,
}).then(async r => { process.stdout.write(r.status + "\t" + await r.text()) })
.catch(e => { process.stderr.write(e.message); process.exit(1) })
' "$1" "$2" "${3:-}"
}
# gfsc_http TOKEN METHOD PATH [BODY] → "STATUS\tBODY" (gfs token, in gfsc pod).
gfsc_http() {
kc -n "$GFS_NS" exec "$GFSC_POD" -- node -e '
const [token, method, path, body] = process.argv.slice(1)
fetch("http://localhost:'"$GFSC_PORT"'" + path, {
method,
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
body: body || undefined,
}).then(async r => { process.stdout.write(r.status + "\t" + await r.text()) })
.catch(e => { process.stderr.write(e.message); process.exit(1) })
' "$1" "$2" "$3" "${4:-}"
}
psql_one() {
kc -n "$CONTROL_NS" exec deploy/control-postgres -- \
psql -v ON_ERROR_STOP=1 -U postgres -d profiles -tA -c "$1" 2>/dev/null | tr -d '[:space:]'
}
http_status() { printf '%s' "$1" | cut -f1; }
http_body() { printf '%s' "$1" | cut -f2-; }
CONTROL_ADMIN_USER="${CONTROL_ADMIN_USER:-admin}"
# The seed (scripts/e2e/seed-e2e-data.sh) sets the admin password from the
# canonical repository .env, so the login resolves it the same way; the local
# default applies only when neither the .env nor the environment has one.
operator_admin_password() {
local local_default=""
if [[ "$CONTEXT" == "clerum-test" || "$CONTEXT" =~ ^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$ ]]; then
local_default="$(printf '%s%s' 'changeme123' '!')"
fi
e2e_resolve_admin_password "$REPO_ROOT" "$local_default"
}
ensure_operator_session() {
if [[ -n "${E2E_ADMIN_TOKEN:-}" ]]; then
return 0
fi
local admin_password login_payload login_resp login_status login_body session_cookie
admin_password="$(operator_admin_password)" || die "operator admin credential is required to acquire the Control UI session"
login_payload="$(jq -cn --arg u "$CONTROL_ADMIN_USER" --arg p "$admin_password" '{username:$u,password:$p}')"
login_resp="$(kc -n "$CONTROL_NS" exec -i "$CONTROL_POD" -- node -e '
let input = ""
process.stdin.setEncoding("utf8")
process.stdin.on("data", chunk => { input += chunk })
process.stdin.on("end", async () => {
try {
const payload = JSON.parse(input)
const response = await fetch("http://localhost:'"$CONTROL_API_PORT"'/api/v1/admin/auth/login", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
})
const cookies = typeof response.headers.getSetCookie === "function"
? response.headers.getSetCookie()
: [response.headers.get("set-cookie")].filter(Boolean)
const raw = cookies.map(String).find(cookie => cookie.startsWith("control_ui_admin_session=")) || ""
const value = raw ? raw.split(";")[0].split("=").slice(1).join("=") : ""
process.stdout.write(response.status + "\t" + value + "\t" + await response.text())
} catch (error) {
process.stderr.write(error.message)
process.exit(1)
}
})
' <<<"$login_payload")"
login_status="$(http_status "$login_resp")"
session_cookie="$(printf '%s' "$login_resp" | cut -f2)"
login_body="$(printf '%s' "$login_resp" | cut -f3-)"
[[ "$login_status" == "200" && -n "$session_cookie" ]] || die "operator admin login failed (status=$login_status body=$login_body)"
export E2E_ADMIN_TOKEN="$session_cookie"
ok "operator session obtained through seeded admin login"
}
ensure_operator_session
# ─── mint the operator gfs token (read+write+delete) ─────────────────────────
resp="$(admin_http POST /api/v1/gfs/token '{"scopes":["gfs.read","gfs.write","gfs.delete"]}')"
[[ "$(http_status "$resp")" == "200" ]] || die "operator /gfs/token mint failed: $resp"
OP_TOKEN="$(http_body "$resp" | jq -r '.token')"
[[ -n "$OP_TOKEN" && "$OP_TOKEN" != "null" ]] && ok "operator gfs token minted" || die "no operator token in: $resp"
# ─── seed a scratch root (provisioner / InternalControl JWT) ─────────────────
resp="$(ic_http POST /api/v1/gfs/seed "{\"drive\":\"$DRIVE\",\"rootDirectories\":[\"/$SCRATCH_NAME\"]}")"
case "$(http_status "$resp")" in
200 | 201 | 409) ok "scratch root seeded (/$SCRATCH_NAME)" ;;
*) die "seed /$SCRATCH_NAME failed: $resp" ;;
esac
SCRATCH_RID="$(psql_one "SELECT resource_id FROM gfs_resources WHERE drive='$DRIVE' AND name='$SCRATCH_NAME' AND deleted_at IS NULL ORDER BY created_at DESC LIMIT 1;")"
[[ -n "$SCRATCH_RID" ]] && ok "resolved scratch rid ($SCRATCH_RID)" || die "scratch dir not found in gfs_resources after seed"
# ─── operator creates + reads a file (proves the governed read+write plane) ──
resp="$(gfsc_http "$OP_TOKEN" POST "/v1/resources/$SCRATCH_RID/children" "$(jq -cn --arg n "$OP_FILE" '{name:$n, content:"operator wrote this"}')")"
[[ "$(http_status "$resp")" == "201" ]] || die "operator create in scratch failed: $resp"
OP_FILE_RID="$(http_body "$resp" | jq -r '.data.rid')"
[[ -n "$OP_FILE_RID" && "$OP_FILE_RID" != "null" ]] && ok "operator created a file in scratch (201)" || die "no rid in create response: $resp"
resp="$(gfsc_http "$OP_TOKEN" GET "/v1/resources/$OP_FILE_RID/content")"
[[ "$(http_status "$resp")" == "200" && "$(http_body "$resp")" == "operator wrote this" ]] \
&& ok "operator read the file back (bytes match)" || bad "operator read-back mismatch: $resp"
# ─── 3. Folder owner delegates Layer-2 WRITE to the host (no operator) ────────
grant_body="$(jq -cn --arg d "$DRIVE" --arg r "$SCRATCH_RID" --arg id "1st:${HOST_NS}/${HOST_LEAF}" \
'{drive:$d, resourceId:$r, subject:{type:"host", id:$id}, permissions:["write"], inherit:false}')"
resp="$(admin_http PUT /api/v1/gfs/grants "$grant_body")"
[[ "$(http_status "$resp")" =~ ^20 ]] && ok "host granted write on scratch (delegation accepted)" \
|| die "grant write to $HOST_SUBJECT failed: $resp"
# ─── mint the host (agent) gfs token ─────────────────────────────────────────
# Provisioners may mint host tokens with read and write only
# (control-api/src/gfs/provisionerScopePolicy.ts), so a request that includes
# gfs.delete must be refused before any token exists.
resp="$(ic_http POST "/api/v1/auth/gfs/${HOST_LEAF}/tokens" "{\"namespace\":\"$HOST_NS\",\"scopes\":[\"gfs.write\",\"gfs.read\",\"gfs.delete\"]}")"
if [[ "$(http_status "$resp")" == "403" && "$(http_body "$resp" | jq -r '.error // empty')" == "scope_forbidden" ]]; then
ok "provisioner cannot mint gfs.delete into a host token (403 scope_forbidden)"
else bad "host-token mint with gfs.delete should be 403 scope_forbidden, got: $resp"; fi
resp="$(ic_http POST "/api/v1/auth/gfs/${HOST_LEAF}/tokens" "{\"namespace\":\"$HOST_NS\",\"scopes\":[\"gfs.write\",\"gfs.read\"]}")"
[[ "$(http_status "$resp")" == "200" ]] || die "provisioner host-token mint failed: $resp"
HOST_TOKEN="$(http_body "$resp" | jq -r '.token')"
MINTED_SUBJECT="$(http_body "$resp" | jq -r '.subject')"
[[ "$MINTED_SUBJECT" == "$HOST_SUBJECT" ]] && ok "host token minted for $HOST_SUBJECT" \
|| die "minted subject '$MINTED_SUBJECT' != expected '$HOST_SUBJECT'"
# ─── 4. Agent writes to scratch (GOVERNED — store grant, NOT intrinsic) ──────
resp="$(gfsc_http "$HOST_TOKEN" POST "/v1/resources/$SCRATCH_RID/children" "$(jq -cn --arg n "$AGENT_FILE" '{name:$n, content:"agent wrote this"}')")"
[[ "$(http_status "$resp")" == "201" ]] || die "GOVERNED agent write failed (grant not honored?): $resp"
AGENT_FILE_RID="$(http_body "$resp" | jq -r '.data.rid')"
AGENT_FILE_URI="$(http_body "$resp" | jq -r '.data.gfsUri')"
ok "agent wrote to scratch via its host token (201, governed by the grant)"
# audit: the write produced an allow row for the host subject.
audit_n="$(psql_one "SELECT count(*) FROM gfs_audit WHERE subject='$HOST_SUBJECT' AND op='write' AND outcome='allow';")"
[[ "${audit_n:-0}" -ge 1 ]] && ok "agent write was audited (gfs_audit allow row present)" \
|| bad "no audit row for $HOST_SUBJECT write (got count=$audit_n)"
# ─── 4b. Destructive DELETE is denied for the agent (no delete scope or grant) ─
resp="$(gfsc_http "$HOST_TOKEN" DELETE "/v1/resources/$AGENT_FILE_RID" '{"ifMatch":0}')"
[[ "$(http_status "$resp")" == "403" ]] && ok "agent delete is default-denied (403, destructive bit)" \
|| bad "agent delete should be 403 (default-deny), got: $resp"
# ─── 5. Resolve by gfs:// URI ────────────────────────────────────────────────
resp="$(gfsc_http "$OP_TOKEN" GET "/v1/resolve?uri=$AGENT_FILE_URI")"
[[ "$(http_status "$resp")" == "200" && "$(http_body "$resp" | jq -r '.data.gfsUri')" == "$AGENT_FILE_URI" ]] \
&& ok "agent file resolves by gfs:// URI" || bad "resolve by URI failed: $resp"
# ─── 6. Download by gfs:// URI (operator reads the agent's bytes) ────────────
resp="$(gfsc_http "$OP_TOKEN" GET "/v1/resources/$AGENT_FILE_RID/content")"
[[ "$(http_status "$resp")" == "200" && "$(http_body "$resp")" == "agent wrote this" ]] \
&& ok "agent file downloads by URI (bytes match)" || bad "download mismatch: $resp"
echo "== gfs E2E gate: ${pass} passed, ${fail} failed =="
[[ "$fail" -eq 0 ]] || exit 1