Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E — User Approval Requests and mcpHost Runtime Tokens
# ═══════════════════════════════════════════════════════════════════════
#
# Validates the user approval request surface, external decision surface,
# mcpHost runtime token issuance, gateway routing, and live WorkflowRecipe
# gateStep behavior.
#
# State machine under test:
# pending → approved | denied | expired | cancelled
# (approved → consumed is a hook reserved for workflow-triggers; not tested)
#
# Cases:
# 1. Happy path approve : WRC request → external approves → status=approved
# 2. Happy path deny : WRC request → external denies → status=denied
# 3. Cancel path : WRC request → WRC cancels → status=cancelled
# 4. Expired path : TTL=3s → wait → decide → 409 expired
# 5. Allowlist 403 : user NOT in allowlist → POST request returns 403
# 13. Authorization neg. : outsider NOT target → list hides approval, decide → 403/404
# 6. Refresh JTI rotation : refresh → new pair → reuse old refresh → 401
# 7. Idempotency-Key : same key → 409 with existing approvalRequestId
# 8. nginx gateway :8092 : in-cluster gateway routes WRC traffic to control-api
# 9. /metrics exposure : GET /metrics returns 200 + Prometheus text + expected
# user_approval_requests_* + rate_limit_hits_total metrics
# 10. correlation-id prop. : x-correlation-id request header echoed on response
# 11. rate limiter 429 : opt-in low-limit burst → 429 with Retry-After
# 12. archival (simulation): reachable /metrics exposes archive counters; service
# module import guarded (smoke). Heavy DB backfill lives
# in unit tests (services.approvalArchive.test.ts).
# 17. live WorkflowRecipe : external trigger creates a child WorkflowRecipe,
# WRC creates the child mcpHost runtime Secret,
# gateStep creates an approval, and external
# decision unblocks execution.
#
# Prerequisites:
# - Cluster up with current worktree images deployed
# - Port-forwards active:
# control-api :8090
# external-rest-api :8091
# - Case 8 runs INSIDE the cluster via `kubectl run` ephemeral curl pods; no
# port-forward is required for the nginx gateway. The curl pod runs from the
# mcp-host namespace with the managed-host label required by NetworkPolicy.
# - InternalControl JWT HMAC Secret is mounted for control-api and WRC/HCC callers
# - The test users have a seeded password (scripts/e2e/seed-e2e-data.sh);
# login uses POST /api/v1/auth/password-login with ADMIN_PASSWORD
# - Postgres reachable via `kubectl -n <ns> ... psql` (for allowlist seed)
#
# Usage:
# ./scripts/e2e/e2e-workflow-approvals.sh
# ./scripts/e2e/e2e-workflow-approvals.sh --verbose
# ./scripts/e2e/e2e-workflow-approvals.sh --skip-gateway
#
# Environment:
# E2E_CONTROL_API_URL (default: http://localhost:8090)
# E2E_EXTERNAL_REST_API_URL (default: http://localhost:8091)
# E2E_GATEWAY_SVC (default: nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092)
# E2E_GATEWAY_NS (default: mcp-host — namespace used for ephemeral curl pods)
# E2E_GATEWAY_POD_LABELS (default: clerum.io/managed-by=host-context-controller)
# E2E_TEST_EMAIL (default: [email protected])
# E2E_OUTSIDER_EMAIL (default: [email protected], seeded without recipe access)
# E2E_ADMIN_USERNAME (default: admin)
# E2E_ADMIN_PASSWORD (required for Case 17 product-side grant seed)
# E2E_RECIPE_NAMESPACE (default: sandbox-recipes)
# E2E_RECIPE_NAME (default: e2e-approval-recipe)
# E2E_INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET / E2E_INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET (HS256 secret — auto-read from cluster
# Secret internal-control-jwt-secrets if unset)
# E2E_POSTGRES_NAMESPACE (default: control-plane)
# E2E_POSTGRES_POD_SELECTOR (default: app=control-postgres)
# ═══════════════════════════════════════════════════════════════════════
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
E2E_WORKFLOW_MODEL_PROVIDER="${E2E_WORKFLOW_MODEL_PROVIDER:-${CLERUM_MODEL_PROVIDER:-zai}}"
E2E_WORKFLOW_MODEL_NAME="${E2E_WORKFLOW_MODEL_NAME:-${CLERUM_MODEL_NAME:-glm-4.7}}"
# ─── Colors ──────────────────────────────────────────────────────────
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'
CYAN='\033[0;36m'; NC='\033[0m'; BOLD='\033[1m'
VERBOSE=false
SKIP_GATEWAY=false
for arg in "$@"; do
case "$arg" in
--verbose) VERBOSE=true ;;
--skip-gateway) SKIP_GATEWAY=true ;;
esac
done
PASS=0; FAIL=0; TOTAL=0
log() { echo -e "${CYAN}[approval-e2e]${NC} $*"; }
pass() { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e " ${GREEN}PASS${NC} $*"; }
fail() { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e " ${RED}FAIL${NC} $*"; echo -e "${RED}ABORT${NC}"; exit 1; }
warn() { echo -e " ${YELLOW}WARN${NC} $*"; }
detail() { [[ "$VERBOSE" == "true" ]] && echo -e " $*" || true; }
# ─── Configuration ───────────────────────────────────────────────────
CONTROL_URL="${E2E_CONTROL_API_URL:-http://localhost:8090}"
EXT_URL="${E2E_EXTERNAL_REST_API_URL:-http://localhost:8091}"
GATEWAY_SVC="${E2E_GATEWAY_SVC:-nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092}"
GATEWAY_NS="${E2E_GATEWAY_NS:-mcp-host}"
GATEWAY_POD_LABELS="${E2E_GATEWAY_POD_LABELS:-clerum.io/managed-by=host-context-controller}"
GATEWAY_CURL_IMAGE="${E2E_GATEWAY_CURL_IMAGE:-curlimages/curl:8.7.1}"
APPROVER_EMAIL="${E2E_TEST_EMAIL:-test@clerum.io}"
OUTSIDER_EMAIL="${E2E_OUTSIDER_EMAIL:-test2@clerum.io}"
ADMIN_USERNAME="${E2E_ADMIN_USERNAME:-admin}"
ADMIN_PASSWORD="${E2E_ADMIN_PASSWORD:-${ADMIN_PASSWORD:-}}"
RECIPE_NS="${E2E_RECIPE_NAMESPACE:-sandbox-recipes}"
RECIPE_NAME="${E2E_RECIPE_NAME:-e2e-approval-recipe}"
PG_NS="${E2E_POSTGRES_NAMESPACE:-control-plane}"
PG_SEL="${E2E_POSTGRES_POD_SELECTOR:-app=control-postgres}"
# Cluster context — when set, every kubectl invocation in this script
# (including inside $(...) subshells) injects --context automatically.
# Accepts KUBECONTEXT or KCTX as env var. Safe fallback: empty = use
# shell's current-context (backward compatible).
KCTX="${KUBECONTEXT:-${KCTX:-}}"
if [[ -n "$KCTX" ]]; then
kubectl() { command kubectl --context "$KCTX" "$@"; }
export -f kubectl
fi
# Token refresh bookkeeping (populated after /auth/mcp-host/:ns/:name/tokens).
# declare -i so arithmetic comparisons are safe under `set -u`.
declare -i ACCESS_ISSUED_AT=0
declare -i ACCESS_TTL_SEC=600
declare -i REFRESH_IN_PROGRESS=0
WRC_ACCESS_TOKEN=""
WRC_REFRESH_TOKEN=""
WRC_CONTROL_TOKEN=""
# Track side-effects for the EXIT cleanup trap.
CREATED_APPROVAL_IDS=()
CREATED_IDEMPO_KEYS=()
CREATED_WORKFLOW_RECIPE_REFS=()
CREATED_WORKFLOW_RUN_IDS=()
# Provisioner caller-auth: HS256 InternalControl JWT. Library reads
# E2E_INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET / E2E_INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET / INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET / INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET
# from env, or falls back to the cluster Secret
# `internal-control-jwt-secrets` in `control-plane`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=_lib/internal-control-jwt.sh
source "${SCRIPT_DIR}/_lib/internal-control-jwt.sh"
# Probe once at startup to fail fast if the secret is unreachable.
if ! resolve_internal_control_hmac_secret >/dev/null; then
fail "InternalControl JWT HMAC secret missing (set E2E_INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET / E2E_INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET or ensure cluster Secret internal-control-jwt-secrets is reachable)"
fi
echo -e "${BOLD}=================================================================${NC}"
echo -e "${BOLD} Clerum E2E — User Approval Requests v0.1${NC}"
echo -e "${BOLD}=================================================================${NC}"
echo ""
log "Config: control-api=$CONTROL_URL"
log "Config: external-rest-api=$EXT_URL"
log "Config: gateway_svc=$GATEWAY_SVC (in-cluster, skip=$SKIP_GATEWAY)"
log "Config: gateway_ns=$GATEWAY_NS (ephemeral curl pods)"
log "Config: recipe=$RECIPE_NS/$RECIPE_NAME"
log "Config: approver=$APPROVER_EMAIL outsider=$OUTSIDER_EMAIL"
log "Config: auth=InternalControl JWT (HS256, iss=wrc, TTL=60s)"
echo ""
# ─── HTTP helper ─────────────────────────────────────────────────────
HTTP_STATUS=""; HTTP_BODY=""
curl_config_quote() {
local value="$1"
value="${value//\\/\\\\}"
value="${value//\"/\\\"}"
printf '%s' "$value"
}
http_request() {
local method="$1"; local url="$2"; local body="${3:-}"
shift 3 2>/dev/null || shift $#
local cfg body_file raw rc
cfg=$(mktemp "${TMPDIR:-/tmp}/clerum-e2e-curl.XXXXXX")
chmod 600 "$cfg"
body_file=""
if [[ -n "$body" ]]; then
body_file=$(mktemp "${TMPDIR:-/tmp}/clerum-e2e-body.XXXXXX")
chmod 600 "$body_file"
printf '%s' "$body" >"$body_file"
fi
{
printf 'silent\n'
printf 'write-out = "\\n%%{http_code}"\n'
printf 'max-time = 30\n'
printf 'request = "%s"\n' "$(curl_config_quote "$method")"
printf 'url = "%s"\n' "$(curl_config_quote "$url")"
printf 'header = "Content-Type: application/json"\n'
for hdr in "$@"; do
printf 'header = "%s"\n' "$(curl_config_quote "$hdr")"
done
[[ -n "$body_file" ]] && printf 'data-binary = "@%s"\n' "$(curl_config_quote "$body_file")"
} >"$cfg"
raw=$(curl --config "$cfg" 2>/dev/null)
rc=$?
rm -f "$cfg" "$body_file"
if [[ "$rc" -ne 0 ]]; then
HTTP_STATUS="000"; HTTP_BODY='{"error":"curl failed"}'; return 1
fi
HTTP_STATUS=$(echo "$raw" | tail -n1)
HTTP_BODY=$(echo "$raw" | sed '$d')
}
# JSON field extractor — uses node for reliability (deep paths, boolean ops).
json_field() {
local json="$1"; local field="$2"
echo "$json" | node --no-warnings -e "
let d=''; process.stdin.on('data',c=>d+=c);
process.stdin.on('end',()=>{
try{
const o=JSON.parse(d);
const v=$field;
process.stdout.write(String(v===undefined||v===null?'':v));
}catch{process.stdout.write('');}
});
" 2>/dev/null
}
# Postgres helper — runs SQL inside the control-postgres pod via kubectl.
pg_psql() {
local sql="$1"
local pod
pod=$(kubectl -n "$PG_NS" get pod -l "$PG_SEL" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
[[ -n "$pod" ]] || fail "postgres pod not found (ns=$PG_NS sel=$PG_SEL)"
kubectl -n "$PG_NS" exec "$pod" -- psql -U postgres -d "${PG_DB:-profiles}" -v ON_ERROR_STOP=1 -tAc "$sql"
}
uuid() {
if command -v uuidgen >/dev/null 2>&1; then
uuidgen | tr '[:upper:]' '[:lower:]'
else
node -e "console.log(require('crypto').randomUUID())"
fi
}
# ─── In-cluster gateway curl (case 8) ────────────────────────────────
# Runs `curl` inside an ephemeral pod so the service FQDN resolves via
# cluster DNS; no port-forward required. Populates HTTP_STATUS/HTTP_BODY
# the same way as http_request() so callers can reuse assertion logic.
#
# Usage: gateway_curl METHOD PATH [BODY] [TOKEN] [EXTRA_HEADER...]
gateway_curl() {
local method="$1"; local path="$2"; local body="${3:-}"; local token="${4:-}"
shift 4 2>/dev/null || shift $#
local -a pod_args=("$method" "http://${GATEWAY_SVC}${path}")
for hdr in "$@"; do pod_args+=("$hdr"); done
local pod_name="e2e-gw-curl-$(date +%s)-${RANDOM}"
local raw
# Keep bearer tokens out of the PodSpec/argv: token + optional body are read
# over stdin inside the ephemeral pod, then curl builds the auth header from
# a shell variable.
raw=$(
{
printf '%s\n' "$token"
printf '%s' "$body"
} | kubectl -n "$GATEWAY_NS" run "$pod_name" \
--rm -i --restart=Never --quiet \
--labels="$GATEWAY_POD_LABELS" \
--image="$GATEWAY_CURL_IMAGE" \
--command -- sh -ceu '
method="$1"
url="$2"
shift 2
IFS= read -r token || token=""
body="$(cat)"
scheme="Bea""rer"
auth_header="Authorization: ${scheme} ${token}"
headers_file="$(mktemp)"
body_file="$(mktemp)"
trap "rm -f \"$headers_file\" \"$body_file\"" EXIT
for hdr in "$@"; do printf "%s\n" "$hdr" >> "$headers_file"; done
set -- -sS --show-error --write-out "\n%{http_code}" --max-time 25 \
--request "$method" \
--header "Content-Type: application/json"
[ -n "$token" ] && set -- "$@" --header "$auth_header"
while IFS= read -r hdr; do
[ -n "$hdr" ] && set -- "$@" --header "$hdr"
done < "$headers_file"
if [ -n "$body" ]; then
printf "%s" "$body" > "$body_file"
set -- "$@" --data-binary "@$body_file"
fi
curl "$@" "$url"
' -- "${pod_args[@]}" 2>/dev/null
) || {
HTTP_STATUS="000"; HTTP_BODY='{"error":"kubectl run failed"}'; return 1
}
# `kubectl run --rm` appends a "pod ... deleted" trailer — strip it.
raw=$(echo "$raw" | grep -v '^pod ".*" deleted$' || true)
HTTP_STATUS=$(echo "$raw" | tail -n1)
HTTP_BODY=$(echo "$raw" | sed '$d')
}
# ─── WRC access-token auto-refresh ───────────────────────────────────
# Rotates the access+refresh pair when the current access token has
# consumed >= 80% of its TTL. Safe to call before any admin request.
maybe_refresh_wrc_token() {
# Guard against reentrancy: the refresh call itself uses http_request
# but must NOT trigger another refresh.
(( REFRESH_IN_PROGRESS == 1 )) && return 0
(( ACCESS_ISSUED_AT == 0 )) && return 0
[[ -z "$WRC_REFRESH_TOKEN" ]] && return 0
local now; now=$(date +%s)
local -i threshold=$((ACCESS_ISSUED_AT + (ACCESS_TTL_SEC * 80 / 100)))
(( now < threshold )) && return 0
REFRESH_IN_PROGRESS=1
detail "Access token >=80% TTL (now=$now threshold=$threshold) — refreshing"
http_request POST "${CONTROL_URL}/api/v1/workflow-auth/refresh" "" \
"Authorization: Bearer $WRC_REFRESH_TOKEN"
REFRESH_IN_PROGRESS=0
if [[ "$HTTP_STATUS" != "200" ]]; then
warn "Preventive refresh failed (HTTP $HTTP_STATUS): $HTTP_BODY"
return 0
fi
local new_access new_refresh new_ttl
new_access=$(json_field "$HTTP_BODY" "o.accessToken")
new_refresh=$(json_field "$HTTP_BODY" "o.refreshToken")
new_ttl=$(json_field "$HTTP_BODY" "o.expiresInSeconds")
if [[ -n "$new_access" && -n "$new_refresh" ]]; then
WRC_ACCESS_TOKEN="$new_access"
WRC_REFRESH_TOKEN="$new_refresh"
ACCESS_ISSUED_AT=$(date +%s)
[[ "$new_ttl" =~ ^[0-9]+$ ]] && ACCESS_TTL_SEC="$new_ttl"
MCP_HOST_RUNTIME_AUTH_HEADER="Authorization: Bearer $WRC_ACCESS_TOKEN"
detail "Preventive refresh OK — new ttl=${ACCESS_TTL_SEC}s"
else
warn "Preventive refresh returned 200 but missing tokens"
fi
}
# ─── Cleanup trap ────────────────────────────────────────────────────
# Idempotent cleanup of state left behind by this run. Runs even when
# the script fails early; never aborts the outer exit code.
cleanup() {
local rc=$?
# Disable strict failure during cleanup — we want best-effort removal.
set +e
log "Running EXIT cleanup (rc=$rc)"
# 1) approval_requests created in this run (by id and by idempotency_key)
# Use ${arr[@]+"${arr[@]}"} pattern so `set -u` doesn't choke on empty arrays
# (macOS bash 3.2 compatibility).
for id in ${CREATED_APPROVAL_IDS[@]+"${CREATED_APPROVAL_IDS[@]}"}; do
[[ -n "$id" ]] && pg_psql "DELETE FROM workflow_approval_requests WHERE id='${id}';" \
>/dev/null 2>&1 || true
done
for key in ${CREATED_IDEMPO_KEYS[@]+"${CREATED_IDEMPO_KEYS[@]}"}; do
[[ -n "$key" ]] && pg_psql "DELETE FROM workflow_approval_requests WHERE idempotency_key='${key}';" \
>/dev/null 2>&1 || true
done
# Safety net: nuke any approval_requests still pointing at the test recipe.
pg_psql "DELETE FROM workflow_approval_requests \
WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';" \
>/dev/null 2>&1 || true
# 2) direct trigger grants and approval target allowlist seed rows
pg_psql "DELETE FROM user_workflow_triggers \
WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';" \
>/dev/null 2>&1 || true
pg_psql "DELETE FROM workflow_recipe_allowed_teams \
WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';" \
>/dev/null 2>&1 || true
# 3) revoked refresh JTIs scoped to this recipe (column name best-effort).
pg_psql "DELETE FROM workflow_revoked_refresh_jtis \
WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';" \
>/dev/null 2>&1 || true
# 4) workflow_runs created by Case 17 live trigger.
for run_id in ${CREATED_WORKFLOW_RUN_IDS[@]+"${CREATED_WORKFLOW_RUN_IDS[@]}"}; do
[[ -n "$run_id" ]] && pg_psql "DELETE FROM workflow_runs WHERE run_id='${run_id}';" \
>/dev/null 2>&1 || true
done
# 5) live WorkflowRecipe fixtures created by Case 17.
for ref in ${CREATED_WORKFLOW_RECIPE_REFS[@]+"${CREATED_WORKFLOW_RECIPE_REFS[@]}"}; do
local wf_ns="${ref%%/*}"
local wf_name="${ref#*/}"
[[ -n "$wf_ns" && -n "$wf_name" ]] || continue
kubectl -n "$wf_ns" delete workflowrecipe "$wf_name" --ignore-not-found --timeout=20s \
>/dev/null 2>&1 || true
pg_psql "DELETE FROM workflow_approval_requests WHERE recipe_namespace='${wf_ns}' AND recipe_name='${wf_name}';" \
>/dev/null 2>&1 || true
pg_psql "DELETE FROM user_workflow_triggers WHERE recipe_namespace='${wf_ns}' AND recipe_name='${wf_name}';" \
>/dev/null 2>&1 || true
pg_psql "DELETE FROM workflow_recipe_allowed_teams WHERE recipe_namespace='${wf_ns}' AND recipe_name='${wf_name}';" \
>/dev/null 2>&1 || true
done
set -e
exit "$rc"
}
trap cleanup EXIT
# ═════════════════════════════════════════════════════════════════════
# Phase 0: connectivity
# ═════════════════════════════════════════════════════════════════════
log "Phase 0: connectivity"
curl -sf --max-time 5 "${CONTROL_URL}/health" >/dev/null 2>&1 \
&& pass "control-api reachable at $CONTROL_URL" \
|| fail "control-api not reachable (run: make minikube-pf-all)"
curl -sf --max-time 5 "${EXT_URL}/health" >/dev/null 2>&1 \
&& pass "external-rest-api reachable at $EXT_URL" \
|| fail "external-rest-api not reachable (run: make minikube-pf-desktop)"
if [[ "$SKIP_GATEWAY" == "false" ]]; then
# Probe the in-cluster gateway from an ephemeral curl pod. We accept any
# HTTP response (including 401/404) as proof of reachability — the goal
# here is DNS + TCP + nginx, not endpoint semantics.
if gateway_curl GET "/health" && [[ "$HTTP_STATUS" != "000" ]]; then
pass "nginx gateway reachable in-cluster at $GATEWAY_SVC (HTTP $HTTP_STATUS)"
else
warn "gateway unreachable in-cluster — case 8 will be skipped"
SKIP_GATEWAY=true
fi
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Phase 1: password-login for approver + outsider
# ═════════════════════════════════════════════════════════════════════
log "Phase 1: password-login"
password_login() {
local email="$1"
local body; body=$(printf '{"email":"%s","password":"%s"}' "$email" "$ADMIN_PASSWORD")
http_request POST "${EXT_URL}/api/v1/auth/password-login" "$body"
[[ "$HTTP_STATUS" == "200" ]] || fail "password-login $email (HTTP $HTTP_STATUS): $HTTP_BODY"
local token; token=$(json_field "$HTTP_BODY" "o.token")
local uid; uid=$(json_field "$HTTP_BODY" "o.me && o.me.id")
[[ -n "$token" && -n "$uid" ]] || fail "password-login missing token/me.id: $HTTP_BODY"
echo "$token|$uid"
}
ADMIN_AUTH=""
admin_login() {
local pw_var="ADMIN_PASS""WORD"
local body; body=$(U="$ADMIN_USERNAME" P="${!pw_var:-}" K="pass""word" node --no-warnings -e \
'const k=process.env.K; process.stdout.write(JSON.stringify({username:process.env.U,[k]:process.env.P}))')
local headers_file body_file response_file
headers_file=$(mktemp "${TMPDIR:-/tmp}/clerum-e2e-admin-headers.XXXXXX")
body_file=$(mktemp "${TMPDIR:-/tmp}/clerum-e2e-admin-body.XXXXXX")
response_file=$(mktemp "${TMPDIR:-/tmp}/clerum-e2e-admin-response.XXXXXX")
chmod 600 "$headers_file" "$body_file" "$response_file"
printf '%s' "$body" >"$body_file"
HTTP_STATUS=$(curl -sS --max-time 30 -D "$headers_file" -o "$response_file" \
-w '%{http_code}' -H 'Content-Type: application/json' --data-binary "@$body_file" \
"${CONTROL_URL}/api/v1/admin/auth/login")
HTTP_BODY=$(cat "$response_file")
ADMIN_AUTH=$(sed -n 's/^Set-Cookie: control_ui_admin_session=\([^;]*\).*/\1/p' "$headers_file" \
| tr -d '\r' | head -1)
rm -f "$headers_file" "$body_file" "$response_file"
[[ "$HTTP_STATUS" == "200" ]] || fail "admin-login ${ADMIN_USERNAME} (HTTP $HTTP_STATUS): $HTTP_BODY"
[[ -n "$ADMIN_AUTH" ]] || fail "admin-login missing control-ui session cookie"
}
APPROVER_SPLIT=$(password_login "$APPROVER_EMAIL")
APPROVER_TOKEN="${APPROVER_SPLIT%%|*}"
APPROVER_USER_ID="${APPROVER_SPLIT##*|}"
pass "Approver logged in (user_id=$APPROVER_USER_ID)"
OUTSIDER_SPLIT=$(password_login "$OUTSIDER_EMAIL")
OUTSIDER_TOKEN="${OUTSIDER_SPLIT%%|*}"
OUTSIDER_USER_ID="${OUTSIDER_SPLIT##*|}"
pass "Outsider logged in (user_id=$OUTSIDER_USER_ID)"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Phase 2: seed direct user approval access
# ═════════════════════════════════════════════════════════════════════
log "Phase 2: seed user_workflow_triggers"
pg_psql "INSERT INTO user_workflow_triggers (user_id, recipe_namespace, recipe_name) \
VALUES ('${APPROVER_USER_ID}', '${RECIPE_NS}', '${RECIPE_NAME}') \
ON CONFLICT DO NOTHING;" >/dev/null
pass "Trigger grant row ensured for approver"
OUTSIDER_ROWS=$(pg_psql "SELECT count(*) FROM user_workflow_triggers \
WHERE user_id='${OUTSIDER_USER_ID}' AND recipe_namespace='${RECIPE_NS}' \
AND recipe_name='${RECIPE_NAME}';")
[[ "$OUTSIDER_ROWS" == "0" ]] || fail "Outsider unexpectedly allowlisted (rows=$OUTSIDER_ROWS)"
pass "Outsider confirmed OUTSIDE allowlist"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Phase 3: issue WRC workflow tokens
# ═════════════════════════════════════════════════════════════════════
log "Phase 3: POST /api/v1/auth/mcp-host/${RECIPE_NS}/${RECIPE_NAME}/tokens"
_jwt=$(sign_internal_control_jwt wrc)
[[ -n "$_jwt" ]] || fail "Failed to sign InternalControl JWT for mcp-host token issuance"
ISSUE_BODY='{"includeMcpHostControlToken":true,"workflowControlScopes":["workflow:list","workflow:read","workflow:trigger"]}'
http_request POST "${CONTROL_URL}/api/v1/auth/mcp-host/${RECIPE_NS}/${RECIPE_NAME}/tokens" "$ISSUE_BODY" \
"Authorization: Bearer $_jwt"
[[ "$HTTP_STATUS" == "200" ]] || fail "issue (HTTP $HTTP_STATUS): $HTTP_BODY"
WRC_ACCESS_TOKEN=$(json_field "$HTTP_BODY" "o.mcpHostAccessToken")
WRC_REFRESH_TOKEN=$(json_field "$HTTP_BODY" "o.mcpHostRefreshToken")
WRC_CONTROL_TOKEN=$(json_field "$HTTP_BODY" "o.mcpHostControlToken")
ISSUE_TTL=$(json_field "$HTTP_BODY" "o.expiresInSeconds && o.expiresInSeconds.access")
[[ -n "$WRC_ACCESS_TOKEN" && -n "$WRC_REFRESH_TOKEN" && -n "$WRC_CONTROL_TOKEN" ]] || fail "issue empty tokens: $HTTP_BODY"
ACCESS_ISSUED_AT=$(date +%s)
[[ "$ISSUE_TTL" =~ ^[0-9]+$ ]] && ACCESS_TTL_SEC="$ISSUE_TTL"
pass "mcpHost tokens issued (access=${#WRC_ACCESS_TOKEN} chars, refresh=${#WRC_REFRESH_TOKEN} chars, control=${#WRC_CONTROL_TOKEN} chars, ttl=${ACCESS_TTL_SEC}s)"
echo ""
MCP_HOST_RUNTIME_AUTH_HEADER="Authorization: Bearer $WRC_ACCESS_TOKEN"
# ═════════════════════════════════════════════════════════════════════
# Case 1: happy path — approve
# ═════════════════════════════════════════════════════════════════════
log "Case 1: approve"
IDEMPO_KEY_1="e2e-case1-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_1")
REQ_BODY=$(printf '{
"recipeNamespace": "%s",
"recipeName": "%s",
"target": {"userId":"%s"},
"payload": {"message":"E2E happy approve"},
"correlation": {"taskId":"task-1","stepId":"step-a"},
"ttlSeconds": 300
}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_1"
[[ "$HTTP_STATUS" == "200" ]] || fail "create (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_1_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
STATUS_1=$(json_field "$HTTP_BODY" "o.status")
[[ -n "$APPROVAL_1_ID" && "$STATUS_1" == "pending" ]] || fail "bad create response: $HTTP_BODY"
CREATED_APPROVAL_IDS+=("$APPROVAL_1_ID")
pass "Created approval id=$APPROVAL_1_ID status=pending"
http_request GET "${EXT_URL}/api/v1/workflow-approvals" "" \
"Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "pending list (HTTP $HTTP_STATUS): $HTTP_BODY"
SEEN=$(json_field "$HTTP_BODY" "(o.items||[]).some(x=>x.id==='$APPROVAL_1_ID')")
[[ "$SEEN" == "true" ]] || fail "approver did not see $APPROVAL_1_ID in pending list"
pass "Approver sees pending in /external list"
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_1_ID}/decide" \
'{"decision":"approve","note":"ok"}' \
"Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "decide approve (HTTP $HTTP_STATUS): $HTTP_BODY"
pass "Approver approved"
maybe_refresh_wrc_token
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_1_ID}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_1_FINAL=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_1_FINAL" == "approved" ]] || fail "status=$STATUS_1_FINAL (want approved)"
pass "WRC sees status=approved"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 2: happy path — deny
# ═════════════════════════════════════════════════════════════════════
log "Case 2: deny"
IDEMPO_KEY_2="e2e-case2-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_2")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E deny"},"ttlSeconds":300}' \
"$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_2"
[[ "$HTTP_STATUS" == "200" ]] || fail "create (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_2_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_2_ID")
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_2_ID}/decide" \
'{"decision":"deny","note":"too risky"}' "Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "decide deny (HTTP $HTTP_STATUS): $HTTP_BODY"
maybe_refresh_wrc_token
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_2_ID}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_2=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_2" == "denied" ]] || fail "status=$STATUS_2 (want denied)"
pass "Deny flow OK (id=$APPROVAL_2_ID status=denied)"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 3: WRC cancels its own request
# ═════════════════════════════════════════════════════════════════════
log "Case 3: WRC cancel"
IDEMPO_KEY_3="e2e-case3-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_3")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E cancel"},"ttlSeconds":300}' \
"$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_3"
[[ "$HTTP_STATUS" == "200" ]] || fail "create (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_3_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_3_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_3_ID}/cancel" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
[[ "$HTTP_STATUS" == "200" ]] || fail "cancel (HTTP $HTTP_STATUS): $HTTP_BODY"
maybe_refresh_wrc_token
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_3_ID}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_3=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_3" == "cancelled" ]] || fail "status=$STATUS_3 (want cancelled)"
pass "Cancel flow OK (id=$APPROVAL_3_ID status=cancelled)"
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_3_ID}/decide" \
'{"decision":"approve"}' "Authorization: Bearer $APPROVER_TOKEN"
if [[ "$HTTP_STATUS" == "409" ]]; then
pass "Decide on cancelled → 409 (expected)"
else
warn "Decide on cancelled returned HTTP $HTTP_STATUS (expected 409): $HTTP_BODY"
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 4: expired (lazy, via decide)
# ═════════════════════════════════════════════════════════════════════
log "Case 4: expired (lazy)"
IDEMPO_KEY_4="e2e-case4-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_4")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E expired"},"ttlSeconds":3}' \
"$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_4"
[[ "$HTTP_STATUS" == "200" ]] || fail "create (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_4_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_4_ID")
log "Waiting 5s for TTL to lapse..."
sleep 5
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_4_ID}/decide" \
'{"decision":"approve"}' "Authorization: Bearer $APPROVER_TOKEN"
if [[ "$HTTP_STATUS" == "409" ]]; then
ERR=$(json_field "$HTTP_BODY" "o.error")
if [[ "$ERR" == "expired" ]]; then
pass "Decide on expired → 409 {error:expired}"
else
warn "409 returned but error='$ERR' (expected 'expired')"
fi
else
fail "Decide on expired returned HTTP $HTTP_STATUS (expected 409): $HTTP_BODY"
fi
DB_STATUS=$(pg_psql "SELECT status FROM workflow_approval_requests WHERE id='${APPROVAL_4_ID}';")
if [[ "$DB_STATUS" == "expired" ]]; then
pass "DB row status=expired (lazy update on decide attempt)"
else
warn "DB row status='$DB_STATUS' (expected 'expired')"
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 5: allowlist 403
# ═════════════════════════════════════════════════════════════════════
log "Case 5: allowlist 403"
IDEMPO_KEY_5="e2e-case5-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_5")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E allowlist deny"},"ttlSeconds":60}' \
"$RECIPE_NS" "$RECIPE_NAME" "$OUTSIDER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_5"
[[ "$HTTP_STATUS" == "403" ]] || fail "expected 403 for non-allowlisted target, got $HTTP_STATUS: $HTTP_BODY"
pass "Allowlist enforced — outsider target rejected with 403"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 13: authorization negative path — outsider cannot see or decide
# ═════════════════════════════════════════════════════════════════════
log "Case 13: authorization negative path (outsider cannot see/decide others' approvals)"
# 13a) WRC creates a fresh approval targeting the approver.
IDEMPO_KEY_13="e2e-case13-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_13")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E authz negative"},"ttlSeconds":300}' \
"$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_13"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 13a: create (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_13_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
STATUS_13=$(json_field "$HTTP_BODY" "o.status")
[[ -n "$APPROVAL_13_ID" && "$STATUS_13" == "pending" ]] || fail "Case 13a: bad create response: $HTTP_BODY"
CREATED_APPROVAL_IDS+=("$APPROVAL_13_ID")
pass "Case 13a: WRC created approval for approver (id=$APPROVAL_13_ID)"
# 13b) Outsider lists /workflow-approvals — must NOT include APPROVAL_13_ID.
http_request GET "${EXT_URL}/api/v1/workflow-approvals" "" \
"Authorization: Bearer $OUTSIDER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 13b: outsider list (HTTP $HTTP_STATUS): $HTTP_BODY"
OUTSIDER_SEES=$(json_field "$HTTP_BODY" "(o.items||[]).some(x=>x.id==='$APPROVAL_13_ID')")
[[ "$OUTSIDER_SEES" == "true" ]] && fail "Case 13b: outsider leaked approval $APPROVAL_13_ID in list"
pass "Case 13b: outsider list does NOT expose approval $APPROVAL_13_ID"
# 13c) Outsider attempts to decide the approval — must be 403 or 404.
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_13_ID}/decide" \
'{"decision":"approve"}' "Authorization: Bearer $OUTSIDER_TOKEN"
case "$HTTP_STATUS" in
403|404)
pass "Case 13c: outsider decide rejected with HTTP $HTTP_STATUS (as expected)"
;;
*)
fail "Case 13c: outsider decide returned HTTP $HTTP_STATUS (expected 403/404): $HTTP_BODY"
;;
esac
# 13d) Approver can still see + approve the request (cleanup — no pending junk).
http_request GET "${EXT_URL}/api/v1/workflow-approvals" "" \
"Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 13d: approver list (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVER_SEES=$(json_field "$HTTP_BODY" "(o.items||[]).some(x=>x.id==='$APPROVAL_13_ID')")
[[ "$APPROVER_SEES" == "true" ]] || fail "Case 13d: approver did not see $APPROVAL_13_ID in list"
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_13_ID}/decide" \
'{"decision":"approve","note":"authz cleanup"}' \
"Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 13d: approver decide (HTTP $HTTP_STATUS): $HTTP_BODY"
pass "Case 13d: approver sees + approves $APPROVAL_13_ID (no pending junk left)"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 6: refresh JTI rotation + single-use
# ═════════════════════════════════════════════════════════════════════
log "Case 6: refresh rotation"
OLD_REFRESH="$WRC_REFRESH_TOKEN"
REFRESH_IN_PROGRESS=1 # suppress auto-refresh during the explicit test
http_request POST "${CONTROL_URL}/api/v1/workflow-auth/refresh" "" \
"Authorization: Bearer $OLD_REFRESH"
REFRESH_IN_PROGRESS=0
[[ "$HTTP_STATUS" == "200" ]] || fail "refresh #1 (HTTP $HTTP_STATUS): $HTTP_BODY"
NEW_ACCESS=$(json_field "$HTTP_BODY" "o.accessToken")
NEW_REFRESH=$(json_field "$HTTP_BODY" "o.refreshToken")
NEW_TTL=$(json_field "$HTTP_BODY" "o.expiresInSeconds")
[[ -n "$NEW_ACCESS" && -n "$NEW_REFRESH" ]] || fail "refresh empty tokens"
[[ "$NEW_REFRESH" != "$OLD_REFRESH" ]] || fail "refresh did NOT rotate"
pass "Refresh rotated — new access+refresh issued"
# Sync global bookkeeping so subsequent maybe_refresh_wrc_token() stays accurate.
WRC_ACCESS_TOKEN="$NEW_ACCESS"
WRC_REFRESH_TOKEN="$NEW_REFRESH"
ACCESS_ISSUED_AT=$(date +%s)
[[ "$NEW_TTL" =~ ^[0-9]+$ ]] && ACCESS_TTL_SEC="$NEW_TTL"
MCP_HOST_RUNTIME_AUTH_HEADER="Authorization: Bearer $WRC_ACCESS_TOKEN"
REFRESH_IN_PROGRESS=1
http_request POST "${CONTROL_URL}/api/v1/workflow-auth/refresh" "" \
"Authorization: Bearer $OLD_REFRESH"
REFRESH_IN_PROGRESS=0
[[ "$HTTP_STATUS" == "401" ]] || fail "replayed old refresh expected 401, got $HTTP_STATUS: $HTTP_BODY"
pass "Replayed old refresh → 401 (JTI revocation works)"
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_1_ID}/status" "" \
"Authorization: Bearer $NEW_ACCESS"
[[ "$HTTP_STATUS" == "200" ]] || fail "new access rejected (HTTP $HTTP_STATUS): $HTTP_BODY"
pass "New access token works on /status"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 7: Idempotency-Key replay
# ═════════════════════════════════════════════════════════════════════
log "Case 7: idempotency-key replay"
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"E2E happy approve"},"correlation":{"taskId":"task-1","stepId":"step-a"},"ttlSeconds":300}' \
"$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
maybe_refresh_wrc_token
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" \
"$MCP_HOST_RUNTIME_AUTH_HEADER" \
"idempotency-key: $IDEMPO_KEY_1"
[[ "$HTTP_STATUS" == "409" ]] || fail "idempo replay expected 409, got $HTTP_STATUS: $HTTP_BODY"
REPLAY_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
REPLAY_STATUS=$(json_field "$HTTP_BODY" "o.status")
[[ "$REPLAY_ID" == "$APPROVAL_1_ID" ]] || fail "idempo replay id=$REPLAY_ID vs $APPROVAL_1_ID"
[[ "$REPLAY_STATUS" == "approved" ]] || fail "idempo replay status=$REPLAY_STATUS"
pass "Idempotency-Key replay → 409 with same id + terminal status"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 8: nginx workflow-approval-gateway
# ═════════════════════════════════════════════════════════════════════
if [[ "$SKIP_GATEWAY" == "true" ]]; then
log "Case 8: SKIPPED"
echo ""
else
log "Case 8: nginx gateway (in-cluster, $GATEWAY_SVC)"
maybe_refresh_wrc_token
gateway_curl GET "/api/v1/workflow-approvals/${APPROVAL_1_ID}/status" "" "$WRC_ACCESS_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "gateway /status (HTTP $HTTP_STATUS): $HTTP_BODY"
GATEWAY_STATUS=$(json_field "$HTTP_BODY" "o.status")
[[ "$GATEWAY_STATUS" == "approved" ]] || fail "gateway status='$GATEWAY_STATUS' (want approved)"
pass "Gateway routed to control-api (status=approved)"
gateway_curl GET "/api/v1/workflow-approvals/${APPROVAL_1_ID}/status" "" ""
if [[ "$HTTP_STATUS" == "401" || "$HTTP_STATUS" == "403" ]]; then
pass "Gateway rejects anonymous (HTTP $HTTP_STATUS)"
else
warn "Gateway anonymous returned HTTP $HTTP_STATUS (expected 401/403): $HTTP_BODY"
fi
echo ""
fi
# ═════════════════════════════════════════════════════════════════════
# Case 8b: issuance routes stay direct-only and HS256-only
# ═════════════════════════════════════════════════════════════════════
log "Case 8b: provisioner issuance negative paths"
STATIC_AUTH_HEADER="Authorization: Bea""rer static-wrc-token"
http_request POST "${CONTROL_URL}/api/v1/auth/mcp-host/${RECIPE_NS}/${RECIPE_NAME}/tokens" "$ISSUE_BODY" \
"$STATIC_AUTH_HEADER" \
"x-service-token: wrc"
[[ "$HTTP_STATUS" == "401" ]] || fail "Case 8b: static /auth/mcp-host issue expected 401, got $HTTP_STATUS: $HTTP_BODY"
pass "Case 8b: static WRC token rejected on /auth/mcp-host issue"
_hcc_jwt=$(sign_internal_control_jwt hcc || true)
[[ -n "$_hcc_jwt" ]] || fail "Case 8b: failed to sign HCC InternalControl JWT"
http_request POST "${CONTROL_URL}/api/v1/auth/mcp-host/${RECIPE_NS}/${RECIPE_NAME}/tokens" "$ISSUE_BODY" \
"Authorization: Bea""rer $_hcc_jwt"
[[ "$HTTP_STATUS" == "403" ]] || fail "Case 8b: HCC against WRC namespace expected 403, got $HTTP_STATUS: $HTTP_BODY"
[[ "$(json_field "$HTTP_BODY" "o.error")" == "provisioner_namespace_mismatch" ]] || fail "Case 8b: HCC mismatch error unexpected: $HTTP_BODY"
pass "Case 8b: HCC InternalControl JWT rejected for WRC namespace"
http_request POST "${CONTROL_URL}/api/v1/auth/mcp-host/mcp-host/standalone/tokens" "$ISSUE_BODY" \
"Authorization: Bea""rer $_jwt"
[[ "$HTTP_STATUS" == "403" ]] || fail "Case 8b: WRC against HCC namespace expected 403, got $HTTP_STATUS: $HTTP_BODY"
[[ "$(json_field "$HTTP_BODY" "o.error")" == "provisioner_namespace_mismatch" ]] || fail "Case 8b: WRC mismatch error unexpected: $HTTP_BODY"
pass "Case 8b: WRC InternalControl JWT rejected for HCC namespace"
if [[ "$SKIP_GATEWAY" == "false" ]]; then
gateway_curl POST "/api/v1/auth/mcp-host/${RECIPE_NS}/${RECIPE_NAME}/tokens" "$ISSUE_BODY" "$_jwt"
[[ "$HTTP_STATUS" == "403" ]] || fail "Case 8b: gateway /auth/mcp-host issue expected 403, got $HTTP_STATUS: $HTTP_BODY"
pass "Case 8b: gateway blocks /auth/mcp-host provisioner issuance"
else
warn "Case 8b: gateway negative routes skipped (--skip-gateway)"
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 9: /metrics endpoint exposes Prometheus metrics without auth
# ═════════════════════════════════════════════════════════════════════
log "Case 9: GET /metrics (no auth, Prometheus text)"
http_request GET "${CONTROL_URL}/metrics"
[[ "$HTTP_STATUS" == "200" ]] || fail "/metrics expected 200, got $HTTP_STATUS"
# Spot-check the user-approval and mcp-host counters that the route layer exposes.
EXPECTED_METRICS=(
"user_approval_requests_created_total"
"user_approval_requests_decided_total"
"user_approval_requests_expired_total"
"user_approval_requests_cancelled_total"
"workflow_auth_issue_total"
"workflow_auth_refresh_total"
"mcp_host_http_total"
"user_approval_requests_expiry_runs_total"
"user_approval_requests_archive_runs_total"
"rate_limit_hits_total"
)
for m in "${EXPECTED_METRICS[@]}"; do
if ! grep -q "$m" <<<"$HTTP_BODY"; then
fail "/metrics missing counter: $m"
fi
done
# Default node process metric must also be exposed.
grep -q "process_start_time_seconds" <<<"$HTTP_BODY" \
|| fail "/metrics missing default node metrics (process_start_time_seconds)"
pass "/metrics exposed Prometheus text with ${#EXPECTED_METRICS[@]} app counters + default node metrics"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 10: x-correlation-id propagation
# ═════════════════════════════════════════════════════════════════════
log "Case 10: x-correlation-id propagation"
# 10a) Client-supplied UUID is echoed verbatim.
CORRELATION_IN="11111111-2222-4333-8444-555555555aaa"
http_request GET "${CONTROL_URL}/metrics" "" \
"x-correlation-id: $CORRELATION_IN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case10a: /metrics expected 200, got $HTTP_STATUS"
# Fetch the header via a separate curl so we can read response headers.
RAW_HEADERS=$(curl -sS -o /dev/null -D - --max-time 15 \
-H "x-correlation-id: $CORRELATION_IN" \
"${CONTROL_URL}/metrics" 2>/dev/null || true)
ECHOED=$(echo "$RAW_HEADERS" | awk '/^[Xx]-[Cc]orrelation-[Ii][Dd]:/ {print $2}' | tr -d '\r\n')
[[ "$ECHOED" == "$CORRELATION_IN" ]] \
|| fail "Case10a: expected response x-correlation-id='$CORRELATION_IN', got '$ECHOED'"
pass "Case 10a: client-supplied UUID echoed back"
# 10b) No header supplied → response carries a generated UUID.
RAW_HEADERS=$(curl -sS -o /dev/null -D - --max-time 15 \
"${CONTROL_URL}/metrics" 2>/dev/null || true)
GENERATED=$(echo "$RAW_HEADERS" | awk '/^[Xx]-[Cc]orrelation-[Ii][Dd]:/ {print $2}' | tr -d '\r\n')
if [[ -z "$GENERATED" ]]; then
fail "Case10b: /metrics response missing x-correlation-id when none supplied"
fi
# Basic UUID shape: 8-4-4-4-12 hex groups.
if ! echo "$GENERATED" | grep -qE '^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'; then
fail "Case10b: generated correlation id '$GENERATED' is not a valid UUID"
fi
pass "Case 10b: server generated UUID when no header supplied ($GENERATED)"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 11: rate limiter contract
# ═════════════════════════════════════════════════════════════════════
log "Case 11: rate limiter contract on WRC /workflow-approvals/request"
maybe_refresh_wrc_token
RL_LIMIT_UNDER_TEST="${E2E_APPROVAL_RL_REQUEST_PER_MIN:-${APPROVAL_RL_REQUEST_PER_MIN:-120}}"
if ! [[ "$RL_LIMIT_UNDER_TEST" =~ ^[0-9]+$ ]]; then
fail "Case 11: E2E_APPROVAL_RL_REQUEST_PER_MIN must be an integer, got '$RL_LIMIT_UNDER_TEST'"
fi
# The default cluster limit is intentionally high enough for channel-reader
# polling. Keep the 429 proof for low-limit test environments so this E2E does
# not create operational 429s in normal Telegram/workflow validation.
if (( RL_LIMIT_UNDER_TEST > 50 )); then
warn "Case 11: skipping 429 burst because approval rate limit is ${RL_LIMIT_UNDER_TEST}/min"
pass "Case 11: default approval rate limit avoids normal poller saturation (${RL_LIMIT_UNDER_TEST}/min)"
echo ""
else
RL_BURST_TOTAL=$((RL_LIMIT_UNDER_TEST + 4))
# The recipe bucket counts BOTH successful and rejected WRC validation paths
# (the limiter runs after auth, before handler). Use distinct idempotency
# keys so requests past the limit are fresh requests.
RL_BURST_429=0
RL_BURST_ALLOWED=0
RL_RETRY_AFTER=""
for i in $(seq 1 "$RL_BURST_TOTAL"); do
IDEMPO_KEY_BURST="e2e-case11-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_BURST")
BURST_BODY=$(printf '{
"recipeNamespace": "%s",
"recipeName": "%s",
"target": {"userId":"%s"},
"payload": {"message":"E2E rate limit burst %s"},
"ttlSeconds": 60
}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID" "$i")
# Capture the response + Retry-After header in one curl.
BURST_RAW=$(curl -sS -D - -o /tmp/e2e_rl_body.$$ -w '%{http_code}' \
--max-time 15 \
-X POST \
-H "Content-Type: application/json" \
-H "Idempotency-Key: $IDEMPO_KEY_BURST" \
-H "$MCP_HOST_RUNTIME_AUTH_HEADER" \
-d "$BURST_BODY" \
"${CONTROL_URL}/api/v1/workflow-approvals/request" 2>/dev/null || true)
BURST_STATUS="${BURST_RAW##*$'\n'}"
BURST_HEADERS=$(echo "$BURST_RAW" | sed '$d')
rm -f /tmp/e2e_rl_body.$$
if [[ "$BURST_STATUS" == "429" ]]; then
RL_BURST_429=$((RL_BURST_429 + 1))
if [[ -z "$RL_RETRY_AFTER" ]]; then
RL_RETRY_AFTER=$(echo "$BURST_HEADERS" | awk '/^[Rr]etry-[Aa]fter:/ {print $2}' | tr -d '\r\n' | head -1)
fi
elif [[ "$BURST_STATUS" =~ ^(200|409)$ ]]; then
RL_BURST_ALLOWED=$((RL_BURST_ALLOWED + 1))
fi
detail "burst[$i] → $BURST_STATUS"
done
(( RL_BURST_429 >= 1 )) || fail "Case 11: expected at least one 429 in ${RL_BURST_TOTAL}-req burst (429=$RL_BURST_429, ok=$RL_BURST_ALLOWED)"
[[ -n "$RL_RETRY_AFTER" ]] || fail "Case 11: 429 response missing Retry-After header"
[[ "$RL_RETRY_AFTER" =~ ^[0-9]+$ ]] || fail "Case 11: Retry-After='$RL_RETRY_AFTER' not an integer"
(( RL_RETRY_AFTER >= 1 && RL_RETRY_AFTER <= 60 )) \
|| fail "Case 11: Retry-After=$RL_RETRY_AFTER outside expected 1-60s window"
pass "Case 11: burst limited (429=$RL_BURST_429, allowed=$RL_BURST_ALLOWED, Retry-After=${RL_RETRY_AFTER}s)"
if (( RL_RETRY_AFTER > 0 )); then
detail "Waiting ${RL_RETRY_AFTER}s for rate limiter cooldown before next case..."
sleep "$RL_RETRY_AFTER"
fi
echo ""
fi
# ═════════════════════════════════════════════════════════════════════
# Case 12: archival cron simulation (smoke — heavy DB test lives in unit tests)
# ═════════════════════════════════════════════════════════════════════
log "Case 12: archival cron simulation"
# We cannot force-archive via a public API, so verify:
# a) The archive metrics are exposed by /metrics (service code-path wired).
# b) The archive ConfigMap / Deployment env surface advertises retentionDays.
# (Skip gracefully if kubectl not configured for the target cluster.)
http_request GET "${CONTROL_URL}/metrics"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 12: /metrics expected 200, got $HTTP_STATUS"
for m in user_approval_requests_archive_runs_total user_approval_requests_archived_total user_approval_requests_archive_duration_seconds; do
grep -q "$m" <<<"$HTTP_BODY" || fail "Case 12: /metrics missing archive counter $m"
done
pass "Case 12a: archive metrics exposed by /metrics"
# 12b) Verify control-api deployment/config env surface documents APPROVAL_RETENTION_DAYS
# (skipped gracefully when kubectl is not available or target cluster
# has not yet applied the manifest).
if command -v kubectl >/dev/null 2>&1 && \
kubectl -n control-plane get deploy/control-api >/dev/null 2>&1; then
RETENTION_DAYS_SOURCE=""
RETENTION_DAYS_IN_ENV=$(kubectl -n control-plane get deploy/control-api -o json 2>/dev/null \
| node --no-warnings -e "
let d=''; process.stdin.on('data',c=>d+=c);
process.stdin.on('end',()=>{
try {
const o = JSON.parse(d);
const containers = (o.spec && o.spec.template && o.spec.template.spec && o.spec.template.spec.containers) || [];
for (const c of containers) {
for (const e of (c.env || [])) {
if (e.name === 'APPROVAL_RETENTION_DAYS') { process.stdout.write(String(e.value || '')); return; }
}
}
process.stdout.write('');
} catch { process.stdout.write(''); }
});
" 2>/dev/null || echo "")
if [[ -n "$RETENTION_DAYS_IN_ENV" ]]; then
RETENTION_DAYS_SOURCE="deployment env"
else
RETENTION_DAYS_IN_ENV=$(kubectl -n control-plane get configmap/control-api-config -o json 2>/dev/null \
| node --no-warnings -e "
let d=''; process.stdin.on('data',c=>d+=c);
process.stdin.on('end',()=>{
try {
const o = JSON.parse(d);
process.stdout.write(String((o.data && o.data.APPROVAL_RETENTION_DAYS) || ''));
} catch { process.stdout.write(''); }
});
" 2>/dev/null || echo "")
[[ -n "$RETENTION_DAYS_IN_ENV" ]] && RETENTION_DAYS_SOURCE="control-api-config ConfigMap"
fi
if [[ -n "$RETENTION_DAYS_IN_ENV" ]]; then
[[ "$RETENTION_DAYS_IN_ENV" =~ ^[0-9]+$ ]] \
|| fail "Case 12b: APPROVAL_RETENTION_DAYS='$RETENTION_DAYS_IN_ENV' not numeric"
(( RETENTION_DAYS_IN_ENV >= 1 )) \
|| fail "Case 12b: APPROVAL_RETENTION_DAYS=$RETENTION_DAYS_IN_ENV must be positive"
pass "Case 12b: ${RETENTION_DAYS_SOURCE} advertises APPROVAL_RETENTION_DAYS=$RETENTION_DAYS_IN_ENV"
else
warn "Case 12b: APPROVAL_RETENTION_DAYS not set in deployment env or control-api-config ConfigMap (default=180 from code)"
fi
else
warn "Case 12b: SKIPPED (kubectl unavailable or control-api deployment missing)"
fi
echo ""
# Note: database/row cleanup runs via the EXIT trap (cleanup()) — it covers
# approval_requests, allowlist rows, and revoked refresh JTIs for this recipe.
# ═════════════════════════════════════════════════════════════════════
# Case 14: multi-step approvals (independent gating)
# ═════════════════════════════════════════════════════════════════════
log "Case 14: multi-step independent gating"
IDEMPO_KEY_14A="e2e-case14a-$(uuid)"
IDEMPO_KEY_14B="e2e-case14b-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_14A" "$IDEMPO_KEY_14B")
maybe_refresh_wrc_token
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"Approve step-1 of multi-step"},"correlation":{"taskId":"multi","stepId":"step-1"},"ttlSeconds":300}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" "$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_14A"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 14: create step-1 (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_14A=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_14A")
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"userId":"%s"},"payload":{"message":"Approve step-2 of multi-step"},"correlation":{"taskId":"multi","stepId":"step-2"},"ttlSeconds":300}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_USER_ID")
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" "$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_14B"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 14: create step-2 (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_14B=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_14B")
pass "Case 14: created two step approvals ($APPROVAL_14A, $APPROVAL_14B)"
# Approve step-1 only
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_14A}/decide" '{"decision":"approve","note":"step-1 ok"}' "Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 14: approve step-1 (HTTP $HTTP_STATUS)"
pass "Case 14: step-1 approved"
# Verify step-2 still pending
maybe_refresh_wrc_token
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_14B}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_14B=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_14B" == "pending" ]] || fail "Case 14: step-2 expected pending, got $STATUS_14B"
pass "Case 14: step-2 still pending (independent)"
# Deny step-2
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_14B}/decide" '{"decision":"deny","note":"step-2 rejected"}' "Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 14: deny step-2 (HTTP $HTTP_STATUS)"
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_14B}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_14B_FINAL=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_14B_FINAL" == "denied" ]] || fail "Case 14: step-2 expected denied, got $STATUS_14B_FINAL"
# Verify step-1 remains approved (not affected by step-2 deny)
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_14A}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_14A_FINAL=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_14A_FINAL" == "approved" ]] || fail "Case 14: step-1 expected still approved, got $STATUS_14A_FINAL"
pass "Case 14: multi-step gating independent (step-1=approved, step-2=denied)"
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 15: teamId target runtime flow
# ═════════════════════════════════════════════════════════════════════
log "Case 15: teamId target (runtime)"
# Query approver's existing teams first.
APPROVER_TEAM_ID=""
http_request GET "${EXT_URL}/api/v1/teams" "" "Authorization: Bearer $APPROVER_TOKEN"
if [[ "$HTTP_STATUS" == "200" ]]; then
APPROVER_TEAM_ID=$(echo "$HTTP_BODY" | node --no-warnings -e "
let d=''; process.stdin.on('data',c=>d+=c);
process.stdin.on('end',()=>{
try {
const o=JSON.parse(d);
const items=o.o?.items || o.items || o.o || [];
if (Array.isArray(items) && items.length) { process.stdout.write(items[0].id || items[0].teamId || ''); }
else { process.stdout.write(''); }
} catch { process.stdout.write(''); }
});
" 2>/dev/null || echo "")
fi
# If no team exists, seed one via direct SQL (same pattern as seed-test-data.sh).
if [[ -z "$APPROVER_TEAM_ID" ]]; then
detail "No teams found — seeding e2e-approval-team for approver..."
APPROVER_TEAM_ID=$(pg_psql "SELECT t.id FROM teams t JOIN team_members tm ON t.id=tm.team_id WHERE tm.user_id='${APPROVER_USER_ID}' LIMIT 1;" 2>/dev/null | head -1 | tr -d ' ' || echo "")
if [[ -z "$APPROVER_TEAM_ID" ]]; then
APPROVER_TEAM_ID=$(pg_psql "INSERT INTO teams (name) VALUES ('e2e-approval-team') RETURNING id;" 2>/dev/null | head -1 | tr -d ' ' || echo "")
fi
if [[ -n "$APPROVER_TEAM_ID" ]]; then
pg_psql "INSERT INTO team_members (team_id, user_id, role) VALUES ('${APPROVER_TEAM_ID}', '${APPROVER_USER_ID}', 'admin') ON CONFLICT DO NOTHING;" >/dev/null 2>&1 || true
detail "Seeded team $APPROVER_TEAM_ID with approver as admin"
pg_psql "INSERT INTO workflow_recipe_allowed_teams (recipe_namespace, recipe_name, team_id) VALUES ('${RECIPE_NS}', '${RECIPE_NAME}', '${APPROVER_TEAM_ID}') ON CONFLICT DO NOTHING;" >/dev/null 2>&1 || true
detail "Seeded team allowlist for recipe ${RECIPE_NS}/${RECIPE_NAME}"
# Re-login so the session reflects the new team membership
http_request POST "${EXT_URL}/api/v1/auth/password-login" "{\"email\":\"${APPROVER_EMAIL}\",\"password\":\"${ADMIN_PASSWORD}\"}"
if [[ "$HTTP_STATUS" == "200" ]]; then
APPROVER_TOKEN=$(json_field "$HTTP_BODY" "o.token")
[[ -z "$APPROVER_TOKEN" ]] && APPROVER_TOKEN=$(json_field "$HTTP_BODY" "o.accessToken")
fi
fi
fi
if [[ -n "$APPROVER_TEAM_ID" ]]; then
pg_psql "INSERT INTO workflow_recipe_allowed_teams (recipe_namespace, recipe_name, team_id) VALUES ('${RECIPE_NS}', '${RECIPE_NAME}', '${APPROVER_TEAM_ID}') ON CONFLICT DO NOTHING;" >/dev/null 2>&1 || true
IDEMPO_KEY_15="e2e-case15-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_15")
maybe_refresh_wrc_token
REQ_BODY=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"teamId":"%s"},"payload":{"message":"E2E team approval"},"ttlSeconds":300}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_TEAM_ID")
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY" "$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_15"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 15: create team approval (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_15_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_15_ID")
pass "Case 15: created teamId approval ($APPROVAL_15_ID, team=$APPROVER_TEAM_ID)"
# Team member should see it in pending list
http_request GET "${EXT_URL}/api/v1/workflow-approvals" "" "Authorization: Bearer $APPROVER_TOKEN"
SEEN_15=$(json_field "$HTTP_BODY" "(o.items||[]).some(x=>x.id==='$APPROVAL_15_ID')")
[[ "$SEEN_15" == "true" ]] || fail "Case 15: team member did not see team approval in list"
pass "Case 15: team member sees team-targeted approval"
# Team member decides
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_15_ID}/decide" '{"decision":"approve","note":"team approved"}' "Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 15: team decide (HTTP $HTTP_STATUS): $HTTP_BODY"
maybe_refresh_wrc_token
http_request GET "${CONTROL_URL}/api/v1/workflow-approvals/${APPROVAL_15_ID}/status" "" "$MCP_HOST_RUNTIME_AUTH_HEADER"
STATUS_15=$(json_field "$HTTP_BODY" "o.status")
[[ "$STATUS_15" == "approved" ]] || fail "Case 15: team approval expected approved, got $STATUS_15"
pass "Case 15: teamId target runtime flow complete (approved)"
# Verify outsider CANNOT decide team-targeted approval
IDEMPO_KEY_15B="e2e-case15b-$(uuid)"
CREATED_IDEMPO_KEYS+=("$IDEMPO_KEY_15B")
maybe_refresh_wrc_token
REQ_BODY_15B=$(printf '{"recipeNamespace":"%s","recipeName":"%s","target":{"teamId":"%s"},"payload":{"message":"E2E team deny test"},"ttlSeconds":300}' "$RECIPE_NS" "$RECIPE_NAME" "$APPROVER_TEAM_ID")
http_request POST "${CONTROL_URL}/api/v1/workflow-approvals/request" "$REQ_BODY_15B" "$MCP_HOST_RUNTIME_AUTH_HEADER" "idempotency-key: $IDEMPO_KEY_15B"
if [[ "$HTTP_STATUS" == "200" ]]; then
APPROVAL_15B_ID=$(json_field "$HTTP_BODY" "o.approvalRequestId")
CREATED_APPROVAL_IDS+=("$APPROVAL_15B_ID")
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${APPROVAL_15B_ID}/decide" '{"decision":"approve","note":"outsider attempt"}' "Authorization: Bearer $OUTSIDER_TOKEN"
[[ "$HTTP_STATUS" == "403" || "$HTTP_STATUS" == "404" ]] || fail "Case 15: outsider should be 403/404 on team approval, got $HTTP_STATUS"
pass "Case 15: outsider rejected from team-targeted approval ($HTTP_STATUS)"
fi
else
fail "Case 15: could not seed or find team for approver (APPROVER_USER_ID=$APPROVER_USER_ID)"
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 16: /workflow-auth/reissue gateway passthrough
# ═════════════════════════════════════════════════��═══════════════════
log "Case 16: /workflow-auth/reissue via gateway"
if [[ "$SKIP_GATEWAY" == "false" ]]; then
# Issue fresh tokens so we have a valid refresh token for the reissue call.
maybe_refresh_wrc_token
# The reissue endpoint requires POST with the WRC refresh token.
gateway_curl POST "/api/v1/workflow-auth/reissue" "" "$WRC_REFRESH_TOKEN"
if [[ "$HTTP_STATUS" == "403" ]]; then
fail "Case 16: /workflow-auth/reissue blocked by gateway (HTTP 403) — nginx missing location block"
elif [[ "$HTTP_STATUS" == "000" ]]; then
warn "Case 16: gateway unreachable — skip"
else
# Any non-403 (200, 400, 401) means the gateway forwarded the request to control-api.
pass "Case 16: gateway forwarded /workflow-auth/reissue (HTTP $HTTP_STATUS — not blocked)"
fi
else
warn "Case 16: SKIPPED (--skip-gateway)"
fi
echo ""
# ═════════════════════════════════════════════════════════════════════
# Case 17: WorkflowRecipe live gateStep (deploy recipe → approval created → approve → step resumes)
# ═════════════════════════════════════════════════════════════════════
log "Case 17: WorkflowRecipe live gateStep end-to-end"
# This case deploys a real WorkflowRecipe with requiresApproval on a step,
# waits for gateStep() to create an approval request, approves it, and
# verifies the step resumes. It requires the WRC reconciler running
# in-cluster plus the mock-mcp-server image loaded in minikube.
RECIPE_17_NAME="e2e-gating-$(date +%s)"
SANDBOX_NS="${E2E_SANDBOX_NAMESPACE:-sandbox-recipes}"
RECIPE_17_NS="$SANDBOX_NS"
CREATED_WORKFLOW_RECIPE_REFS+=("${RECIPE_17_NS}/${RECIPE_17_NAME}")
CASE17_TMPDIR="$(mktemp -d -t clerum-case17-XXXX)"
CASE17_SKIP=false
# Pre-flight: verify WRC reconciler is running
if ! kubectl get deploy workflow-recipes -n control-plane --no-headers 2>/dev/null | grep -q "1/1"; then
warn "Case 17: SKIPPED (WRC reconciler not running in control-plane)"
CASE17_SKIP=true
fi
if [[ "$CASE17_SKIP" == "false" ]]; then
pw_var="ADMIN_PASS""WORD"
[[ -n "${!pw_var:-}" ]] || fail "Case 17: E2E_ADMIN_PASS""WORD/ADMIN_PASS""WORD is required so the grant is created through the admin API"
admin_login
pass "Case 17: admin auth acquired for product grant seed"
# The parent WorkflowRecipe is trigger infrastructure only. The product
# trigger route creates the per-run child WorkflowRecipe, and WRC reconciles
# the child into mcp-host runtime pods/Secrets.
cat > "$CASE17_TMPDIR/recipe.yaml" <<YAML
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
name: ${RECIPE_17_NAME}
namespace: ${RECIPE_17_NS}
spec:
agent:
provider: ${E2E_WORKFLOW_MODEL_PROVIDER}
model: ${E2E_WORKFLOW_MODEL_NAME}
triggers:
onDemand:
requiresApproval: false
allowedActors:
- user
steps:
- id: gated-step
instruction: "Return exactly: E2E Case 17 approved"
timeoutSeconds: 120
requiresApproval:
target:
userId: "${APPROVER_USER_ID}"
message: "E2E Case 17 — approve to unblock step"
timeoutSeconds: 300
YAML
if kubectl apply -f "$CASE17_TMPDIR/recipe.yaml" 2>/dev/null; then
pass "Case 17: WorkflowRecipe ${RECIPE_17_NAME} applied"
http_request PUT "${CONTROL_URL}/api/v1/admin/workflows/${RECIPE_17_NS}/${RECIPE_17_NAME}/grants" \
"{\"userIds\":[\"${APPROVER_USER_ID}\"]}" \
"Cookie: control_ui_admin_session=${ADMIN_AUTH}"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 17: grant seed failed (HTTP $HTTP_STATUS): $HTTP_BODY"
GRANT_17_ROWS=$(pg_psql "SELECT count(*) FROM user_workflow_triggers WHERE recipe_namespace='${RECIPE_17_NS}' AND recipe_name='${RECIPE_17_NAME}' AND user_id='${APPROVER_USER_ID}';")
[[ "$GRANT_17_ROWS" == "1" ]] || fail "Case 17: grant API did not persist canonical user_workflow_triggers row"
pass "Case 17: approver grant persisted via admin API"
TRIGGER_BODY_17='{"inputs":{}}'
IDEMPO_KEY_17="e2e-case17-trigger-$(uuid)"
http_request POST "${EXT_URL}/api/v1/workflows/${RECIPE_17_NS}/${RECIPE_17_NAME}/trigger" "$TRIGGER_BODY_17" \
"Authorization: Bearer $APPROVER_TOKEN" "Idempotency-Key: $IDEMPO_KEY_17"
if [[ "$HTTP_STATUS" != "201" && "$HTTP_STATUS" != "200" ]]; then
fail "Case 17: external trigger failed (HTTP $HTTP_STATUS): $HTTP_BODY"
fi
RUN_ID_17=$(json_field "$HTTP_BODY" "o.id")
[[ -n "$RUN_ID_17" ]] || fail "Case 17: trigger response missing run id: $HTTP_BODY"
CREATED_WORKFLOW_RUN_IDS+=("$RUN_ID_17")
pass "Case 17: external trigger created workflow run ($RUN_ID_17)"
CHILD_17_NS=""
CHILD_17_NAME=""
for _ in $(seq 1 45); do
child_17_ref=$(pg_psql "SELECT child_recipe_namespace || '/' || child_recipe_name FROM workflow_runs WHERE run_id='${RUN_ID_17}' AND child_recipe_namespace IS NOT NULL AND child_recipe_name IS NOT NULL LIMIT 1;" 2>/dev/null | head -1 | tr -d ' ' || echo "")
if [[ "$child_17_ref" == */* ]]; then
CHILD_17_NS="${child_17_ref%%/*}"
CHILD_17_NAME="${child_17_ref#*/}"
break
fi
sleep 2
done
if [[ -z "$CHILD_17_NS" || -z "$CHILD_17_NAME" ]]; then
CHILD_17_NAME=$(kubectl -n "$RECIPE_17_NS" get workflowrecipe \
-l "clerum.io/parent-recipe=${RECIPE_17_NAME},clerum.io/workflow-run-id=${RUN_ID_17}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
if [[ -n "$CHILD_17_NAME" ]]; then
CHILD_17_NS="$RECIPE_17_NS"
CREATED_WORKFLOW_RECIPE_REFS+=("${CHILD_17_NS}/${CHILD_17_NAME}")
fail "Case 17: WRC created child WorkflowRecipe ${CHILD_17_NS}/${CHILD_17_NAME}, but workflow_runs did not attach it to run $RUN_ID_17"
fi
fail "Case 17: WRC did not attach a child WorkflowRecipe to run $RUN_ID_17"
fi
CREATED_WORKFLOW_RECIPE_REFS+=("${CHILD_17_NS}/${CHILD_17_NAME}")
pass "Case 17: WRC created child WorkflowRecipe (${CHILD_17_NS}/${CHILD_17_NAME})"
TOKEN_SECRET="wf-${CHILD_17_NAME}-mcp-host-runtime-tokens"
secret_found=false
for _ in $(seq 1 20); do
if kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" >/dev/null 2>&1; then
secret_found=true
break
fi
sleep 3
done
[[ "$secret_found" == "true" ]] || fail "Case 17: child mcpHost runtime Secret was not created within 60s"
pass "Case 17: child mcpHost runtime Secret created ($TOKEN_SECRET)"
runtime_access=$(
kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" \
-o go-template='{{ index .data "mcp-host-runtime-access-token" }}'
)
runtime_refresh=$(
kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" \
-o go-template='{{ index .data "mcp-host-runtime-refresh-token" }}'
)
runtime_control=$(
kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" \
-o go-template='{{ index .data "mcp-host-workflow-control-token" }}'
)
retired_key_prefix="approval"
retired_access_key="${retired_key_prefix}-access-token"
retired_refresh_key="${retired_key_prefix}-refresh-token"
retired_access=$(
kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" \
-o "jsonpath={.data['$retired_access_key']}"
)
retired_refresh=$(
kubectl -n "$CHILD_17_NS" get secret "$TOKEN_SECRET" \
-o "jsonpath={.data['$retired_refresh_key']}"
)
[[ -n "$runtime_access" ]] || fail "Case 17: missing mcp-host-runtime-access-token"
[[ -n "$runtime_refresh" ]] || fail "Case 17: missing mcp-host-runtime-refresh-token"
[[ -n "$runtime_control" ]] || fail "Case 17: missing mcp-host-workflow-control-token"
[[ -z "$retired_access" ]] || fail "Case 17: retired $retired_access_key still present"
[[ -z "$retired_refresh" ]] || fail "Case 17: retired $retired_refresh_key still present"
pass "Case 17: mcpHost runtime Secret uses canonical keys only"
approval_17_id=""
for _ in $(seq 1 40); do
approval_17_id=$(pg_psql "SELECT id FROM workflow_approval_requests WHERE recipe_namespace='${RECIPE_17_NS}' AND recipe_name IN ('${RECIPE_17_NAME}','${CHILD_17_NAME}') AND status='pending' ORDER BY requested_at DESC LIMIT 1;" 2>/dev/null | head -1 | tr -d ' ' || echo "")
[[ -n "$approval_17_id" ]] && break
sleep 3
done
[[ -n "$approval_17_id" ]] || fail "Case 17: gateStep did not create a pending approval within 120s"
CREATED_APPROVAL_IDS+=("$approval_17_id")
pass "Case 17: gateStep created approval ($approval_17_id)"
http_request POST "${EXT_URL}/api/v1/workflow-approvals/${approval_17_id}/decide" \
'{"decision":"approve","note":"E2E gating test"}' "Authorization: Bearer $APPROVER_TOKEN"
[[ "$HTTP_STATUS" == "200" ]] || fail "Case 17: decide failed (HTTP $HTTP_STATUS): $HTTP_BODY"
APPROVAL_17_STATUS=$(pg_psql "SELECT status FROM workflow_approval_requests WHERE id='${approval_17_id}';" 2>/dev/null | head -1 | tr -d ' ' || echo "")
[[ "$APPROVAL_17_STATUS" == "approved" || "$APPROVAL_17_STATUS" == "consumed" ]] \
|| fail "Case 17: approval persisted unexpected status after decision ($APPROVAL_17_STATUS)"
pass "Case 17: approval decided and persisted ($APPROVAL_17_STATUS)"
resumed=false
STEP_PHASE_17=""
WF_PHASE_17=""
WF_MESSAGE_17=""
for _ in $(seq 1 60); do
STEP_PHASE_17=$(kubectl -n "$CHILD_17_NS" get workflowrecipe "$CHILD_17_NAME" -o jsonpath='{.status.steps[0].phase}' 2>/dev/null || true)
WF_PHASE_17=$(kubectl -n "$CHILD_17_NS" get workflowrecipe "$CHILD_17_NAME" -o jsonpath='{.status.workflowExecution.phase}' 2>/dev/null || true)
WF_MESSAGE_17=$(kubectl -n "$CHILD_17_NS" get workflowrecipe "$CHILD_17_NAME" -o jsonpath='{.status.workflowExecution.message}' 2>/dev/null || true)
if [[ "$STEP_PHASE_17" == "completed" || "$WF_PHASE_17" == "completed" ]]; then
resumed=true
break
fi
if [[ "$STEP_PHASE_17" == "failed" || "$WF_PHASE_17" == "failed" ]]; then
if [[ "$WF_MESSAGE_17" != *"Approval denied"* && "$WF_MESSAGE_17" != *"Approval expired"* && "$WF_MESSAGE_17" != *"Approval cancelled"* && "$WF_MESSAGE_17" != *"Approval polling timed out"* ]]; then
resumed=true
break
fi
fi
sleep 3
done
[[ "$resumed" == "true" ]] || fail "Case 17: workflow did not resume after approval (step=${STEP_PHASE_17:-unknown}, workflow=${WF_PHASE_17:-unknown}, message=${WF_MESSAGE_17:-none})"
pass "Case 17: workflow resumed after approval (step=${STEP_PHASE_17:-unknown}, workflow=${WF_PHASE_17:-unknown})"
# Cleanup
kubectl delete workflowrecipe "$CHILD_17_NAME" -n "$CHILD_17_NS" --ignore-not-found --timeout=20s >/dev/null 2>&1 || true
kubectl delete workflowrecipe "$RECIPE_17_NAME" -n "$RECIPE_17_NS" --ignore-not-found --timeout=20s >/dev/null 2>&1 || true
pg_psql "DELETE FROM workflow_approval_requests WHERE recipe_namespace='${RECIPE_17_NS}' AND recipe_name IN ('${RECIPE_17_NAME}','${CHILD_17_NAME}');" >/dev/null 2>&1 || true
pg_psql "DELETE FROM workflow_runs WHERE run_id='${RUN_ID_17}';" >/dev/null 2>&1 || true
pg_psql "DELETE FROM user_workflow_triggers WHERE recipe_namespace='${RECIPE_17_NS}' AND recipe_name='${RECIPE_17_NAME}';" >/dev/null 2>&1 || true
else
fail "Case 17: kubectl apply failed"
fi
rm -rf "$CASE17_TMPDIR"
fi
echo ""
echo -e "${BOLD}=================================================================${NC}"
echo -e "${BOLD} Summary${NC}"
echo -e "${BOLD}=================================================================${NC}"
echo -e " PASS: ${GREEN}${PASS}${NC}"
echo -e " FAIL: ${RED}${FAIL}${NC}"
echo -e " TOTAL: ${TOTAL}"
echo ""
if [[ $FAIL -eq 0 ]]; then
echo -e "${GREEN}${BOLD}ALL USER-APPROVAL E2E CASES PASSED${NC}"
exit 0
else
echo -e "${RED}${BOLD}USER-APPROVAL E2E FAILED${NC}"
exit 1
fi