Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.0 KB
Raw Blame
#!/usr/bin/env bash
# e2e-stateless-cron-block.sh -- cron x stateless policy E2E (PR #735).
# Default/false forbids cron_manage create/enable without HITL or a live
# schedule. The tool/HITL policy is covered deterministically in unit tests
# because tool selection is model-driven in the live chat path. This T2 gate
# proves the runtime lifecycle contract with deterministic heartbeat evidence:
# activeCronSchedules blocks suspension, and clearing it releases suspension.
# It accelerates HCC cadences only with E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1
# and fails cleanup if the saved values are not restored and read back exactly.
#
# Registered gate: e2e-stateless-cron-block (run directly; not wired to Make).
set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
. "${HERE}/e2e-lib.sh"

require_safe_kube_context

now_ms() { python3 -c 'import time; print(int(time.time() * 1000))'; }

HOST_REF="${E2E_STATELESS_HOST_REF:-chatllm-stateless}"
EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"
RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"
DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:-test@clerum.io}"
DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"

MCP_HOST_NS="${MCP_HOST_NS:-mcp-host}"
HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
HCC_NS="${HCC_NS:-${CONTROL_NS:-control-plane}}"

E2E_TURN_TIMEOUT="${E2E_TURN_TIMEOUT:-120}"
POD_READY_TIMEOUT="${POD_READY_TIMEOUT:-180}"
SUSPEND_WINDOW="${SUSPEND_WINDOW:-180}"      # idle(60)+emitter+drain+poll+slack
BLOCK_HOLD="${BLOCK_HOLD:-150}"              # >= 2 heartbeat windows the host must stay up
REASON_WAIT="${REASON_WAIT:-120}"           # bounded wait for the SuspendBlocked reason

# Fast HCC cadence overlay (mirrors the wake-recovery gate).
TEST_IDLE_MINUTES="${TEST_IDLE_MINUTES:-1}"
TEST_IDLE_FLOOR_MINUTES="${TEST_IDLE_FLOOR_MINUTES:-1}"
TEST_DRAIN_GRACE_MS="${TEST_DRAIN_GRACE_MS:-20000}"
TEST_POLL_MS="${TEST_POLL_MS:-5000}"

RUN_ID="$$-$(now_ms)"
THREAD_ID="cron-block-${RUN_ID}"
SESSION_TOKEN=""; RPC_TOKEN=""
HCC_ENV_SAVED=""
CRON_ALLOW_ENV_SAVED=""
CRON_ALLOW_ENV_WAS_SET=0
CRON_ALLOW_ENV_TOUCHED=0
CRON_PIN_PID=""
CRON_POD_UID=""

# kctl(), require_safe_kube_context, header/ok/fail/log/print_results and
# ready_pod_name all come from e2e-lib.sh (sourced above).

lifecycle_state() { kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true; }
lifecycle_reason() { kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.status.lifecycle.reason}' 2>/dev/null || true; }
deployment_replicas() { kct=$(kctl get deployment "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo ""); printf '%s' "$kct"; }
current_ready_pod() { ready_pod_name "$MCP_HOST_NS" "app=${HOST_REF}" 2>/dev/null || true; }
pod_uid() { kctl get pod "$1" -n "$MCP_HOST_NS" -o jsonpath='{.metadata.uid}' 2>/dev/null || true; }

upsert_heartbeat_conditions() {
  local pod_uid_arg=$1 active_work=$2 conditions=$3 state=$4
  kctl -n control-plane exec deployment/control-postgres -- psql -U postgres -d profiles -v ON_ERROR_STOP=1     -c "INSERT INTO host_heartbeats (host_ref, pod_uid, active_work, conditions, last_activity_ts, state, received_at) VALUES (\$\$${HOST_REF}\$\$, \$\$${pod_uid_arg}\$\$, ${active_work}, \$\$${conditions}\$\$::jsonb, 0, \$\$${state}\$\$, NOW()) ON CONFLICT (host_ref) DO UPDATE SET pod_uid=EXCLUDED.pod_uid, active_work=EXCLUDED.active_work, conditions=EXCLUDED.conditions, last_activity_ts=EXCLUDED.last_activity_ts, state=EXCLUDED.state, received_at=NOW();" >/dev/null
}

stop_active_cron_pin() {
  local pod_uid_arg=${1:-}
  if [ -n "$CRON_PIN_PID" ]; then
    kill "$CRON_PIN_PID" 2>/dev/null || true
    wait "$CRON_PIN_PID" 2>/dev/null || true
    CRON_PIN_PID=""
  fi
  if [ -n "$pod_uid_arg" ]; then
    local clear_conditions='{"activeTask":false,"awaitingApproval":false,"pendingResults":false,"activeCronSchedules":false}'
    upsert_heartbeat_conditions "$pod_uid_arg" false "$clear_conditions" active >/dev/null 2>&1 || true
  fi
}

start_active_cron_pin() {
  local pod_uid_arg=$1
  local conditions='{"activeTask":false,"awaitingApproval":false,"pendingResults":false,"activeCronSchedules":true}'
  stop_active_cron_pin
  (
    while true; do
      upsert_heartbeat_conditions "$pod_uid_arg" true "$conditions" active >/dev/null 2>&1 || true
      sleep 2
    done
  ) &
  CRON_PIN_PID=$!
}

RPC_SCOPES_JSON='["host:message:invoke","host:approval:write","host:session:read","host:status:read","host:wake:write"]'

mint_session_token() {
  local resp code body
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST "${EXT_BASE}/api/v1/auth/password-login" \
    -H 'Content-Type: application/json' \
    -d "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')") || {
    echo "password-login request failed at ${EXT_BASE}" >&2; return 1; }
  code="$(echo "$resp" | tail -n1)"; body="$(echo "$resp" | sed '$d')"
  [ "$code" = "200" ] || { echo "password-login -> HTTP ${code}: ${body}" >&2; return 1; }
  SESSION_TOKEN="$(echo "$body" | jq -r '.token // empty')"
  [ -n "$SESSION_TOKEN" ] || { echo "password-login returned no .token" >&2; return 1; }
}

mint_rpc_token() {
  local resp code body
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST "${EXT_BASE}/api/v1/rpc/token" \
    -H "Authorization: Bearer ${SESSION_TOKEN}" -H 'Content-Type: application/json' \
    -d "$(jq -cn --arg h "$HOST_REF" --argjson s "$RPC_SCOPES_JSON" '{hostRefs:[$h],scopes:$s}')") || {
    echo "rpc/token request failed at ${EXT_BASE}" >&2; return 1; }
  code="$(echo "$resp" | tail -n1)"; body="$(echo "$resp" | sed '$d')"
  [ "$code" = "200" ] || { echo "rpc/token -> HTTP ${code}: ${body}" >&2; return 1; }
  RPC_TOKEN="$(echo "$body" | jq -r '.token // empty')"
  [ -n "$RPC_TOKEN" ] || { echo "rpc/token returned no .token" >&2; return 1; }
}

TURN_STATUS=""; TURN_BODY=""
send_turn_raw() {
  local content=$1 payload resp
  payload="$(jq -cn --arg c "$content" --arg t "$THREAD_ID" '{content:$c,threadId:$t}')"
  resp=$(curl -sS -m "$E2E_TURN_TIMEOUT" -w '\n%{http_code}' -X POST \
    "${RPC_BASE}/api/v1/rpc/hosts/${HOST_REF}/messages" \
    -H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json' \
    -d "$payload") || { echo "message POST failed at ${RPC_BASE}" >&2; return 1; }
  TURN_STATUS="$(echo "$resp" | tail -n1)"; TURN_BODY="$(echo "$resp" | sed '$d')"
}

post_wake() {
  local resp
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \
    "${RPC_BASE}/api/v1/rpc/hosts/${HOST_REF}/wake" \
    -H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json') || return 1
  WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"
  case "$WAKE_STATUS" in
    200|202) return 0 ;;
    *) echo "wake POST -> HTTP ${WAKE_STATUS}: ${WAKE_BODY}" >&2; return 1 ;;
  esac
}

wake_if_suspended() {
  local label=$1 st
  st="$(lifecycle_state)"
  if [ "$st" = "suspended" ]; then
    log "${label}: host suspended -- waking"
    mint_rpc_token || return 1
    post_wake || { fail "${label}: wake POST failed"; return 1; }
    wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || { fail "${label}: wake produced no Ready pod"; return 1; }
  fi
}

wait_for_ready_pod() {
  local timeout=$1 waited=0 pod
  while [ "$waited" -lt "$timeout" ]; do
    pod="$(current_ready_pod)"
    if [ -n "$pod" ]; then printf '%s' "$pod"; return 0; fi
    sleep 3; waited=$((waited + 3))
  done
  return 1
}

wait_for_suspended() {
  local timeout=$1 waited=0 st reps running
  while [ "$waited" -lt "$timeout" ]; do
    st="$(lifecycle_state)"; reps="$(deployment_replicas)"
    if ! running="$(running_pod_count "$MCP_HOST_NS" "app=${HOST_REF}")"; then
      echo "wait_for_suspended: failed to count Running pods for ${HOST_REF}" >&2
      return 1
    fi
    if [ "$st" = "suspended" ] && [ "${reps:-1}" = "0" ] && [ "${running:-1}" = "0" ]; then return 0; fi
    sleep 5; waited=$((waited + 5))
  done
  return 1
}

probe_pod_cron_policy_artifacts() {
  local expected_allow=$1 pod
  pod="$(current_ready_pod)"
  [ -n "$pod" ] || { fail "cron tool probe: no Ready pod"; return 1; }
  # Runs inside the deployed mcp-host image, but as a fresh Node process.
  # This proves shipped policy artifacts and env-gated behavior; it does not
  # claim to inspect the already-running scheduler instance.
  kctl exec -i -n "$MCP_HOST_NS" "$pod" -- node - "$expected_allow" < "${HERE}/_lib/stateless-cron-policy-probe.cjs"
}

save_and_set_hcc_cadences() {
  if [ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" != 1 ]; then
    fail "refusing to accelerate stateless lifecycle cadences; set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 on an owned profile"
    return 1
  fi
  header "PRECONDITION (labeled setup) -- HCC test cadences (idle=${TEST_IDLE_MINUTES}min, drain=${TEST_DRAIN_GRACE_MS}ms)"
  local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES \
    CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS)
  local k cur saved=""
  for k in "${keys[@]}"; do
    cur="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
      -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}" 2>/dev/null || true)"
    saved+="${k}=${cur}"$'\n'
  done
  HCC_ENV_SAVED="$saved"
  kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
    "CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=${TEST_IDLE_MINUTES}" \
    "CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=${TEST_IDLE_FLOOR_MINUTES}" \
    "CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=${TEST_DRAIN_GRACE_MS}" \
    "CONTEXT_MAPPER_HEARTBEAT_POLL_MS=${TEST_POLL_MS}" >/dev/null 2>&1 || {
    fail "failed to set HCC test cadences"; return 1; }
  kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || {
    fail "HCC rollout did not settle after cadence set"; return 1; }
  ok "HCC test cadences applied"
}

restore_hcc_cadences() {
  [ -n "$HCC_ENV_SAVED" ] || return 0
  log "restoring HCC cadence env"
  local args=() line k v actual
  while IFS= read -r line; do
    [ -z "$line" ] && continue
    k="${line%%=*}"; v="${line#*=}"
    if [ -n "$v" ]; then args+=("${k}=${v}"); else args+=("${k}-"); fi
  done <<< "$HCC_ENV_SAVED"
  if [ ${#args[@]} -eq 0 ]; then return 0; fi
  if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null 2>&1; then
    fail "failed to restore HCC cadence env"
    return 1
  fi
  if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=120s >/dev/null 2>&1; then
    fail "HCC rollout did not settle after cadence restore"
    return 1
  fi
  while IFS= read -r line; do
    [ -z "$line" ] && continue
    k="${line%%=*}"; v="${line#*=}"
    actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
      -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}" 2>/dev/null || true)"
    if [ "$actual" != "$v" ]; then
      fail "HCC cadence restore readback mismatch for ${k}: expected '${v}', got '${actual}'"
      return 1
    fi
  done <<< "$HCC_ENV_SAVED"
  HCC_ENV_SAVED=""
}

restore_cron_allow_env() {
  [ "$CRON_ALLOW_ENV_TOUCHED" = "1" ] || return 0
  local arg
  if [ "$CRON_ALLOW_ENV_WAS_SET" = "1" ]; then
    arg="CLERUM_STATELESS_ALLOW_CRON_MANAGE=${CRON_ALLOW_ENV_SAVED}"
  else
    arg="CLERUM_STATELESS_ALLOW_CRON_MANAGE-"
  fi
  log "restoring CLERUM_STATELESS_ALLOW_CRON_MANAGE on ${HOST_REF}"
  kctl set env "deployment/${HOST_REF}" -n "$MCP_HOST_NS" "$arg" >/dev/null 2>&1 || true
  kctl rollout status "deployment/${HOST_REF}" -n "$MCP_HOST_NS" --timeout=180s >/dev/null 2>&1 || true
}

set_stateless_cron_allow() {
  local value=$1 cur
  if [ "$CRON_ALLOW_ENV_TOUCHED" = "0" ]; then
    cur="$(kctl get "deployment/${HOST_REF}" -n "$MCP_HOST_NS" \
      -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='CLERUM_STATELESS_ALLOW_CRON_MANAGE')].value}" 2>/dev/null || true)"
    CRON_ALLOW_ENV_SAVED="$cur"
    [ -n "$cur" ] && CRON_ALLOW_ENV_WAS_SET=1 || CRON_ALLOW_ENV_WAS_SET=0
    CRON_ALLOW_ENV_TOUCHED=1
  fi
  kctl set env "deployment/${HOST_REF}" -n "$MCP_HOST_NS" \
    "CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value}" >/dev/null 2>&1 || {
    fail "failed to set CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value}"; return 1; }
  kctl rollout status "deployment/${HOST_REF}" -n "$MCP_HOST_NS" --timeout=180s >/dev/null 2>&1 || {
    fail "${HOST_REF} rollout did not settle after CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value}"; return 1; }
  wake_if_suspended "CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value}" || return 1
  wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || {
    fail "${HOST_REF} did not become Ready after CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value}"; return 1; }
  ok "CLERUM_STATELESS_ALLOW_CRON_MANAGE=${value} applied"
}

cleanup() {
  local rc=$?
  stop_active_cron_pin "$CRON_POD_UID" >/dev/null 2>&1 || true
  restore_cron_allow_env
  restore_hcc_cadences || rc=1
  print_results
  exit "$rc"
}
trap cleanup EXIT

# ---- Prereqs ---------------------------------------------------------------
header "Prerequisites"
for bin in jq curl kubectl; do
  command -v "$bin" >/dev/null 2>&1 || { fail "required tool '${bin}' not installed"; exit 1; }
done
kctl get host "$HOST_REF" -n "$MCP_HOST_NS" >/dev/null 2>&1 \
  && ok "Host '${HOST_REF}' exists" \
  || { fail "Host '${HOST_REF}' not found in ${MCP_HOST_NS}"; exit 1; }
stateless="$(kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.spec.lifecycle.stateless}' 2>/dev/null || true)"
[ "$stateless" = "true" ] && ok "Host is stateless (spec.lifecycle.stateless=true)" \
  || { fail "Host '${HOST_REF}' is not stateless -- cannot exercise cron-block"; exit 1; }
curl -sS -m 8 -o /dev/null "${EXT_BASE}/health" && ok "external-rest-api reachable" || { fail "external-rest-api NOT reachable at ${EXT_BASE}"; exit 1; }
curl -sS -m 8 -o /dev/null "${RPC_BASE}/health" && ok "rpc-proxy reachable" || { fail "rpc-proxy NOT reachable at ${RPC_BASE}"; exit 1; }
mint_session_token && ok "session token minted for ${DEV_EMAIL}" || { fail "cannot login as ${DEV_EMAIL}"; exit 1; }
mint_rpc_token && ok "user RPC token minted" || { fail "cannot mint RPC token for ${HOST_REF}"; exit 1; }

save_and_set_hcc_cadences || exit 1
wake_if_suspended "prereq" || exit 1

# ---- Baseline: the host converses ------------------------------------------
header "Baseline -- the host serves a normal turn"
send_turn_raw "Reply with exactly: CRON_BLOCK_BASELINE_${RUN_ID}"
[ "$TURN_STATUS" = "200" ] && ok "baseline turn served (200)" \
  || { fail "baseline turn did not serve (HTTP ${TURN_STATUS}: $(printf '%s' "$TURN_BODY" | head -c 200))"; exit 1; }

# ---- Default forbid has no live cron blocker -------------------------------
header "Default forbid -- no active cron blocker remains"
set_stateless_cron_allow false || exit 1
probe_pod_cron_policy_artifacts false && ok "pod-image cron_manage policy probe rejects create/enable in forbid mode" || exit 1
if wait_for_suspended "$SUSPEND_WINDOW"; then
  ok "default forbid left no active cron blocker; host suspended"
else
  fail "forbid mode did not suspend; state=$(lifecycle_state), replicas=$(deployment_replicas), reason=$(lifecycle_reason)"
  exit 1
fi

header "Allow=true -- activeCronSchedules pins the stateless host awake"
set_stateless_cron_allow true || exit 1
probe_pod_cron_policy_artifacts true && ok "pod-image cron_manage policy probe remains approval-gated and enabled in allow mode" || exit 1
wake_if_suspended "allow=true setup" || exit 1
cron_pod="$(current_ready_pod)"
[ -n "$cron_pod" ] || { fail "allow=true setup had no Ready pod"; exit 1; }
CRON_POD_UID="$(pod_uid "$cron_pod")"
[ -n "$CRON_POD_UID" ] || { fail "could not resolve pod UID for ${cron_pod}"; exit 1; }
start_active_cron_pin "$CRON_POD_UID"
ok "activeCronSchedules heartbeat pin started for pod ${cron_pod}"

log "informational: cron_manage registration, forbid/allow policy, HITL gate, imported-schedule cleanup, and CronScheduler->heartbeat supplier wiring are deterministic local coverage; this T2 phase proves HCC lifecycle response to activeCronSchedules heartbeat without depending on model tool choice"

# ---- BLOCKED: the schedule pins the host awake -----------------------------
header "An active schedule pins the host awake (SuspendBlocked: activeCronSchedules)"
reason_seen=0; waited=0
while [ "$waited" -lt "$REASON_WAIT" ]; do
  r="$(lifecycle_reason)"
  if [ "$r" = "SuspendBlocked: activeCronSchedules" ]; then reason_seen=1; break; fi
  st="$(lifecycle_state)"
  if [ "$st" = "suspended" ]; then
    fail "host suspended while activeCronSchedules=true -- the schedule blocker did NOT keep it awake"
    exit 1
  fi
  sleep 5; waited=$((waited + 5))
done
[ "$reason_seen" = "1" ] && ok "reason == 'SuspendBlocked: activeCronSchedules' -- the schedule is the blocker" \
  || { fail "reason never became 'SuspendBlocked: activeCronSchedules' within ${REASON_WAIT}s (last: '$(lifecycle_reason)')"; kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.status.lifecycle}' >&2 2>/dev/null || true; exit 1; }

# Sustained: across >= BLOCK_HOLD (multiple heartbeat windows) the host must
# NOT suspend while the schedule is active.
held=0; violated=0
while [ "$held" -lt "$BLOCK_HOLD" ]; do
  st="$(lifecycle_state)"; reps="$(deployment_replicas)"
  if [ "$st" = "suspended" ] || [ "${reps:-1}" = "0" ]; then violated=1; break; fi
  sleep 10; held=$((held + 10))
done
[ "$violated" = "0" ] && ok "host stayed up for ${BLOCK_HOLD}s with activeCronSchedules=true (never suspended, replicas stayed 1)" \
  || { fail "host suspended (state=${st}, replicas=${reps}) while activeCronSchedules=true -- the block failed"; exit 1; }

# ---- RELEASE: clear activeCronSchedules and the host suspends --------------
header "Clearing activeCronSchedules releases the host"
stop_active_cron_pin "$CRON_POD_UID"
ok "activeCronSchedules heartbeat pin cleared"

if wait_for_suspended "$SUSPEND_WINDOW"; then
  ok "host suspended after activeCronSchedules cleared (state=suspended, replicas=0, pod gone) -- the cron blocker WAS the blocker"
else
  fail "host did NOT suspend within ${SUSPEND_WINDOW}s after activeCronSchedules cleared (state=$(lifecycle_state), replicas=$(deployment_replicas), reason=$(lifecycle_reason))"
  exit 1
fi

# print_results is emitted by the EXIT trap (cleanup); do not double-print here.