Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
36.8 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
export RECIPE_NS="${RECIPE_NS:-sandbox-recipes}"
source "${SCRIPT_DIR}/e2e-lib.sh"
require_safe_kube_context

MCP_SERVER_NS="${MCP_SERVER_NS:-mcp-server}"
CONTROL_PORT="${CONTROL_API_PORT:-8090}"
CONTROL_URL="${E2E_CONTROL_API_URL:-http://127.0.0.1:${CONTROL_PORT}}"
E2E_ADMIN_AUTH="${E2E_CONTROL_API_ADMIN_TOKEN:-}"
ADMIN_USERNAME="${E2E_ADMIN_USERNAME:-${ADMIN_USERNAME:-admin}}"
ADMIN_PASSWORD="${E2E_ADMIN_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"

DIRECT_NAME="e2e-layer3a-snippet-direct-db"
DIRECT_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-direct-db.yaml"
API_MONGO_PG_NAME="e2e-layer3a-snippet-api-mongo-pg-chain"
API_MONGO_PG_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-api-mongo-postgres-chain.yaml"
API_MONGO_PG_SECRET_NAME="e2e-layer3a-snippet-api-mongo-pg-chain-secret"
HTTP_NAME="e2e-layer3a-snippet-http-egress"
HTTP_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-http-egress.yaml"
MANUAL_MCP_NAME="e2e-layer3a-snippet-manual-mcp"
MANUAL_MCP_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-manual-mcp.yaml"
MANUAL_MCP_TIMEOUT_NAME="e2e-layer3a-snippet-manual-mcp-timeout"
MANUAL_MCP_TIMEOUT_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-manual-mcp-timeout.yaml"
HYBRID_NAME="e2e-layer3a-hybrid-secret-pvc-5step-long-name"
HYBRID_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-hybrid-agentic.yaml"
SECRET_REF_NAME="e2e-layer3a-snippet-secret-ref"
SECRET_REF_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-secret-ref.yaml"
SECRET_REF_SECRET_NAME="e2e-layer3a-snippet-secret-ref-secret"

NEG_PLATFORM_NAME="e2e-layer3a-snippet-negative-platform-secret"
NEG_PLATFORM_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-platform-secret.yaml"
NEG_MISSING_SECRET_NAME="e2e-layer3a-snippet-negative-missing-secret"
NEG_MISSING_SECRET_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-missing-secret.yaml"
NEG_HTTP_EGRESS_NAME="e2e-layer3a-snippet-negative-http-egress"
NEG_HTTP_EGRESS_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-http-egress.yaml"
NEG_HTTP_ADMISSION_NAME="e2e-layer3a-snippet-negative-http-admission"
NEG_HTTP_ADMISSION_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-http-admission.yaml"
NEG_MCP_WILDCARD_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-mcp-wildcard.yaml"
NEG_UNDECLARED_POSTGRES_NAME="e2e-layer3a-snippet-negative-undeclared-postgres"
NEG_UNDECLARED_POSTGRES_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-undeclared-postgres.yaml"
NEG_UNSAFE_ARTIFACT_NAME="e2e-layer3a-snippet-negative-unsafe-artifact"
NEG_UNSAFE_ARTIFACT_FILE="${PROJECT_DIR}/tests/e2e/fixtures/layer3a-snippet-negative-unsafe-artifact.yaml"

workflow_service_name() {
  local recipe_name=$1 suffix=$2
  RECIPE_NAME="$recipe_name" SERVICE_SUFFIX="$suffix" python3 - <<'PY'
import hashlib
import os

max_len = 63
prefix = "wf-"
recipe_name = os.environ["RECIPE_NAME"]
suffix = os.environ["SERVICE_SUFFIX"]
direct = f"{prefix}{recipe_name}{suffix}"
if len(direct) <= max_len:
    print(direct)
    raise SystemExit(0)

digest = hashlib.sha256(recipe_name.encode()).hexdigest()[:8]
reserved_len = len(prefix) + len(suffix) + len(digest) + 1
max_stem_len = max(1, max_len - reserved_len)
stem = recipe_name[:max_stem_len].rstrip("-") or recipe_name[:max_stem_len]
print(f"{prefix}{stem}-{digest}{suffix}")
PY
}

snippet_runner_service_name() {
  workflow_service_name "$1" "-snippet-runner"
}

artifact_reader_service_name() {
  workflow_service_name "$1" "-artifact-reader"
}

ALL_RECIPES=(
  "$DIRECT_NAME"
  "$API_MONGO_PG_NAME"
  "$HTTP_NAME"
  "$MANUAL_MCP_NAME"
  "$MANUAL_MCP_TIMEOUT_NAME"
  "$HYBRID_NAME"
  "$SECRET_REF_NAME"
  "$NEG_PLATFORM_NAME"
  "$NEG_MISSING_SECRET_NAME"
  "$NEG_HTTP_EGRESS_NAME"
  "$NEG_HTTP_ADMISSION_NAME"
  e2e-layer3a-snippet-negative-mcp-wildcard
  "$NEG_UNDECLARED_POSTGRES_NAME"
  "$NEG_UNSAFE_ARTIFACT_NAME"
)

cleanup_one() {
  local name=$1
  local children child
  children=$(kctl get workflowrecipe -n "$RECIPE_NS" \
    -l "clerum.io/parent-recipe=${name}" \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
  for child in $children; do
    kctl delete workflowrecipe "$child" -n "$RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true
  done
  kctl delete workflowrecipe "$name" -n "$RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true
  wait_for_workflowrecipe_deleted "$RECIPE_NS" "$name" "$TIMEOUT_DELETE" >/dev/null 2>&1 || true
  for ns in "$SANDBOX_NS" "$MCP_SERVER_NS" "$RECIPE_NS"; do
    kctl delete pod,svc,configmap,secret,networkpolicy,pvc,statefulset,deployment,job \
      -n "$ns" -l "clerum.io/recipe=${name}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
    for child in $children; do
      kctl delete pod,svc,configmap,secret,networkpolicy,pvc,statefulset,deployment,job \
        -n "$ns" -l "clerum.io/recipe=${child}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
    done
  done
  kctl delete mcpserver,context -n "$MCP_SERVER_NS" -l "clerum.io/recipe=${name}" \
    --ignore-not-found --wait=false >/dev/null 2>&1 || true
  for child in $children; do
    kctl delete mcpserver,context -n "$MCP_SERVER_NS" -l "clerum.io/recipe=${child}" \
      --ignore-not-found --wait=false >/dev/null 2>&1 || true
  done
  kctl delete context "wf-${name}" -n "$MCP_SERVER_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true
}

create_secret_ref_secret() {
  cat <<YAML | kctl apply -f - >/dev/null
apiVersion: v1
kind: Secret
metadata:
  name: ${SECRET_REF_SECRET_NAME}
  namespace: ${SANDBOX_NS}
  labels:
    clerum.io/recipe: ${SECRET_REF_NAME}
    clerum.io/owner-recipe: ${SECRET_REF_NAME}
    clerum.io/test-fixture: layer3a-snippet-secret-ref
type: Opaque
stringData:
  apiKey: e2e-secret-token-value
YAML
  ok "${SECRET_REF_NAME} test Secret created in ${SANDBOX_NS}"
}

create_api_mongo_pg_secret() {
  cat <<YAML | kctl apply -f - >/dev/null
apiVersion: v1
kind: Secret
metadata:
  name: ${API_MONGO_PG_SECRET_NAME}
  namespace: ${SANDBOX_NS}
  labels:
    clerum.io/recipe: ${API_MONGO_PG_NAME}
    clerum.io/owner-recipe: ${API_MONGO_PG_NAME}
    clerum.io/test-fixture: layer3a-snippet-api-mongo-pg-chain
type: Opaque
stringData:
  apiKey: e2e-vendor-api-key-value
  pgPassword: e2e-postgres-password
YAML
  ok "${API_MONGO_PG_NAME} API and PostgreSQL Secret created in ${SANDBOX_NS}"
}

cleanup_all() {
  for name in "${ALL_RECIPES[@]}"; do
    cleanup_one "$name"
  done
}

if [ "${1:-}" = "--cleanup-only" ]; then
  cleanup_all
  exit 0
fi

if [ -z "$E2E_ADMIN_AUTH" ] && [ -z "$ADMIN_PASSWORD" ]; then
  fail "E2E_CONTROL_API_ADMIN_TOKEN or E2E_ADMIN_PASSWORD is required for the full snippet runtime gate because the hybrid agentic case must trigger a real on-demand child run"
  exit 1
fi

trap cleanup_all EXIT

wait_for_phase() {
  local name=$1 expected=$2 timeout=${3:-300} elapsed=0 phase=""
  while [ "$elapsed" -lt "$timeout" ]; do
    phase=$(kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o jsonpath='{.status.workflowExecution.phase}' 2>/dev/null || true)
    [ "$phase" = "$expected" ] && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  echo "last phase for ${name}: ${phase:-<empty>}"
  return 1
}

wait_for_labeled_pod_ready() {
  local namespace=$1 label=$2 timeout=${3:-240} elapsed=0 ready=""
  while [ "$elapsed" -lt "$timeout" ]; do
    ready=$(kctl get pods -n "$namespace" -l "$label" \
      -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
    [ "$ready" = "True" ] && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  kctl get pods -n "$namespace" -l "$label" 2>/dev/null || true
  return 1
}

wait_for_named_pod_ready() {
  local namespace=$1 pod_name=$2 timeout=${3:-240} elapsed=0 ready=""
  while [ "$elapsed" -lt "$timeout" ]; do
    ready=$(kctl get pod "$pod_name" -n "$namespace" \
      -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
    [ "$ready" = "True" ] && return 0
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  kctl get pod "$pod_name" -n "$namespace" 2>/dev/null || true
  return 1
}

apply_recipe() {
  local name=$1 file=$2
  cleanup_one "$name"
  kctl apply -f "$file" >/dev/null
  ok "WorkflowRecipe '${name}' applied in ${RECIPE_NS}"
}

curl_config_quote() {
  local value=$1
  value="${value//\\/\\\\}"
  value="${value//\"/\\\"}"
  printf '%s' "$value"
}

http_json() {
  local method=$1 url=$2 body=${3:-}
  shift 3 2>/dev/null || shift $#

  local cfg body_file raw rc header_file
  cfg=$(mktemp "${TMPDIR:-/tmp}/clerum-snippet-e2e-curl.XXXXXX")
  chmod 600 "$cfg"
  header_file="${HTTP_HEADER_FILE:-}"
  body_file=""
  if [ -n "$body" ]; then
    body_file=$(mktemp "${TMPDIR:-/tmp}/clerum-snippet-e2e-body.XXXXXX")
    chmod 600 "$body_file"
    printf '%s' "$body" >"$body_file"
  fi

  {
    printf 'silent\n'
    printf 'show-error\n'
    printf 'write-out = "\\n%%{http_code}"\n'
    printf 'max-time = 30\n'
    printf 'request = "%s"\n' "$(curl_config_quote "$method")"
    printf 'url = "%s"\n' "$(curl_config_quote "$url")"
    [ -n "$header_file" ] && printf 'dump-header = "%s"\n' "$(curl_config_quote "$header_file")"
    printf 'header = "Content-Type: application/json"\n'
    for hdr in "$@"; do
      printf 'header = "%s"\n' "$(curl_config_quote "$hdr")"
    done
    [ -n "$body_file" ] && printf 'data-binary = "@%s"\n' "$(curl_config_quote "$body_file")"
  } >"$cfg"

  raw=$(curl --config "$cfg" 2>/dev/null)
  rc=$?
  rm -f "$cfg" "$body_file"
  if [ "$rc" -ne 0 ]; then
    HTTP_STATUS="000"
    HTTP_BODY='{"error":"curl failed"}'
    return 1
  fi
  HTTP_STATUS=$(printf '%s' "$raw" | tail -n1)
  HTTP_BODY=$(printf '%s' "$raw" | sed '$d')
}


admin_session_from_headers() {
  local header_file=$1
  awk '
    BEGIN { IGNORECASE = 1 }
    /^[Ss]et-[Cc]ookie:[[:space:]]*control_ui_admin_session=/ {
      sub(/^[Ss]et-[Cc]ookie:[[:space:]]*control_ui_admin_session=/, "")
      sub(/;.*/, "")
      print
      exit
    }
  ' "$header_file"
}
json_get() {
  local json=$1 path=$2
  JSON_INPUT="$json" JSON_PATH="$path" python3 - <<'PY'
import json
import os

value = json.loads(os.environ["JSON_INPUT"])
for key in os.environ["JSON_PATH"].split("."):
    if not key:
        continue
    if not isinstance(value, dict):
        value = None
        break
    value = value.get(key)
if value is None:
    value = ""
print(value)
PY
}

require_admin_auth() {
  if [ -z "$E2E_ADMIN_AUTH" ]; then
    local body header_file
    body=$(ADMIN_USERNAME="$ADMIN_USERNAME" ADMIN_PASSWORD="$ADMIN_PASSWORD" node --no-warnings -e '
const body = {
  username: process.env.ADMIN_USERNAME,
  password: process.env.ADMIN_PASSWORD,
}
process.stdout.write(JSON.stringify(body))
')
    header_file=$(mktemp "${TMPDIR:-/tmp}/clerum-snippet-e2e-headers.XXXXXX")
    chmod 600 "$header_file"
    if ! HTTP_HEADER_FILE="$header_file" http_json POST "${CONTROL_URL}/api/v1/admin/auth/login" "$body"; then
      rm -f "$header_file"
      fail "admin login request failed for hybrid on-demand workflow (HTTP ${HTTP_STATUS}): ${HTTP_BODY}" >&2
      exit 1
    fi
    if [ "$HTTP_STATUS" != "200" ]; then
      rm -f "$header_file"
      fail "admin login failed for hybrid on-demand workflow (HTTP ${HTTP_STATUS})" >&2
      exit 1
    fi
    E2E_ADMIN_AUTH=$(json_get "$HTTP_BODY" token)
    if [ -z "$E2E_ADMIN_AUTH" ]; then
      E2E_ADMIN_AUTH=$(admin_session_from_headers "$header_file")
    fi
    rm -f "$header_file"
    if [ -z "$E2E_ADMIN_AUTH" ]; then
      fail "admin login response missing token for hybrid on-demand workflow" >&2
      exit 1
    fi
    log "admin JWT obtained for hybrid on-demand workflow" >&2
  fi
  printf '%s' "$E2E_ADMIN_AUTH"
}

trigger_workflow_as_admin() {
  local name=$1 auth=$2 body idempotency_key run_id
  body='{"inputs":{"requestId":"layer3a-hybrid-agentic-e2e"}}'
  idempotency_key="snippet-hybrid-${name}-$(date +%s)-${RANDOM}"
  if ! http_json POST "${CONTROL_URL}/api/v1/admin/workflows/${RECIPE_NS}/${name}/trigger" \
    "$body" \
    "Authorization: Bearer ${auth}" \
    "Idempotency-Key: ${idempotency_key}"; then
    fail "admin trigger request failed for ${name} (HTTP ${HTTP_STATUS}): ${HTTP_BODY}"
    exit 1
  fi
  if [ "$HTTP_STATUS" != "201" ] && [ "$HTTP_STATUS" != "200" ]; then
    fail "admin trigger failed for ${name} (HTTP ${HTTP_STATUS}): ${HTTP_BODY}"
    exit 1
  fi
  run_id=$(json_get "$HTTP_BODY" id)
  if [ -z "$run_id" ]; then
    fail "admin trigger response missing run id for ${name}: ${HTTP_BODY}"
    exit 1
  fi
  printf '%s' "$run_id"
}

wait_for_child_recipe_by_run_id() {
  local run_id=$1 timeout=${2:-180} elapsed=0 child=""
  while [ "$elapsed" -lt "$timeout" ]; do
    child=$(kctl get workflowrecipe -n "$RECIPE_NS" \
      -l "clerum.io/workflow-run-id=${run_id}" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
    if [ -n "$child" ]; then
      printf '%s' "$child"
      return 0
    fi
    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

assert_snippet_runner_contract() {
  local name=$1
  local pod="${name}-snippet-runner"
  if wait_for_named_pod_ready "$SANDBOX_NS" "$pod" 240; then
    ok "${name} snippet runner pod is ready"
  else
    fail "${name} snippet runner pod did not become ready"
    kctl describe pod "$pod" -n "$SANDBOX_NS" 2>/dev/null || true
    exit 1
  fi

  kctl get svc "$(snippet_runner_service_name "$name")" -n "$SANDBOX_NS" >/dev/null
  ok "${name} snippet runner service exists"

  local pod_json
  pod_json=$(kctl get pod "$pod" -n "$SANDBOX_NS" -o json)
  if POD_JSON="$pod_json" python3 - <<'PY'
import json
import os

pod = json.loads(os.environ["POD_JSON"])
spec = pod["spec"]
container = spec["containers"][0]
env_names = {item["name"] for item in container.get("env", [])}
if spec.get("automountServiceAccountToken") is not False:
    raise SystemExit("service account token must be disabled")
sc = container.get("securityContext", {})
pod_sc = spec.get("securityContext", {})
if pod_sc.get("runAsNonRoot") is not True:
    raise SystemExit("pod runAsNonRoot must be true")
if sc.get("readOnlyRootFilesystem") is not True:
    raise SystemExit("readOnlyRootFilesystem must be true")
if sc.get("allowPrivilegeEscalation") is not False:
    raise SystemExit("allowPrivilegeEscalation must be false")
if "ALL" not in sc.get("capabilities", {}).get("drop", []):
    raise SystemExit("all Linux capabilities must be dropped")
if "CLERUM_WORKFLOW_NAME" not in env_names:
    raise SystemExit("workflow binding is missing")
mounts = {item["name"] for item in container.get("volumeMounts", [])}
for required in ["workflow-config", "recipe-output", "tmp"]:
    if required not in mounts:
        raise SystemExit(f"required mount missing: {required}")
PY
  then
    ok "${name} snippet runner pod hardening and env contract are correct"
  else
    fail "${name} snippet runner pod contract mismatch"
    exit 1
  fi
}

assert_coordinator_snippet_contract() {
  local name=$1
  local env_names
  env_names=$(kctl get pod "${name}-coordinator" -n "$SANDBOX_NS" -o jsonpath='{.spec.containers[0].env[*].name}' 2>/dev/null || true)
  if printf "%s\n" "$env_names" | tr ' ' '\n' | grep -Fxq CLERUM_SNIPPET_RUNNER_URL; then
    ok "${name} coordinator receives snippet runner URL"
  else
    fail "${name} coordinator missing snippet runner URL"
    exit 1
  fi
}

assert_no_mcp_host() {
  local name=$1
  if kctl get pod "${name}-mcp-host" -n "$SANDBOX_NS" >/dev/null 2>&1; then
    fail "${name} unexpectedly created mcp-host pod"
    exit 1
  fi
  ok "${name} did not create mcp-host pod"
}

assert_artifact_reader_exists() {
  local name=$1
  if kctl get pod "${name}-artifact-reader" -n "$SANDBOX_NS" >/dev/null 2>&1 &&
     kctl get svc "$(artifact_reader_service_name "$name")" -n "$SANDBOX_NS" >/dev/null 2>&1; then
    ok "${name} has platform artifact-reader for output downloads"
  else
    fail "${name} missing platform artifact-reader"
    exit 1
  fi
}

assert_public_http_egress_policy() {
  local name=$1 component=$2 policy_name=$3
  local pod_json
  local policy_json
  local all_policies_json
  pod_json=$(kctl get pod "${name}-snippet-runner" -n "$SANDBOX_NS" -o json)
  policy_json=$(kctl get networkpolicy "$policy_name" -n "$SANDBOX_NS" -o json)
  all_policies_json=$(kctl get networkpolicy -n "$SANDBOX_NS" -o json)
  if POD_JSON="$pod_json" POLICY_JSON="$policy_json" ALL_POLICIES_JSON="$all_policies_json" COMPONENT="$component" python3 - <<'PY'
import json
import os
import ipaddress

pod = json.loads(os.environ["POD_JSON"])
policy = json.loads(os.environ["POLICY_JSON"])
all_policies = json.loads(os.environ["ALL_POLICIES_JSON"]).get("items", [])
component = os.environ["COMPONENT"]
selector = policy.get("spec", {}).get("podSelector", {}).get("matchLabels", {})
if selector.get("clerum.io/component") != component:
    raise SystemExit(f"component selector mismatch: {selector}")
pod_labels = pod.get("metadata", {}).get("labels", {})

blocked = [
    "0.0.0.0/8",
    "10.0.0.0/8",
    "100.64.0.0/10",
    "127.0.0.0/8",
    "169.254.0.0/16",
    "172.16.0.0/12",
    "192.0.0.0/24",
    "192.0.2.0/24",
    "192.31.196.0/24",
    "192.52.193.0/24",
    "192.88.99.0/24",
    "192.168.0.0/16",
    "192.175.48.0/24",
    "198.18.0.0/15",
    "198.51.100.0/24",
    "203.0.113.0/24",
    "224.0.0.0/4",
    "240.0.0.0/4",
]
blocked_nets = [ipaddress.ip_network(item) for item in blocked]

def is_public_resolved_cidr(cidr):
    try:
        network = ipaddress.ip_network(cidr, strict=False)
    except ValueError:
        return False
    return (
        network.version == 4
        and network.prefixlen == 32
        and not any(network.subnet_of(blocked_net) or network.overlaps(blocked_net) for blocked_net in blocked_nets)
    )

resolved_cidrs = set(
    item.strip()
    for item in policy.get("metadata", {})
    .get("annotations", {})
    .get("clerum.io/runtime-http-egress-current-cidrs", "")
    .split(",")
    if item.strip()
)
previous_cidrs = set(
    item.strip()
    for item in policy.get("metadata", {})
    .get("annotations", {})
    .get("clerum.io/runtime-http-egress-previous-cidrs", "")
    .split(",")
    if item.strip()
)
effective_resolved_cidrs = resolved_cidrs | previous_cidrs

def selector_matches(selector_spec, labels):
    selector_spec = selector_spec or {}
    for key, value in (selector_spec.get("matchLabels") or {}).items():
        if labels.get(key) != value:
            return False
    for expression in selector_spec.get("matchExpressions") or []:
        key = expression.get("key")
        operator = expression.get("operator")
        values = expression.get("values") or []
        if not key or not operator:
            raise SystemExit(f"unsupported podSelector expression: {expression}")
        has_key = key in labels
        label_value = labels.get(key)
        if operator == "In":
            if not has_key or label_value not in values:
                return False
        elif operator == "NotIn":
            if has_key and label_value in values:
                return False
        elif operator == "Exists":
            if not has_key:
                return False
        elif operator == "DoesNotExist":
            if has_key:
                return False
        else:
            raise SystemExit(f"unsupported podSelector operator {operator}: {expression}")
    return True

def rule_ports(rule):
    ports = rule.get("ports")
    if not ports:
        return None
    return {(item.get("protocol", "TCP"), item.get("port")) for item in ports}

def rule_has_http_surface(rule):
    ports = rule_ports(rule)
    if ports is None:
        return True
    return bool({("TCP", 80), ("TCP", 443)} & ports)

def cidrs_for_rule(rule):
    cidrs = set()
    for target in rule.get("to", []):
        block = target.get("ipBlock")
        if not block or set(target.keys()) != {"ipBlock"}:
            raise SystemExit(f"HTTP egress peer must be ipBlock-only, got: {target}")
        cidr = block.get("cidr")
        if not cidr:
            raise SystemExit(f"HTTP egress ipBlock missing cidr: {target}")
        cidrs.add(cidr)
    return cidrs

def is_resolved_public_http_rule(rule, require_annotation):
    cidrs = cidrs_for_rule(rule)
    if not cidrs or not all(is_public_resolved_cidr(cidr) for cidr in cidrs):
        return False
    if require_annotation:
        if not resolved_cidrs:
            raise SystemExit("runtime HTTP egress current CIDR annotation missing")
        if cidrs != effective_resolved_cidrs:
            raise SystemExit(
                f"resolved CIDR annotation mismatch: policy={cidrs} current={resolved_cidrs} previous={previous_cidrs}"
            )
    return True

def is_safe_public_http_rule(rule, require_annotation=False):
    ports = rule_ports(rule)
    if ports is None:
        return False
    if not {("TCP", 80), ("TCP", 443)} & ports:
        return None
    if ports != {("TCP", 80), ("TCP", 443)}:
        return False
    if is_resolved_public_http_rule(rule, require_annotation):
        return True
    return False

matching_safe_rule_found = False
for rule in policy.get("spec", {}).get("egress", []):
    if is_safe_public_http_rule(rule, require_annotation=True):
        matching_safe_rule_found = True

if not matching_safe_rule_found:
    raise SystemExit("public HTTP egress rule not found")

primary_name = policy.get("metadata", {}).get("name")
for item in all_policies:
    item_selector = item.get("spec", {}).get("podSelector", {})
    if not selector_matches(item_selector, pod_labels):
        continue
    item_name = item.get("metadata", {}).get("name", "<unknown>")
    for index, rule in enumerate(item.get("spec", {}).get("egress", [])):
        if not rule_has_http_surface(rule):
            continue
        if item_name != primary_name:
            raise SystemExit(
                f"unexpected HTTP egress surface from NetworkPolicy {item_name} selecting snippet pod egress[{index}]: {rule}"
            )
        safe = is_safe_public_http_rule(rule, require_annotation=True)
        if safe is False:
            raise SystemExit(f"unsafe HTTP egress rule in NetworkPolicy {item_name} egress[{index}]: {rule}")
PY
  then
    ok "${name} ${component} public HTTP egress NetworkPolicy is pinned to resolved public CIDRs"
  else
    fail "${name} ${component} public HTTP egress NetworkPolicy mismatch"
    kctl get networkpolicy "$policy_name" -n "$SANDBOX_NS" -o yaml 2>/dev/null || true
    exit 1
  fi
}

assert_snippet_secret_env_contract() {
  local name=$1 alias=$2 secret_name=$3 secret_key=$4
  local pod_json
  pod_json=$(kctl get pod "${name}-snippet-runner" -n "$SANDBOX_NS" -o json)
  if POD_JSON="$pod_json" ALIAS="$alias" SECRET_NAME="$secret_name" SECRET_KEY="$secret_key" python3 - <<'PY'
import json
import os

pod = json.loads(os.environ["POD_JSON"])
alias = os.environ["ALIAS"]
secret_name = os.environ["SECRET_NAME"]
secret_key = os.environ["SECRET_KEY"]
expected_env = "CLERUM_SNIPPET_SECRET_" + "".join(
    ch if ch.isalnum() or ch == "_" else "_" for ch in alias
).upper()
env = {item.get("name"): item for item in pod["spec"]["containers"][0].get("env", [])}
entry = env.get(expected_env)
if not entry:
    raise SystemExit(f"missing secret env var {expected_env}")
ref = entry.get("valueFrom", {}).get("secretKeyRef", {})
if ref.get("name") != secret_name or ref.get("key") != secret_key:
    raise SystemExit(f"secretKeyRef mismatch: {ref}")
PY
  then
    ok "${name} snippet runner receives ${alias} through the declared Secret keyRef"
  else
    fail "${name} snippet runner secret env contract mismatch"
    exit 1
  fi
}

assert_status_contract() {
  local name=$1 expected_json=$2
  local status_json
  status_json=$(kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o json)
  if STATUS_JSON="$status_json" EXPECTED_JSON="$expected_json" python3 - <<'PY'
import json
import os

doc = json.loads(os.environ["STATUS_JSON"])
expected = json.loads(os.environ["EXPECTED_JSON"])
steps = {step.get("id"): step for step in doc.get("status", {}).get("steps", [])}
for step_id, executor in expected["executors"].items():
    step = steps.get(step_id)
    if not step:
        raise SystemExit(f"missing step status: {step_id}")
    if step.get("phase") != "completed":
        raise SystemExit(f"{step_id} phase mismatch: {step.get('phase')}")
    if step.get("executor") != executor:
        raise SystemExit(f"{step_id} executor mismatch: {step.get('executor')}")
    output = step.get("output", "")
    for needle in expected.get("outputContains", {}).get(step_id, []):
        if needle not in output:
            raise SystemExit(f"{step_id} output missing {needle!r}: {output}")
artifacts = {item.get("name"): item for item in doc.get("status", {}).get("artifacts", [])}
for artifact_name in expected.get("artifacts", []):
    artifact = artifacts.get(artifact_name)
    if not artifact:
        raise SystemExit(f"missing status artifact: {artifact_name}")
    if artifact.get("path") != f"/output/{artifact_name}":
        raise SystemExit(f"artifact path mismatch for {artifact_name}: {artifact}")
PY
  then
    ok "${name} WorkflowRecipe status proves expected executors, outputs, and artifacts"
  else
    fail "${name} status contract mismatch"
    kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o yaml 2>/dev/null || true
    exit 1
  fi
}

apply_and_expect_failed() {
  local name=$1 file=$2 pattern=$3 label=$4 timeout=${5:-180}
  cleanup_one "$name"
  kctl apply -f "$file" >/dev/null
  if wait_for_phase "$name" failed "$timeout"; then
    ok "${label} fails closed"
  else
    fail "${label} did not fail closed"
    kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o yaml 2>/dev/null || true
    exit 1
  fi
  local doc
  doc=$(kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o json 2>/dev/null || true)
  if DOC_JSON="$doc" PATTERN="$pattern" python3 - <<'PY'
import json
import os
import re

doc = json.loads(os.environ["DOC_JSON"])
haystack = json.dumps(doc.get("status", {}))
if not re.search(os.environ["PATTERN"], haystack, re.IGNORECASE):
    raise SystemExit(haystack)
PY
  then
    ok "${label} failure message/status matches expected guard"
  else
    fail "${label} failure message/status mismatch"
    kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o yaml 2>/dev/null || true
    exit 1
  fi
}

apply_and_expect_rejected() {
  local file=$1 pattern=$2 label=$3 output
  output=$(mktemp)
  if kctl apply -f "$file" >"$output" 2>&1; then
    fail "CRD accepted ${label}"
    cat "$output"
    rm -f "$output"
    exit 1
  fi
  if grep -Eqi "$pattern" "$output"; then
    ok "CRD rejects ${label}"
  else
    fail "CRD rejection for ${label} did not match expected message"
    cat "$output"
    rm -f "$output"
    exit 1
  fi
  rm -f "$output"
}

assert_failed_status_matches() {
  local name=$1 pattern=$2 label=$3
  local doc
  doc=$(kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o json 2>/dev/null || true)
  if DOC_JSON="$doc" PATTERN="$pattern" python3 - <<'PY'
import json
import os
import re

doc = json.loads(os.environ["DOC_JSON"])
haystack = json.dumps(doc.get("status", {}))
if not re.search(os.environ["PATTERN"], haystack, re.IGNORECASE):
    raise SystemExit(haystack)
PY
  then
    ok "${label} failure message/status matches expected guard"
  else
    fail "${label} failure message/status mismatch"
    kctl get workflowrecipe "$name" -n "$RECIPE_NS" -o yaml 2>/dev/null || true
    exit 1
  fi
}

kctl cluster-info >/dev/null
kctl get ns "$SANDBOX_NS" >/dev/null
kctl get ns "$MCP_SERVER_NS" >/dev/null
kctl get crd workflowrecipes.clerum.io >/dev/null
kctl get deploy workflow-recipes -n "$CONTROL_NS" >/dev/null
ok "snippet runtime prerequisites available"

cleanup_all

apply_recipe "$DIRECT_NAME" "$DIRECT_FILE"
assert_snippet_runner_contract "$DIRECT_NAME"
assert_coordinator_snippet_contract "$DIRECT_NAME"
assert_no_mcp_host "$DIRECT_NAME"
if wait_for_phase "$DIRECT_NAME" completed 420; then
  ok "direct DB snippet workflow completed"
else
  fail "direct DB snippet workflow did not complete"
  exit 1
fi
assert_artifact_reader_exists "$DIRECT_NAME"
assert_status_contract "$DIRECT_NAME" '{
  "executors": {"query-mongo":"snippet","query-postgres":"snippet"},
  "outputContains": {"query-mongo":["dao-alpha","mongo","insertedCount"],"query-postgres":["dao-alpha","postgres","insertedCount","mongoRows"]},
  "artifacts": ["direct-db-mongo.json","direct-db-summary.md"]
}'

cleanup_one "$API_MONGO_PG_NAME"
create_api_mongo_pg_secret
kctl apply -f "$API_MONGO_PG_FILE" >/dev/null
ok "WorkflowRecipe '${API_MONGO_PG_NAME}' applied in ${RECIPE_NS}"
if kctl get workflowrecipe "$API_MONGO_PG_NAME" -n "$RECIPE_NS" -o yaml \
  | grep -Eq "e2e-vendor-api-key-value|e2e-postgres-password"; then
  fail "${API_MONGO_PG_NAME} leaked Secret values into the WorkflowRecipe CRD"
  exit 1
fi
ok "${API_MONGO_PG_NAME} WorkflowRecipe stores only Secret references, not API key or DB password values"
assert_snippet_runner_contract "$API_MONGO_PG_NAME"
assert_coordinator_snippet_contract "$API_MONGO_PG_NAME"
assert_no_mcp_host "$API_MONGO_PG_NAME"
assert_snippet_secret_env_contract \
  "$API_MONGO_PG_NAME" \
  "vendor_api_key" \
  "$API_MONGO_PG_SECRET_NAME" \
  "apiKey"
assert_snippet_secret_env_contract \
  "$API_MONGO_PG_NAME" \
  "pg_password" \
  "$API_MONGO_PG_SECRET_NAME" \
  "pgPassword"
assert_public_http_egress_policy \
  "$API_MONGO_PG_NAME" \
  "workflow-snippet-runner" \
  "${API_MONGO_PG_NAME}-snippet-runner-egress"
if wait_for_phase "$API_MONGO_PG_NAME" completed 480; then
  ok "public API -> MongoDB -> PostgreSQL snippet workflow completed"
else
  fail "public API -> MongoDB -> PostgreSQL snippet workflow did not complete"
  kctl logs "${API_MONGO_PG_NAME}-snippet-runner" -n "$SANDBOX_NS" --tail=120 2>/dev/null || true
  exit 1
fi
assert_artifact_reader_exists "$API_MONGO_PG_NAME"
if kctl get workflowrecipe "$API_MONGO_PG_NAME" -n "$RECIPE_NS" -o json \
  | grep -Eq "e2e-vendor-api-key-value|e2e-postgres-password"; then
  fail "${API_MONGO_PG_NAME} leaked Secret values into WorkflowRecipe status"
  exit 1
fi
ok "${API_MONGO_PG_NAME} status/artifacts metadata does not expose API key or DB password values"
assert_status_contract "$API_MONGO_PG_NAME" '{
  "executors": {"fetch-api":"snippet","load-mongo":"snippet","copy-postgres":"snippet","emit-report":"snippet"},
  "outputContains": {
    "fetch-api":["api.ipify.org","ipify","apiKeyResolved"],
    "load-mongo":["mongoRows","ipify"],
    "copy-postgres":["postgres","copied"],
    "emit-report":["reportReady","postgresRows"]
  },
  "artifacts": ["api-ip.json","api-to-mongo.json","mongo-to-postgres.json","api-mongo-postgres-summary.md"]
}'

apply_recipe "$HTTP_NAME" "$HTTP_FILE"
assert_snippet_runner_contract "$HTTP_NAME"
assert_coordinator_snippet_contract "$HTTP_NAME"
assert_no_mcp_host "$HTTP_NAME"
assert_public_http_egress_policy "$HTTP_NAME" "workflow-snippet-runner" "${HTTP_NAME}-snippet-runner-egress"
if wait_for_phase "$HTTP_NAME" completed 300; then
  ok "public HTTP snippet workflow completed"
else
  fail "public HTTP snippet workflow did not complete"
  kctl logs "${HTTP_NAME}-snippet-runner" -n "$SANDBOX_NS" --tail=120 2>/dev/null || true
  exit 1
fi
assert_status_contract "$HTTP_NAME" '{
  "executors": {"call-public-api":"snippet"},
  "outputContains": {"call-public-api":["api.ipify.org","ipify","observedIp"]},
  "artifacts": ["http-egress-result.json"]
}'

apply_recipe "$MANUAL_MCP_NAME" "$MANUAL_MCP_FILE"
if wait_for_labeled_pod_ready "$MCP_SERVER_NS" "clerum.io/recipe=${MANUAL_MCP_NAME},clerum.io/workload=mock-tools" 240; then
  ok "manual MCP snippet transport workload is ready"
else
  fail "manual MCP snippet transport workload did not become ready"
  exit 1
fi
assert_snippet_runner_contract "$MANUAL_MCP_NAME"
assert_coordinator_snippet_contract "$MANUAL_MCP_NAME"
assert_no_mcp_host "$MANUAL_MCP_NAME"
if wait_for_phase "$MANUAL_MCP_NAME" completed 360; then
  ok "manual MCP snippet workflow completed"
else
  fail "manual MCP snippet workflow did not complete"
  exit 1
fi
assert_status_contract "$MANUAL_MCP_NAME" '{
  "executors": {"call-mcp":"snippet"},
  "outputContains": {"call-mcp":["mcpDataUsed","mock-tools","add","record","recall","context-write-ok","42"]},
  "artifacts": ["manual-mcp-result.json"]
}'

apply_recipe "$MANUAL_MCP_TIMEOUT_NAME" "$MANUAL_MCP_TIMEOUT_FILE"
if wait_for_labeled_pod_ready "$MCP_SERVER_NS" "clerum.io/recipe=${MANUAL_MCP_TIMEOUT_NAME},clerum.io/workload=mock-tools" 240; then
  ok "manual MCP timeout snippet transport workload is ready"
else
  fail "manual MCP timeout snippet transport workload did not become ready"
  exit 1
fi
assert_snippet_runner_contract "$MANUAL_MCP_TIMEOUT_NAME"
assert_coordinator_snippet_contract "$MANUAL_MCP_TIMEOUT_NAME"
assert_no_mcp_host "$MANUAL_MCP_TIMEOUT_NAME"
if wait_for_phase "$MANUAL_MCP_TIMEOUT_NAME" failed 180; then
  ok "manual MCP snippet timeout fails boundedly"
else
  fail "manual MCP snippet timeout did not fail boundedly"
  kctl logs "${MANUAL_MCP_TIMEOUT_NAME}-snippet-runner" -n "$SANDBOX_NS" --tail=120 2>/dev/null || true
  kctl get workflowrecipe "$MANUAL_MCP_TIMEOUT_NAME" -n "$RECIPE_NS" -o yaml 2>/dev/null || true
  exit 1
fi
assert_failed_status_matches "$MANUAL_MCP_TIMEOUT_NAME" "timeout|timed out|aborted|step-timeout" "manual MCP snippet timeout"

apply_recipe "$HYBRID_NAME" "$HYBRID_FILE"
if wait_for_labeled_pod_ready "$MCP_SERVER_NS" "clerum.io/recipe=${HYBRID_NAME},clerum.io/workload=mock-tools" 240; then
  ok "hybrid snippet MCP workload is ready"
else
  fail "hybrid snippet MCP workload did not become ready"
  exit 1
fi

HYBRID_ADMIN_TOKEN=$(require_admin_auth)
HYBRID_RUN_ID=$(trigger_workflow_as_admin "$HYBRID_NAME" "$HYBRID_ADMIN_TOKEN")
ok "hybrid snippet workflow triggered through admin onDemand run ${HYBRID_RUN_ID:0:8}"
if HYBRID_CHILD_NAME=$(wait_for_child_recipe_by_run_id "$HYBRID_RUN_ID" 180); then
  ok "hybrid snippet child WorkflowRecipe '${HYBRID_CHILD_NAME}' created for run ${HYBRID_RUN_ID:0:8}"
else
  fail "hybrid snippet child WorkflowRecipe was not created for run ${HYBRID_RUN_ID}"
  exit 1
fi
if wait_for_named_pod_ready "$SANDBOX_NS" "${HYBRID_CHILD_NAME}-mcp-host" 240; then
  ok "hybrid snippet workflow created mcp-host broker pod"
else
  fail "hybrid snippet workflow did not create ready mcp-host broker pod"
  exit 1
fi
assert_snippet_runner_contract "$HYBRID_CHILD_NAME"
assert_coordinator_snippet_contract "$HYBRID_CHILD_NAME"
if wait_for_phase "$HYBRID_CHILD_NAME" completed 420; then
  ok "hybrid snippet workflow completed"
else
  fail "hybrid snippet workflow did not complete"
  exit 1
fi
assert_status_contract "$HYBRID_CHILD_NAME" '{
  "executors": {"prepare":"snippet","agentic-review":"agentic","finalize":"snippet"},
  "outputContains": {"prepare":["prepared"],"agentic-review":["42"],"finalize":["prepared","42"]},
  "artifacts": ["hybrid-prepare.json","hybrid-agentic-summary.md"]
}'

cleanup_one "$SECRET_REF_NAME"
create_secret_ref_secret
kctl apply -f "$SECRET_REF_FILE" >/dev/null
ok "WorkflowRecipe '${SECRET_REF_NAME}' applied in ${RECIPE_NS}"
if kctl get workflowrecipe "$SECRET_REF_NAME" -n "$RECIPE_NS" -o yaml | grep -Fq "e2e-secret-token-value"; then
  fail "${SECRET_REF_NAME} leaked the Secret value into the WorkflowRecipe CRD"
  exit 1
fi
ok "${SECRET_REF_NAME} WorkflowRecipe stores only secretRef metadata, not Secret values"
assert_snippet_runner_contract "$SECRET_REF_NAME"
assert_coordinator_snippet_contract "$SECRET_REF_NAME"
assert_no_mcp_host "$SECRET_REF_NAME"
assert_snippet_secret_env_contract \
  "$SECRET_REF_NAME" \
  "vendor_api_key" \
  "$SECRET_REF_SECRET_NAME" \
  "apiKey"
if wait_for_phase "$SECRET_REF_NAME" completed 300; then
  ok "snippet secretRef workflow completed"
else
  fail "snippet secretRef workflow did not complete"
  kctl logs "${SECRET_REF_NAME}-snippet-runner" -n "$SANDBOX_NS" --tail=120 2>/dev/null || true
  exit 1
fi
assert_status_contract "$SECRET_REF_NAME" '{
  "executors": {"read-secret":"snippet"},
  "outputContains": {"read-secret":["secretRef","tokenPresent","tokenLength","tokenPrefix","e2e-"]},
  "artifacts": ["secret-ref-result.json"]
}'

apply_and_expect_failed \
  "$NEG_PLATFORM_NAME" \
  "$NEG_PLATFORM_FILE" \
  "cannot reference platform-managed secret" \
  "snippet platform-managed credential reference"

apply_and_expect_failed \
  "$NEG_MISSING_SECRET_NAME" \
  "$NEG_MISSING_SECRET_FILE" \
  "snippet secret .* was not found" \
  "snippet missing Kubernetes Secret reference"

apply_and_expect_failed \
  "$NEG_HTTP_EGRESS_NAME" \
  "$NEG_HTTP_EGRESS_FILE" \
  "HTTP host is not allowed" \
  "snippet HTTP call to undeclared host"

apply_and_expect_rejected \
  "$NEG_HTTP_ADMISSION_FILE" \
  "snippet HTTP .*allowedHosts must be declared in spec.runtimeEgress.http.allowedHosts" \
  "snippet HTTP host absent from runtimeEgress"

apply_and_expect_rejected \
  "$NEG_MCP_WILDCARD_FILE" \
  "snippet mcp allowedTools.include must not contain wildcards" \
  "snippet MCP wildcard allowedTools"

apply_and_expect_failed \
  "$NEG_UNDECLARED_POSTGRES_NAME" \
  "$NEG_UNDECLARED_POSTGRES_FILE" \
  "references undeclared postgres workload" \
  "snippet undeclared PostgreSQL workload"

apply_and_expect_failed \
  "$NEG_UNSAFE_ARTIFACT_NAME" \
  "$NEG_UNSAFE_ARTIFACT_FILE" \
  "unsafe artifact filename" \
  "snippet unsafe artifact filename"

print_results