Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
24.9 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E — Registry Full Lifecycle Tests (Push → Install → Execute → Upgrade → Uninstall)
# ═══════════════════════════════════════════════════════════════════════
# Validates the complete registry lifecycle for MCP servers AND recipes:
#   1. Push entries to registry → verify DB storage
#   2. Install MCP servers (local + remote) → verify CRDs + pods
#   3. Install recipe → execute workflow → verify output files
#   4. Upgrade MCP server + recipe → verify version bump
#   5. Uninstall all → verify complete cleanup
#
# Prerequisites:
#   make minikube-status && context = clerum-test
#   Port-forwards: control-api:8090, registry-api:8085
#   Admin setup done (POST /admin/auth/setup)

set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/e2e-lib.sh"

CONTROL_API="http://localhost:8090"
REGISTRY_API="http://localhost:8085"
MCP_NS="mcp-server"
SANDBOX_NS="sandbox-recipes"
POLL_SEC=10
WORKFLOW_TIMEOUT=360
POD_TIMEOUT=120

# ─── Version constants ───────────────────────────────────────────────
# Re-publish of removed entries is allowed (spec §12.1 + partial unique index),
# so we use stable versions. The cleanup loop below soft-deletes stale entries.
E2E_VER="1.0.0"
E2E_VER2="2.0.0"

# ─── State tracking (populated during tests) ─────────────────────────
LOCAL_SERVER_NAME=""
REMOTE_SERVER_NAME=""
RECIPE_NAME=""

# ─── Auth ─────────────────────────────────────────────────────────────
ADMIN_USER="${ADMIN_USER:-admin}"
ADMIN_PASS="${ADMIN_PASS:-changeme123!}"
_TOKEN=""
get_token() {
  [ -n "$_TOKEN" ] && { echo "$_TOKEN"; return; }
  _TOKEN=$(curl -s -X POST "${CONTROL_API}/api/v1/admin/auth/login" \
    -H "Content-Type: application/json" \
    -d "{\"username\":\"${ADMIN_USER}\",\"password\":\"${ADMIN_PASS}\"}" 2>/dev/null | jq -r '.token // empty')
  echo "$_TOKEN"
}

api() {
  local method=$1 path=$2 data=${3:-}
  local t; t=$(get_token)
  [ -z "$t" ] && { echo '{"error":"auth_failed"}'; return 1; }
  if [ -n "$data" ]; then
    curl -s -X "$method" "${CONTROL_API}${path}" \
      -H "Content-Type: application/json" -H "Authorization: Bearer $t" \
      -d "$data" 2>/dev/null
  else
    curl -s -X "$method" "${CONTROL_API}${path}" \
      -H "Authorization: Bearer $t" 2>/dev/null
  fi
}

reg() {
  local method=$1 path=$2 data=${3:-}
  if [ -n "$data" ]; then
    curl -s -X "$method" "${REGISTRY_API}${path}" \
      -H "Content-Type: application/json" \
      -d "$data" 2>/dev/null
  else
    curl -s -X "$method" "${REGISTRY_API}${path}" 2>/dev/null
  fi
}

wait_for_phase() {
  local name=$1 expected=$2 timeout=$3 elapsed=0
  while [ $elapsed -lt "$timeout" ]; do
    local phase
    phase=$(kubectl get workflowrecipe "$name" -n "$SANDBOX_NS" \
      -o jsonpath='{.status.workflowExecution.phase}' 2>/dev/null || echo "unknown")
    [ "$phase" = "$expected" ] && return 0
    [ "$phase" = "failed" ] && [ "$expected" != "failed" ] && {
      local msg; msg=$(kubectl get workflowrecipe "$name" -n "$SANDBOX_NS" \
        -o jsonpath='{.status.workflowExecution.message}' 2>/dev/null || echo "")
      warn "Workflow failed: $msg"
      return 1
    }
    sleep "$POLL_SEC"
    elapsed=$((elapsed + POLL_SEC))
  done
  warn "Timeout (${timeout}s) waiting for phase=$expected (current=$(kubectl get workflowrecipe "$name" -n "$SANDBOX_NS" -o jsonpath='{.status.workflowExecution.phase}' 2>/dev/null || echo 'unknown'))"
  return 1
}

# ─── Valid McpServer CRD bundle (base64) ──────────────────────────────
VALID_BUNDLE=$(printf 'apiVersion: clerum.io/v1alpha1\nkind: McpServer\nmetadata:\n  name: e2e-mock-mcp\nspec:\n  image: clerum/mock-mcp-server:test\n  transport:\n    type: streamableHttp\n    port: 3000\n' | base64 | tr -d '\n')

# ─── Cleanup stale K8s resources from previous runs ──────────────────
# Delete any leftover McpServers, recipes, secrets and deployments created by e2e.
kubectl get mcpserver -n "$MCP_NS" -o name 2>/dev/null | grep -E "e2e-|mock-mcp" | while read -r r; do
  kubectl delete "$r" -n "$MCP_NS" --ignore-not-found 2>/dev/null || true
done
kubectl get workflowrecipe -n "$SANDBOX_NS" -o name 2>/dev/null | grep -E "e2e-|mock-mcp" | while read -r r; do
  kubectl delete "$r" -n "$SANDBOX_NS" --ignore-not-found 2>/dev/null || true
done
kubectl get deploy -n "$MCP_NS" -o name 2>/dev/null | grep -E "e2e-|mock-mcp" | while read -r r; do
  kubectl delete "$r" -n "$MCP_NS" --ignore-not-found 2>/dev/null || true
done
kubectl get secret -n "$MCP_NS" -o name 2>/dev/null | grep "e2e-" | while read -r r; do
  kubectl delete "$r" -n "$MCP_NS" --ignore-not-found 2>/dev/null || true
done
# Cleanup sandbox pods
kubectl delete pods -n "$SANDBOX_NS" -l "clerum.io/managed-by=wrc" --ignore-not-found 2>/dev/null || true
sleep 3

# ─── Cleanup stale registry entries from previous runs ───────────────
for _name in e2e-mock-mcp e2e-remote-mcp e2e-workflow-recipe test-push; do
  for _ver in "1.0.0" "2.0.0"; do
    curl -s -X DELETE "${REGISTRY_API}/api/v1/entries/${_name}/versions/${_ver}" >/dev/null 2>&1 || true
  done
done

# ═══════════════════════════════════════════════════════════════════════
header "Registry Full Lifecycle E2E Tests"
# ═══════════════════════════════════════════════════════════════════════

# Detect context
CONTEXT_NAME=$(kubectl get contexts -n "$MCP_NS" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
[ -z "$CONTEXT_NAME" ] && { fail "No Context CRD found in $MCP_NS namespace"; exit 1; }
log "Using context: $CONTEXT_NAME"

# ═══ Group 1: Push entries to registry ════════════════════════════════
header "Group 1: Push Entries to Registry"

# 1a. Local MCP server (uses mock-mcp-server image already in minikube)
PUSH_LOCAL=$(curl -s -X POST "${REGISTRY_API}/api/v1/entries" \
  -H "Content-Type: application/json" \
  -d "{
  \"name\": \"e2e-mock-mcp\",
  \"version\": \"1.0.0\",
  \"entryType\": \"mcp-server\",
  \"description\": \"E2E test MCP server with echo/add tools\",
  \"author\": \"e2e-test\",
  \"origin\": \"human-authored\",
  \"category\": \"testing\",
  \"tags\": [\"e2e\", \"mock\"],
  \"contentCreatorTag\": \"1st-party\",
  \"configCreatorTag\": \"1st-party\",
  \"mcpServer\": {
    \"serverMode\": \"local\",
    \"transport\": \"streamableHttp\",
    \"port\": 3000,
    \"imageRef\": \"clerum/mock-mcp-server:test\",
    \"tools\": [\"echo\", \"add\", \"multiply\", \"get_time\", \"read_env\"]
  },
  \"bundle\": \"${VALID_BUNDLE}\"
}" 2>/dev/null || echo '{"error":"request_failed"}')
echo "$PUSH_LOCAL" | jq -e '.id' >/dev/null 2>&1 && ok "Pushed local MCP: e2e-mock-mcp v${E2E_VER}" || fail "Failed to push local MCP: $(echo "$PUSH_LOCAL" | jq -r '.error // .')"

# 1b. Remote MCP server
PUSH_REMOTE=$(reg POST "/api/v1/entries" "{
  \"name\": \"e2e-remote-mcp\",
  \"version\": \"${E2E_VER}\",
  \"entryType\": \"mcp-server\",
  \"description\": \"E2E remote MCP server test\",
  \"author\": \"e2e-test\",
  \"origin\": \"human-authored\",
  \"category\": \"testing\",
  \"tags\": [\"e2e\", \"remote\"],
  \"contentCreatorTag\": \"1st-party\",
  \"configCreatorTag\": \"1st-party\",
  \"mcpServer\": {
    \"serverMode\": \"remote\",
    \"transport\": \"streamableHttp\",
    \"remoteEndpoints\": [{\"url\": \"https://mcp.example.com/sse\", \"region\": \"us\"}],
    \"credentialSchema\": {
      \"required\": true,
      \"authType\": \"api-key\",
      \"keys\": [{\"name\": \"API_KEY\", \"label\": \"API Key\", \"kind\": \"api-key\", \"semanticType\": \"plain-string\", \"description\": \"Test key\"}]
    }
  }
}")
echo "$PUSH_REMOTE" | jq -e '.id' >/dev/null 2>&1 && ok "Pushed remote MCP: e2e-remote-mcp v${E2E_VER}" || fail "Failed to push remote MCP"

# 1c. Workflow recipe (use jq to properly escape YAML→JSON)
RECIPE_V1_YAML=$(cat <<'YAMLEOF'
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
  name: e2e-workflow-recipe
spec:
  description: E2E lifecycle test
  agent:
    providerHint: zai
    modelHint: glm-4.7
  steps:
    - id: compute
      instruction: "Use the echo tool with text hello-e2e and the add tool with a=10 b=32. Return the results."
    - id: generate-output
      instruction: "Using the clerum__generate_markdown tool, create a file called e2e-result.md with content: Test passed. Echo result: {{compute:output}}"
      dependsOn:
        - compute
YAMLEOF
)
PUSH_RECIPE=$(reg POST "/api/v1/entries" "$(jq -n \
  --arg ver "$E2E_VER" \
  --arg recipe "$RECIPE_V1_YAML" \
  '{name: "e2e-workflow-recipe", version: $ver, entryType: "recipe",
    description: "E2E lifecycle test recipe", author: "e2e-test",
    origin: "human-authored", category: "workflow", tags: ["e2e"],
    contentCreatorTag: "1st-party", configCreatorTag: "1st-party",
    recipe: $recipe}')")
echo "$PUSH_RECIPE" | jq -e '.id' >/dev/null 2>&1 && ok "Pushed recipe: e2e-workflow-recipe v${E2E_VER}" || fail "Failed to push recipe"

# 1d. Verify in search
SEARCH_COUNT=$(reg GET "/api/v1/entries?q=e2e&limit=10" | jq -r '.data | length // 0')
[ "$SEARCH_COUNT" -ge 3 ] && ok "Search finds $SEARCH_COUNT e2e entries" || fail "Expected >=3 e2e entries, got $SEARCH_COUNT"

# 1e. Verify latest version
LATEST=$(reg GET "/api/v1/entries/e2e-mock-mcp" | jq -r '.version // empty')
[ "$LATEST" = "1.0.0" ] && ok "Latest version: e2e-mock-mcp v${E2E_VER}" || fail "Wrong latest: $LATEST"

# ═══ Group 2: Install Local MCP from Registry ════════════════════════
header "Group 2: Local MCP Server Install"

INSTALL=$(api POST "/api/v1/admin/registry/install" "{
  \"contextRef\": \"${CONTEXT_NAME}\",
  \"registryEntryName\": \"e2e-mock-mcp\",
  \"registryEntryVersion\": \"1.0.0\"
}")
LOCAL_SERVER_NAME=$(echo "$INSTALL" | jq -r '.serverName // empty')

if [ -n "$LOCAL_SERVER_NAME" ]; then
  ok "Installed: $LOCAL_SERVER_NAME"

  # Verify labels
  LABELS=$(kubectl get mcpserver "$LOCAL_SERVER_NAME" -n "$MCP_NS" -o jsonpath='{.metadata.labels}' 2>/dev/null || echo "{}")
  [ "$(echo "$LABELS" | jq -r '.["clerum.io/catalog-id"] // empty')" = "e2e-mock-mcp" ] && ok "Label catalog-id" || fail "Missing catalog-id"
  [ "$(echo "$LABELS" | jq -r '.["clerum.io/server-mode"] // empty')" = "local" ] && ok "Label server-mode=local" || fail "Wrong server-mode"

  # Wait for pod — HCC labels Deployment with app={serverName}
  log "Waiting for MCP pod (${POD_TIMEOUT}s)..."
  if wait_for_pod "$MCP_NS" "app=$LOCAL_SERVER_NAME" "$POD_TIMEOUT" 2>/dev/null; then
    ok "Pod ready for $LOCAL_SERVER_NAME"
  else
    fail "Pod not ready for $LOCAL_SERVER_NAME (HCC did not create Deployment within ${POD_TIMEOUT}s)"
    # Diagnostic output
    kubectl get deployments -n "$MCP_NS" 2>/dev/null | head -5 || true
    kubectl logs deployment/host-context-controller -n control-plane --tail=20 2>/dev/null | grep -iE "$LOCAL_SERVER_NAME|error" || true
  fi

  # Context allowlist
  CTX_SERVERS=$(kubectl get contexts "$CONTEXT_NAME" -n "$MCP_NS" -o jsonpath='{.spec.mcpServers}' 2>/dev/null || echo "[]")
  echo "$CTX_SERVERS" | grep -q "$LOCAL_SERVER_NAME" && ok "Context allowlist updated" || fail "Not in context allowlist"
else
  fail "Install failed: $(echo "$INSTALL" | jq -r '.error // .')"
fi

# ═══ Group 3: Install Remote MCP from Registry ═══════════════════════
header "Group 3: Remote MCP Server Install"

RINSTALL=$(api POST "/api/v1/admin/registry/install" "{
  \"serverName\": \"e2e-remote-srv\",
  \"contextRef\": \"${CONTEXT_NAME}\",
  \"registryEntryName\": \"e2e-remote-mcp\",
  \"registryEntryVersion\": \"1.0.0\",
  \"credentials\": {\"API_KEY\": \"e2e-test-key-value\"}
}")
REMOTE_SERVER_NAME=$(echo "$RINSTALL" | jq -r '.serverName // empty')

if [ "$REMOTE_SERVER_NAME" = "e2e-remote-srv" ]; then
  ok "Installed remote: $REMOTE_SERVER_NAME"

  # Labels
  MODE=$(kubectl get mcpserver "$REMOTE_SERVER_NAME" -n "$MCP_NS" -o jsonpath='{.metadata.labels.clerum\.io/server-mode}' 2>/dev/null || echo "")
  [ "$MODE" = "remote" ] && ok "Label server-mode=remote" || fail "Wrong mode: $MODE"

  # Secret
  kubectl get secret "${REMOTE_SERVER_NAME}-credentials" -n "$MCP_NS" >/dev/null 2>&1 && ok "Credentials Secret exists" || fail "Secret missing"

  # Remote spec
  BASE_URL=$(kubectl get mcpserver "$REMOTE_SERVER_NAME" -n "$MCP_NS" -o jsonpath='{.spec.remote.baseUrl}' 2>/dev/null || echo "")
  [ -n "$BASE_URL" ] && ok "Remote baseUrl: $BASE_URL" || fail "Missing remote.baseUrl"
else
  fail "Remote install failed: $(echo "$RINSTALL" | jq -r '.error // .')"
fi

# ═══ Group 4: Recipe Install + Workflow Execution ═════════════════════
header "Group 4: Recipe Install + Execution + Output"

RINSTALL=$(api POST "/api/v1/admin/registry/install-recipe" "{
  \"registryEntryName\": \"e2e-workflow-recipe\",
  \"registryEntryVersion\": \"1.0.0\"
}")
RECIPE_NAME=$(echo "$RINSTALL" | jq -r '.recipeName // empty')

if [ -n "$RECIPE_NAME" ]; then
  ok "Recipe installed: $RECIPE_NAME"

  # CRD labels
  RLABELS=$(kubectl get workflowrecipe "$RECIPE_NAME" -n "$SANDBOX_NS" -o jsonpath='{.metadata.labels}' 2>/dev/null || echo "{}")
  [ "$(echo "$RLABELS" | jq -r '.["clerum.io/catalog-id"] // empty')" = "e2e-workflow-recipe" ] && ok "Recipe label catalog-id" || fail "Missing recipe catalog-id"

  # Verify workflow pods are created in sandbox-recipes (namespace splitting)
  log "Waiting for workflow pods in $SANDBOX_NS (${POD_TIMEOUT}s)..."
  COORD_READY=false
  MCP_HOST_READY=false
  _elapsed=0
  while [ $_elapsed -lt "$POD_TIMEOUT" ]; do
    _coord=$(kubectl get pods -n "$SANDBOX_NS" -l "clerum.io/recipe=${RECIPE_NAME},clerum.io/component=workflow-coordinator" \
      -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "")
    _mhost=$(kubectl get pods -n "$SANDBOX_NS" -l "clerum.io/recipe=${RECIPE_NAME},clerum.io/component=workflow-mcp-host" \
      -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "")
    [ -n "$_coord" ] && COORD_READY=true
    [ -n "$_mhost" ] && MCP_HOST_READY=true
    [ "$COORD_READY" = true ] && [ "$MCP_HOST_READY" = true ] && break
    sleep "$POLL_SEC"
    _elapsed=$((_elapsed + POLL_SEC))
  done
  [ "$COORD_READY" = true ] && ok "Coordinator pod created in $SANDBOX_NS" || fail "Coordinator pod not found in $SANDBOX_NS"
  [ "$MCP_HOST_READY" = true ] && ok "mcp-host pod created in $SANDBOX_NS" || fail "mcp-host pod not found in $SANDBOX_NS"

  # Wait for workflow execution
  log "Waiting for workflow completion (${WORKFLOW_TIMEOUT}s)..."
  if wait_for_phase "$RECIPE_NAME" "completed" "$WORKFLOW_TIMEOUT"; then
    ok "Workflow completed"

    # Verify steps
    STEPS=$(kubectl get workflowrecipe "$RECIPE_NAME" -n "$SANDBOX_NS" -o jsonpath='{.status.steps}' 2>/dev/null || echo "[]")
    COMPUTE_PHASE=$(echo "$STEPS" | jq -r '.[] | select(.id=="compute") | .phase // "unknown"')
    OUTPUT_PHASE=$(echo "$STEPS" | jq -r '.[] | select(.id=="generate-output") | .phase // "unknown"')
    [ "$COMPUTE_PHASE" = "completed" ] && ok "Step compute: completed" || fail "Step compute: $COMPUTE_PHASE"
    [ "$OUTPUT_PHASE" = "completed" ] && ok "Step generate-output: completed" || fail "Step generate-output: $OUTPUT_PHASE"

    # Verify compute output
    COMPUTE_OUT=$(echo "$STEPS" | jq -r '.[] | select(.id=="compute") | .output // ""')
    echo "$COMPUTE_OUT" | grep -qi "hello" && ok "Compute output contains echo result" || warn "Echo result not found in output"
    echo "$COMPUTE_OUT" | grep -q "42" && ok "Compute output contains 10+32=42" || warn "Add result not found"

    # Verify output file — WRC creates pods with clerum.io/recipe label in sandbox-recipes
    MCP_HOST_POD=$(kubectl get pods -n "$SANDBOX_NS" \
      -l "clerum.io/recipe=${RECIPE_NAME},clerum.io/component=workflow-mcp-host" \
      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
    if [ -n "$MCP_HOST_POD" ]; then
      ok "mcp-host pod found: $MCP_HOST_POD in $SANDBOX_NS"
      # Output file depends on LLM calling clerum__generate_markdown tool correctly.
      # This is non-deterministic — the LLM may complete the step without tool calls.
      # We verify the output dir exists (infra) and check file as best-effort.
      OUTPUT_DIR_EXISTS=$(kubectl exec "$MCP_HOST_POD" -n "$SANDBOX_NS" -- ls -d /output 2>/dev/null && echo "yes" || echo "no")
      [ "$OUTPUT_DIR_EXISTS" = "yes" ] && ok "Output dir /output exists" || warn "Output dir not found (non-root FS?)"

      ARTIFACTS=$(kubectl exec "$MCP_HOST_POD" -n "$SANDBOX_NS" -- ls /output/ 2>/dev/null || echo "")
      if echo "$ARTIFACTS" | grep -q "e2e-result.md"; then
        ok "Output file e2e-result.md exists"
        CONTENT=$(kubectl exec "$MCP_HOST_POD" -n "$SANDBOX_NS" -- cat /output/e2e-result.md 2>/dev/null || echo "")
        echo "$CONTENT" | grep -qi "test passed\|passed\|result" && ok "Output content validated" || warn "Expected content not found (LLM non-determinism)"
      else
        warn "Output file e2e-result.md not found (LLM may not have called clerum__generate_markdown)"
        log "Files in output dir: ${ARTIFACTS:-<empty>}"
      fi
    else
      fail "mcp-host pod not found in $SANDBOX_NS (label: clerum.io/recipe=${RECIPE_NAME})"
      # Diagnostic: show what pods exist
      log "Pods in $SANDBOX_NS:"
      kubectl get pods -n "$SANDBOX_NS" --show-labels 2>/dev/null || true
      log "Pods matching recipe name in all namespaces:"
      kubectl get pods -A -l "clerum.io/recipe=${RECIPE_NAME}" 2>/dev/null || true
    fi
  else
    fail "Workflow did not complete within ${WORKFLOW_TIMEOUT}s"
    # Print phase info for debugging
    kubectl get workflowrecipe "$RECIPE_NAME" -n "$SANDBOX_NS" -o jsonpath='{.status.workflowExecution}' 2>/dev/null | jq . || true
  fi
else
  fail "Recipe install failed: $(echo "$RINSTALL" | jq -r '.error // .')"
fi

# ═══ Group 5: MCP Server Upgrade ═════════════════════════════════════
header "Group 5: MCP Server Upgrade"

# Push v2
PUSH_V2=$(reg POST "/api/v1/entries" "{
  \"name\": \"e2e-mock-mcp\",
  \"version\": \"${E2E_VER2}\",
  \"entryType\": \"mcp-server\",
  \"description\": \"E2E mock MCP v2\",
  \"author\": \"e2e-test\",
  \"origin\": \"human-authored\",
  \"category\": \"testing\",
  \"tags\": [\"e2e\"],
  \"contentCreatorTag\": \"1st-party\",
  \"configCreatorTag\": \"1st-party\",
  \"mcpServer\": {
    \"serverMode\": \"local\",
    \"transport\": \"streamableHttp\",
    \"port\": 3000,
    \"imageRef\": \"clerum/mock-mcp-server:test\"
  },
  \"bundle\": \"${VALID_BUNDLE}\"
}")
echo "$PUSH_V2" | jq -e '.id' >/dev/null 2>&1 && ok "Pushed v${E2E_VER2} of e2e-mock-mcp" || fail "Failed to push v2"

if [ -n "$LOCAL_SERVER_NAME" ]; then
  UPGRADE=$(api POST "/api/v1/admin/registry/upgrade" "{
    \"serverName\": \"${LOCAL_SERVER_NAME}\",
    \"registryEntryName\": \"e2e-mock-mcp\",
    \"registryEntryVersion\": \"2.0.0\"
  }")
  UPGRADED=$(echo "$UPGRADE" | jq -r '.upgraded // false')
  [ "$UPGRADED" = "true" ] && ok "MCP server upgraded to v${E2E_VER2}" || fail "Upgrade failed: $(echo "$UPGRADE" | jq -r '.error // .')"
else
  warn "Skipping upgrade (no local server installed)"
fi

# ═══ Group 6: Recipe Upgrade ══════════════════════════════════════════
header "Group 6: Recipe Upgrade"

RECIPE_V2_YAML=$(cat <<'YAMLEOF'
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
  name: e2e-workflow-recipe
spec:
  description: E2E v2
  agent:
    providerHint: zai
    modelHint: glm-4.7
  steps:
    - id: greet
      instruction: "Say hello world"
YAMLEOF
)
PUSH_RV2=$(reg POST "/api/v1/entries" "$(jq -n \
  --arg ver "$E2E_VER2" \
  --arg recipe "$RECIPE_V2_YAML" \
  '{name: "e2e-workflow-recipe", version: $ver, entryType: "recipe",
    description: "E2E lifecycle recipe v2", author: "e2e-test",
    origin: "human-authored", category: "workflow", tags: ["e2e"],
    contentCreatorTag: "1st-party", configCreatorTag: "1st-party",
    recipe: $recipe}')")
echo "$PUSH_RV2" | jq -e '.id' >/dev/null 2>&1 && ok "Pushed recipe v${E2E_VER2}" || fail "Failed to push recipe v2"

if [ -n "$RECIPE_NAME" ]; then
  # Uninstall v1
  api DELETE "/api/v1/admin/registry/uninstall/${RECIPE_NAME}?type=recipe" >/dev/null 2>&1
  sleep 3
  kubectl get workflowrecipe "$RECIPE_NAME" -n "$SANDBOX_NS" >/dev/null 2>&1 && fail "v1 recipe still exists in ${SANDBOX_NS}" || ok "v1 recipe uninstalled"

  # Install v2
  RV2=$(api POST "/api/v1/admin/registry/install-recipe" "{
    \"registryEntryName\": \"e2e-workflow-recipe\",
    \"registryEntryVersion\": \"2.0.0\"
  }")
  RECIPE_V2_NAME=$(echo "$RV2" | jq -r '.recipeName // empty')
  [ -n "$RECIPE_V2_NAME" ] && ok "Recipe v2 installed: $RECIPE_V2_NAME" || fail "Recipe v2 install failed"
  RECIPE_NAME="$RECIPE_V2_NAME" # Track for cleanup
else
  warn "Skipping recipe upgrade (no v1 installed)"
fi

# ═══ Group 7: Uninstall Local MCP ════════════════════════════════════
header "Group 7: Uninstall Local MCP Server"

if [ -n "$LOCAL_SERVER_NAME" ]; then
  UNINST=$(api DELETE "/api/v1/admin/registry/uninstall/${LOCAL_SERVER_NAME}")
  echo "$UNINST" | jq -r '.deleted[]' 2>/dev/null | grep -q "McpServer" && ok "McpServer deleted" || warn "McpServer delete not confirmed"

  sleep 3
  kubectl get mcpserver "$LOCAL_SERVER_NAME" -n "$MCP_NS" >/dev/null 2>&1 && fail "McpServer still exists" || ok "McpServer gone from cluster"

  # Context cleaned
  CTX_AFTER=$(kubectl get contexts "$CONTEXT_NAME" -n "$MCP_NS" -o jsonpath='{.spec.mcpServers}' 2>/dev/null || echo "[]")
  echo "$CTX_AFTER" | grep -q "$LOCAL_SERVER_NAME" && fail "Still in context" || ok "Removed from context allowlist"
else
  warn "No local server to uninstall"
fi

# ═══ Group 8: Uninstall Remote MCP ═══════════════════════════════════
header "Group 8: Uninstall Remote MCP Server"

if [ -n "$REMOTE_SERVER_NAME" ]; then
  api DELETE "/api/v1/admin/registry/uninstall/${REMOTE_SERVER_NAME}" >/dev/null 2>&1
  sleep 3
  kubectl get mcpserver "$REMOTE_SERVER_NAME" -n "$MCP_NS" >/dev/null 2>&1 && fail "Remote McpServer still exists" || ok "Remote McpServer gone"
  kubectl get secret "${REMOTE_SERVER_NAME}-credentials" -n "$MCP_NS" >/dev/null 2>&1 && fail "Remote Secret still exists" || ok "Remote Secret gone"
else
  warn "No remote server to uninstall"
fi

# ═══ Group 9: Uninstall Recipe ════════════════════════════════════════
header "Group 9: Uninstall Recipe"

if [ -n "$RECIPE_NAME" ]; then
  api DELETE "/api/v1/admin/registry/uninstall/${RECIPE_NAME}?type=recipe" >/dev/null 2>&1
  sleep 5
  kubectl get workflowrecipe "$RECIPE_NAME" -n "$SANDBOX_NS" >/dev/null 2>&1 && fail "Recipe still exists in ${SANDBOX_NS}" || ok "Recipe gone from cluster"
else
  warn "No recipe to uninstall"
fi

# ═══ Group 10: Clean State Verification ═══════════════════════════════
header "Group 10: Clean State"

E2E_MCP=$(kubectl get mcpserver -n "$MCP_NS" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep "^e2e-\|^mcp-e2e-" | wc -l | tr -d ' ')
[ "$E2E_MCP" -eq 0 ] && ok "No e2e McpServers remain" || fail "$E2E_MCP e2e McpServers remain"

E2E_WR=$(kubectl get workflowrecipe -n "$SANDBOX_NS" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep "^e2e-\|^recipe-e2e-" | wc -l | tr -d ' ')
[ "$E2E_WR" -eq 0 ] && ok "No e2e WorkflowRecipes remain" || fail "$E2E_WR e2e WorkflowRecipes remain"

E2E_SEC=$(kubectl get secret -n "$MCP_NS" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep "^e2e-" | wc -l | tr -d ' ')
[ "$E2E_SEC" -eq 0 ] && ok "No e2e Secrets remain" || fail "$E2E_SEC e2e Secrets remain"

# ═══════════════════════════════════════════════════════════════════════
header "Summary"
echo -e "  ${GREEN}PASS${NC}: ${e2e_pass}  ${RED}FAIL${NC}: ${e2e_fail}  Total: ${e2e_total}"
[ "$e2e_fail" -gt 0 ] && exit 1 || exit 0