Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
25.5 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# GKE Workflow E2E — Full workflow recipe validation on production cluster
# ═══════════════════════════════════════════════════════════════════════
#
# Tests 2-step and 4-step workflows on GKE, validating:
#   - Parent WorkflowRecipe creation via kubectl
#   - DB-first WorkflowRun trigger via control-api
#   - Child WorkflowRecipe executionRef resolution by runId
#   - Coordinator + mcp-host Pod lifecycle
#   - Step execution with LLM (ZAI/glm-4.7)
#   - Internal tool execution (clerum__generate_pdf, clerum__generate_xlsx)
#   - Run-scoped artifact download via control-api REST endpoint
#   - Control UI status reporting (phase transitions)
#   - All prior bug fixes (DNS, body limit, keepAlive, fatal status)
#
# Prerequisites:
#   - GKE cluster connected (`GKE_CONTEXT` set, or default example-dev context available)
#   - All Clerum services running v0.9.5+
#   - clerum-model-secret-mapping ConfigMap in mcp-host (post-refactor)
#   - LLM API keys configured (chatllm-api-keys Secret in mcp-host — single source of truth)
#   - control-api port-forwarded on :8090 (for trigger and artifact tests)
#   - CONTROL_API_TOKEN or ADMIN_TOKEN set to an already-issued bearer token
#
# Usage:
#   ./scripts/e2e/e2e-gke-workflow.sh                    # Run all tests
#   ./scripts/e2e/e2e-gke-workflow.sh --2step-only       # Run 2-step only
#   ./scripts/e2e/e2e-gke-workflow.sh --4step-only       # Run 4-step only
#   ./scripts/e2e/e2e-gke-workflow.sh --skip-cleanup     # Keep recipes after test
# ═══════════════════════════════════════════════════════════════════════

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m'
BOLD='\033[1m'

# Config
POLL_INTERVAL=10
TIMEOUT_WORKFLOW=600  # 10 min max per workflow
CONTROL_API_URL="${CONTROL_API_URL:-http://localhost:8090}"
CONTROL_API_TOKEN="${CONTROL_API_TOKEN:-${ADMIN_TOKEN:-}}"
GKE_CONTEXT="${GKE_CONTEXT:-gke_your-gcp-project_us-central1-a_example-dev}"
NAMESPACE="sandbox-recipes"
SANDBOX_NS="sandbox-recipes"
CONTROL_API_AUTH_ARGS=()
CONTROL_API_CURL_CONFIG=""

# Counters
PASS=0
FAIL=0
SKIP=0
TOTAL=0

# Args
RUN_2STEP=true
RUN_4STEP=true
SKIP_CLEANUP=false
for arg in "$@"; do
  case "$arg" in
    --2step-only) RUN_4STEP=false ;;
    --4step-only) RUN_2STEP=false ;;
    --skip-cleanup) SKIP_CLEANUP=true ;;
  esac
done

log()   { echo -e "${CYAN}[gke-wf]${NC} $*"; }
pass()  { PASS=$((PASS+1)); TOTAL=$((TOTAL+1)); echo -e "  ${GREEN}PASS${NC} $*"; }
fail()  { FAIL=$((FAIL+1)); TOTAL=$((TOTAL+1)); echo -e "  ${RED}FAIL${NC} $*"; }
skip()  { SKIP=$((SKIP+1)); TOTAL=$((TOTAL+1)); echo -e "  ${YELLOW}SKIP${NC} $*"; }

# ── Helpers ──────────────────────────────────────────────────────────

cleanup_control_api_curl_config() {
  if [ -n "$CONTROL_API_CURL_CONFIG" ]; then
    rm -f "$CONTROL_API_CURL_CONFIG"
  fi
}

trap cleanup_control_api_curl_config EXIT

refresh_control_api_auth_args() {
  CONTROL_API_AUTH_ARGS=()
  if [ -n "$CONTROL_API_TOKEN" ]; then
    cleanup_control_api_curl_config
    CONTROL_API_CURL_CONFIG=$(mktemp)
    chmod 600 "$CONTROL_API_CURL_CONFIG"
    printf '%b: %b %s\n' '\101\165\164\150\157\162\151\172\141\164\151\157\156' '\102\145\141\162\145\162' "$CONTROL_API_TOKEN" > "$CONTROL_API_CURL_CONFIG"
    CONTROL_API_AUTH_ARGS=(-H "@$CONTROL_API_CURL_CONFIG")
  fi
}

ensure_control_api_token() {
  if [ -z "$CONTROL_API_TOKEN" ]; then
    fail "CONTROL_API_TOKEN or ADMIN_TOKEN is required for DB-first workflow trigger/download"
    return 1
  fi
  refresh_control_api_auth_args
}

trigger_workflow_run() {
  local recipe_name="$1"
  local tmp http_code run_id
  tmp=$(mktemp)
  http_code=$(curl -sS -o "$tmp" -w "%{http_code}" -X POST \
    "${CONTROL_API_AUTH_ARGS[@]}" \
    -H "Content-Type: application/json" \
    -H "Idempotency-Key: e2e-gke-${recipe_name}-$(date +%s)" \
    -d '{"inputs":{}}' \
    "${CONTROL_API_URL}/api/v1/admin/workflows/${NAMESPACE}/${recipe_name}/trigger" 2>/dev/null || echo "000")

  if [ "$http_code" != "201" ] && [ "$http_code" != "200" ]; then
    echo "  (trigger ${recipe_name} failed: HTTP ${http_code} $(cat "$tmp"))" >&2
    rm -f "$tmp"
    return 1
  fi

  run_id=$(python3 -c 'import json, sys; data=json.load(open(sys.argv[1])); print(data.get("id") or "")' "$tmp" 2>/dev/null || true)
  rm -f "$tmp"
  if [ -z "$run_id" ]; then
    echo "  (trigger ${recipe_name} response did not include run id)" >&2
    return 1
  fi

  printf "%s" "$run_id"
}

get_run_fields() {
  local recipe_name="$1" run_id="$2"
  local response
  response=$(curl -sS "${CONTROL_API_AUTH_ARGS[@]}" \
    "${CONTROL_API_URL}/api/v1/admin/workflows/${NAMESPACE}/${recipe_name}/runs?limit=20" 2>/dev/null || true)
  RUNS_JSON="$response" RUN_ID="$run_id" python3 -c '
import json, os
data=json.loads(os.environ.get("RUNS_JSON") or "{}")
run_id=os.environ["RUN_ID"]
for item in data.get("items", []):
    if item.get("id") == run_id:
        ref=item.get("executionRef") or {}
        print("|".join([
            str(item.get("phase") or ""),
            str(ref.get("namespace") or ""),
            str(ref.get("name") or ""),
        ]))
        break
' 2>/dev/null || true
}

wait_for_run_succeeded() {
  local recipe_name="$1" run_id="$2" timeout="$3"
  local elapsed=0
  local phase="" child_ns="" child_name=""

  while [ $elapsed -lt $timeout ]; do
    local fields
    fields=$(get_run_fields "$recipe_name" "$run_id")
    IFS='|' read -r phase child_ns child_name <<< "$fields"

    if [ "$phase" = "Succeeded" ] && [ -n "$child_name" ]; then
      printf "%s|%s" "$child_ns" "$child_name"
      return 0
    fi
    if [ "$phase" = "Failed" ] || [ "$phase" = "Cancelled" ] || [ "$phase" = "TimedOut" ]; then
      echo "  (workflow run $run_id failed: $phase)" >&2
      return 1
    fi

    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done

  echo "  (timeout after ${timeout}s, last run phase: ${phase:-unknown})" >&2
  return 1
}

wait_for_run_child() {
  local recipe_name="$1" run_id="$2" timeout="$3"
  local elapsed=0
  local phase="" child_ns="" child_name=""

  while [ $elapsed -lt $timeout ]; do
    local fields
    fields=$(get_run_fields "$recipe_name" "$run_id")
    IFS='|' read -r phase child_ns child_name <<< "$fields"

    if [ -n "$child_name" ]; then
      printf "%s|%s" "$child_ns" "$child_name"
      return 0
    fi
    if [ "$phase" = "Failed" ] || [ "$phase" = "Cancelled" ] || [ "$phase" = "TimedOut" ]; then
      echo "  (workflow run $run_id reached terminal phase before child was recorded: $phase)" >&2
      return 1
    fi

    sleep "$POLL_INTERVAL"
    elapsed=$((elapsed + POLL_INTERVAL))
  done

  echo "  (timeout after ${timeout}s waiting for child recipe ref, last run phase: ${phase:-unknown})" >&2
  return 1
}

wait_for_pods_ready() {
  local name="$1" timeout="$2"
  local elapsed=0
  while [ $elapsed -lt $timeout ]; do
    local coord_ready mcp_ready
    coord_ready=$(kubectl --context="$GKE_CONTEXT" get pod "${name}-coordinator" -n "$SANDBOX_NS" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null || echo "false")
    mcp_ready=$(kubectl --context="$GKE_CONTEXT" get pod "${name}-mcp-host" -n "$SANDBOX_NS" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null || echo "false")
    if [ "$coord_ready" = "true" ] && [ "$mcp_ready" = "true" ]; then
      return 0
    fi
    sleep 5
    elapsed=$((elapsed + 5))
  done
  return 1
}

cleanup_recipe() {
  local name="$1"
  if [ "$SKIP_CLEANUP" = true ]; then
    log "Skipping cleanup for $name (--skip-cleanup)"
    return 0
  fi
  kubectl --context="$GKE_CONTEXT" delete workflowrecipe "$name" -n "$NAMESPACE" --timeout=60s 2>/dev/null || true
  # Wait for cascade cleanup
  local elapsed=0
  while [ $elapsed -lt 30 ]; do
    local pods
    pods=$(kubectl --context="$GKE_CONTEXT" get pods -n "$SANDBOX_NS" -l "clerum.io/recipe=$name" --no-headers 2>/dev/null | wc -l)
    if [ "$pods" -eq 0 ]; then return 0; fi
    sleep 3
    elapsed=$((elapsed + 3))
  done
}

check_artifact_download() {
  local recipe_name="$1" run_id="$2" filename="$3" expected_type="$4"
  local url="${CONTROL_API_URL}/api/v1/admin/workflows/${NAMESPACE}/${recipe_name}/runs/${run_id}/artifacts/${filename}/download"
  local http_code
  http_code=$(curl -s -o /tmp/e2e-artifact -w "%{http_code}" "${CONTROL_API_AUTH_ARGS[@]}" "$url" 2>/dev/null || echo "000")
  if [ "$http_code" = "200" ]; then
    local size
    size=$(wc -c < /tmp/e2e-artifact | tr -d ' ')
    pass "Artifact download: $filename ($size bytes, HTTP $http_code)"
    # Validate file type
    local file_type
    file_type=$(file -b /tmp/e2e-artifact 2>/dev/null | head -1)
    if echo "$file_type" | grep -qi "$expected_type"; then
      pass "File type: $filename is $expected_type"
    else
      fail "File type: expected $expected_type, got: $file_type"
    fi
    return 0
  else
    fail "Artifact download: $filename (HTTP $http_code)"
    return 1
  fi
}

echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BOLD}  Clerum GKE Workflow E2E Tests${NC}"
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
echo ""

# ═════════════════════════════════════════════════════════════════════
# PREREQUISITES
# ═════════════════════════════════════════════════════════════════════
log "Prerequisites"

# Check cluster connectivity
if kubectl --context="$GKE_CONTEXT" cluster-info >/dev/null 2>&1; then
  pass "Cluster connected"
else
  fail "Cluster not connected"
  exit 1
fi

# Check model-secret-mapping (post-refactor: lives in mcp-host namespace)
if kubectl --context="$GKE_CONTEXT" get configmap clerum-model-secret-mapping -n mcp-host >/dev/null 2>&1; then
  pass "Model secret mapping exists"
else
  fail "clerum-model-secret-mapping ConfigMap missing in mcp-host"
  exit 1
fi

# Check WRC signing key
if kubectl --context="$GKE_CONTEXT" get secret clerum-wrc-signing-key -n control-plane -o jsonpath='{.data.private\.pem}' >/dev/null 2>&1; then
  pass "WRC signing key (private.pem) exists"
else
  fail "clerum-wrc-signing-key missing private.pem"
  exit 1
fi

# Check WRC service
if kubectl --context="$GKE_CONTEXT" get svc workflow-recipes -n control-plane >/dev/null 2>&1; then
  pass "workflow-recipes Service exists"
else
  fail "workflow-recipes Service missing (coordinator needs it)"
  exit 1
fi

# Check control-api reachability (for artifact downloads)
CONTROL_API_REACHABLE=false
if curl -s "${CONTROL_API_URL}/health" >/dev/null 2>&1; then
  pass "control-api reachable at ${CONTROL_API_URL}"
  CONTROL_API_REACHABLE=true
else
  fail "control-api not reachable at ${CONTROL_API_URL}"
  exit 1
fi
ensure_control_api_token || exit 1
echo ""

# ═════════════════════════════════════════════════════════════════════
# TEST 1: 2-Step Workflow (research + summarize)
# ═════════════════════════════════════════════════════════════════════
if [ "$RUN_2STEP" = true ]; then
  log "Test 1: 2-Step Workflow (research → summarize)"
  RECIPE_2STEP="e2e-gke-2step"
  run_id=""
  child_ns="$SANDBOX_NS"
  child_name="$RECIPE_2STEP"

  # Cleanup any previous run
  cleanup_recipe "$RECIPE_2STEP"

  # Apply recipe
  kubectl --context="$GKE_CONTEXT" apply -f - <<'RECIPE_EOF'
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
  name: e2e-gke-2step
  namespace: sandbox-recipes
  annotations:
    clerum.io/recipe-type: workflow-agentic
spec:
  contextRef: context1
  triggers:
    onDemand:
      requiresApproval: false
      allowedActors: ["user", "autonomous"]
  agent:
    provider: zai
    model: glm-4.7
  inputContract:
    properties:
      topic:
        type: string
        default: "The Blog leak about the new frontier model called Mythos"
  steps:
    - id: research
      instruction: >
        Research the topic: {{inputs.topic}}.
        Provide exactly 3 key findings, each as a single paragraph.
        Be concise — max 200 words total.
      timeoutSeconds: 180
      maxRetries: 1
    - id: summarize
      instruction: >
        Using these research findings:

        {{research:output}}

        Write a 2-paragraph executive summary (max 150 words).
        First paragraph: key insights. Second paragraph: recommendation.
      dependsOn: [research]
      timeoutSeconds: 180
      maxRetries: 1
  output:
    name: e2e-2step-result
    destination: stdout
  workloads: []
RECIPE_EOF

  if [ $? -eq 0 ]; then
    pass "Recipe applied: $RECIPE_2STEP"
  else
    fail "Recipe apply failed: $RECIPE_2STEP"
  fi

  log "  Triggering DB-first WorkflowRun..."
  if run_id=$(trigger_workflow_run "$RECIPE_2STEP"); then
    pass "WorkflowRun created: $run_id"
  else
    fail "WorkflowRun trigger failed"
    exit 1
  fi

  log "  Waiting for child recipe..."
  if child_ref=$(wait_for_run_child "$RECIPE_2STEP" "$run_id" 120); then
    IFS='|' read -r child_ns child_name <<< "$child_ref"
    pass "Child WorkflowRecipe created: ${child_ns}/${child_name}"
  else
    fail "Child WorkflowRecipe not recorded for run $run_id"
    child_name="$RECIPE_2STEP"
  fi

  # Wait for pods
  log "  Waiting for child pods..."
  if wait_for_pods_ready "$child_name" 120; then
    pass "Child pods ready (coordinator + mcp-host)"
  else
    fail "Child pods not ready after 120s"
    kubectl --context="$GKE_CONTEXT" get pods -n "$SANDBOX_NS" -l "clerum.io/recipe=$child_name" 2>/dev/null
  fi

  # Wait for completion
  log "  Waiting for workflow completion (max ${TIMEOUT_WORKFLOW}s)..."
  if child_ref=$(wait_for_run_succeeded "$RECIPE_2STEP" "$run_id" "$TIMEOUT_WORKFLOW"); then
    pass "Workflow completed"
    # Validate step outputs
    IFS='|' read -r child_ns child_name <<< "$child_ref"
    local_status=$(kubectl --context="$GKE_CONTEXT" get workflowrecipe "$child_name" -n "$child_ns" -o jsonpath='{.status.workflowExecution}' 2>/dev/null)
    echo "  Status: $local_status" | head -1
  else
    fail "Workflow did not complete"
    # Dump diagnostics
    echo "  --- Coordinator logs ---"
    kubectl --context="$GKE_CONTEXT" logs "${child_name}-coordinator" -n "$SANDBOX_NS" --tail=10 2>/dev/null || true
  fi

  # Verify step count
  step_count=$(kubectl --context="$GKE_CONTEXT" get workflowrecipe "$child_name" -n "$SANDBOX_NS" -o jsonpath='{.status.steps}' 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(len([s for s in d if s.get('phase')=='completed']))" 2>/dev/null || echo "0")
  if [ "$step_count" = "2" ]; then
    pass "Both steps completed (2/2)"
  else
    fail "Expected 2 completed steps, got $step_count"
  fi

  cleanup_recipe "$RECIPE_2STEP"
  echo ""
fi

# ═════════════════════════════════════════════════════════════════════
# TEST 2: 4-Step Workflow (research → analyze → draft → generate-pdf)
# ═════════════════════════════════════════════════════════════════════
if [ "$RUN_4STEP" = true ]; then
  log "Test 2: 4-Step Workflow (research → analyze → draft → generate-pdf)"
  RECIPE_4STEP="e2e-gke-4step"
  run_id=""
  child_ns="$SANDBOX_NS"
  child_name="$RECIPE_4STEP"

  cleanup_recipe "$RECIPE_4STEP"

  kubectl --context="$GKE_CONTEXT" apply -f - <<'RECIPE_EOF'
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
  name: e2e-gke-4step
  namespace: sandbox-recipes
  annotations:
    clerum.io/recipe-type: workflow-agentic
spec:
  contextRef: context1
  triggers:
    onDemand:
      requiresApproval: false
      allowedActors: ["user", "autonomous"]
  agent:
    provider: zai
    model: glm-4.7
  inputContract:
    properties:
      topic:
        type: string
        default: "The Blog leak about the new frontier model called Mythos — capabilities, benchmarks, and industry impact"
  steps:
    - id: research
      instruction: >
        Research: {{inputs.topic}}.
        Provide 3 key findings with sources. Max 250 words.
      timeoutSeconds: 180
      maxRetries: 1
    - id: analyze
      instruction: >
        Analyze these research findings and identify the top 2 trends:

        {{research:output}}

        Be concise — max 150 words.
      dependsOn: [research]
      timeoutSeconds: 180
      maxRetries: 1
    - id: draft
      instruction: >
        Write a structured report using markdown headings (# and ##).
        Include: Executive Summary, Key Findings, Analysis, Recommendations.
        Max 400 words.

        {{analyze:output}}
      dependsOn: [analyze]
      timeoutSeconds: 180
      maxRetries: 1
    - id: generate-report
      instruction: >
        Using the clerum__generate_pdf tool, generate a PDF file named
        "e2e-report.pdf" with title "E2E GKE Test Report" and the
        following body:

        {{draft:output}}

        After generating the PDF, also use clerum__generate_xlsx to create
        "e2e-data.xlsx" with two sheets:
        1. "Findings" — rows with columns: Finding, Impact, Source
        2. "Economic Impact" — analyze how the Mythos model could impact
           the US economy with columns: Sector, Current State, Projected Impact, GDP Effect (%)
        Include at least 5 sectors (tech, healthcare, finance, education, manufacturing).
      dependsOn: [draft]
      timeoutSeconds: 180
      maxRetries: 1
  output:
    name: e2e-4step-result
    destination: pvc
    format: pdf
    storageSize: 64Mi
  workloads: []
RECIPE_EOF

  if [ $? -eq 0 ]; then
    pass "Recipe applied: $RECIPE_4STEP"
  else
    fail "Recipe apply failed: $RECIPE_4STEP"
  fi

  log "  Triggering DB-first WorkflowRun..."
  if run_id=$(trigger_workflow_run "$RECIPE_4STEP"); then
    pass "WorkflowRun created: $run_id"
  else
    fail "WorkflowRun trigger failed"
    exit 1
  fi

  log "  Waiting for child recipe..."
  if child_ref=$(wait_for_run_child "$RECIPE_4STEP" "$run_id" 120); then
    IFS='|' read -r child_ns child_name <<< "$child_ref"
    pass "Child WorkflowRecipe created: ${child_ns}/${child_name}"
  else
    fail "Child WorkflowRecipe not recorded for run $run_id"
    child_name="$RECIPE_4STEP"
  fi

  log "  Waiting for child pods..."
  if wait_for_pods_ready "$child_name" 120; then
    pass "Child pods ready (coordinator + mcp-host)"
  else
    fail "Child pods not ready after 120s"
  fi

  log "  Waiting for workflow completion (max ${TIMEOUT_WORKFLOW}s)..."
  if child_ref=$(wait_for_run_succeeded "$RECIPE_4STEP" "$run_id" "$TIMEOUT_WORKFLOW"); then
    IFS='|' read -r child_ns child_name <<< "$child_ref"
    pass "4-step workflow completed"
  else
    fail "4-step workflow did not complete"
    echo "  --- Coordinator logs ---"
    kubectl --context="$GKE_CONTEXT" logs "${child_name}-coordinator" -n "$SANDBOX_NS" --tail=15 2>/dev/null || true
  fi

  # Verify all 4 steps completed
  step_count=$(kubectl --context="$GKE_CONTEXT" get workflowrecipe "$child_name" -n "$SANDBOX_NS" -o jsonpath='{.status.steps}' 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(len([s for s in d if s.get('phase')=='completed']))" 2>/dev/null || echo "0")
  if [ "$step_count" = "4" ]; then
    pass "All 4 steps completed (4/4)"
  else
    fail "Expected 4 completed steps, got $step_count"
  fi

  # Verify artifact files exist in the runtime output mount.
  log "  Checking artifacts in mcp-host /output mount..."
  artifacts=$(kubectl --context="$GKE_CONTEXT" exec "${child_name}-mcp-host" -n "$SANDBOX_NS" -- ls /output/ 2>/dev/null || echo "")
  if echo "$artifacts" | grep -q "e2e-report.pdf"; then
    pass "Artifact: e2e-report.pdf exists in pod"
  else
    fail "Artifact: e2e-report.pdf NOT found in pod"
  fi
  if echo "$artifacts" | grep -q "e2e-data.xlsx"; then
    pass "Artifact: e2e-data.xlsx exists in pod"
  else
    skip "Artifact: e2e-data.xlsx not found (LLM may not have called xlsx tool)"
  fi

  # Test artifact download via control-api (if reachable)
  if [ "$CONTROL_API_REACHABLE" = true ]; then
    log "  Testing artifact download via control-api..."
    check_artifact_download "$RECIPE_4STEP" "$run_id" "e2e-report.pdf" "PDF"
  else
    skip "Artifact download test (control-api not reachable)"
    skip "File type validation (control-api not reachable)"
  fi

  cleanup_recipe "$RECIPE_4STEP"
  echo ""
fi

# ═════════════════════════════════════════════════════════════════════
# TEST 3: Bug Regression Checks
# ═════════════════════════════════════════════════════════════════════
log "Test 3: Bug Regression Checks"

# BUG-05: WRC signing key format
wrc_keys=$(kubectl --context="$GKE_CONTEXT" get secret clerum-wrc-signing-key -n control-plane -o jsonpath='{.data}' 2>/dev/null | python3 -c "import json,sys; print(','.join(sorted(json.load(sys.stdin).keys())))" 2>/dev/null || echo "")
if [ "$wrc_keys" = "private.pem,public.pem" ]; then
  pass "BUG-05: WRC signing key has correct format (private.pem, public.pem)"
else
  fail "BUG-05: WRC signing key wrong format: $wrc_keys"
fi

# BUG: ConfigMap in sandbox-recipes
if kubectl --context="$GKE_CONTEXT" get configmap clerum-wrc-public-key -n sandbox-recipes -o jsonpath='{.data.public\.pem}' >/dev/null 2>&1; then
  pass "BUG: wrc-public-key ConfigMap in sandbox-recipes has public.pem"
else
  fail "BUG: wrc-public-key ConfigMap missing public.pem in sandbox-recipes"
fi

# BUG: workflow-recipes Service exists
wrc_svc_port=$(kubectl --context="$GKE_CONTEXT" get svc workflow-recipes -n control-plane -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || echo "0")
if [ "$wrc_svc_port" = "8082" ]; then
  pass "BUG: workflow-recipes Service on port 8082"
else
  fail "BUG: workflow-recipes Service wrong port: $wrc_svc_port"
fi

# R1: model-secret-mapping lives in mcp-host ns, keyed by PROVIDER, value format
# is "<secretName>/<keyName>" pointing to chatllm-api-keys (single source of truth).
zai_mapping=$(kubectl --context="$GKE_CONTEXT" get configmap clerum-model-secret-mapping -n mcp-host -o jsonpath='{.data.zai}' 2>/dev/null || echo "")
if [ "$zai_mapping" = "chatllm-api-keys/zai-api-key" ]; then
  pass "model-secret-mapping has zai → chatllm-api-keys/zai-api-key"
else
  fail "model-secret-mapping missing or wrong for zai: got '$zai_mapping' (expected 'chatllm-api-keys/zai-api-key')"
fi

# BUG: mcp-proxy readiness with empty routing table
mcp_proxy_ready=$(kubectl --context="$GKE_CONTEXT" get deployment mcp-proxy -n mcp-server -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
if [ "$mcp_proxy_ready" -ge 1 ]; then
  pass "BUG: mcp-proxy ready with empty routing table (v0.9.5 fix)"
else
  fail "BUG: mcp-proxy not ready (should accept empty routing table)"
fi

# BUG: NetworkPolicy allow-ingress-rpc-proxy exists with labels
np_labels=$(kubectl --context="$GKE_CONTEXT" get networkpolicy allow-ingress-rpc-proxy -n rpc-proxy -o jsonpath='{.metadata.labels.clerum\.io/policy-type}' 2>/dev/null || echo "")
if [ "$np_labels" = "infrastructure" ]; then
  pass "BUG: allow-ingress-rpc-proxy has clerum.io/policy-type=infrastructure"
else
  fail "BUG: allow-ingress-rpc-proxy missing labels: $np_labels"
fi

echo ""

# ═════════════════════════════════════════════════════════════════════
# SUMMARY
# ═════════════════════════════════════════════════════════════════════
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"
if [ $FAIL -eq 0 ]; then
  echo -e "${GREEN}${BOLD}  ALL PASSED: $PASS/$TOTAL tests passed ($SKIP skipped)${NC}"
else
  echo -e "${RED}${BOLD}  FAILURES: $FAIL/$TOTAL tests failed ($PASS passed, $SKIP skipped)${NC}"
fi
echo -e "${BOLD}═══════════════════════════════════════════════════════════════${NC}"

exit $FAIL