Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
68.9 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E gate — Plugin Workload SDK (promptBridge + clientNotifications)
# ═══════════════════════════════════════════════════════════════════════
#
# Minikube-only (plan §8, §9). Proves that a recipe declaring
# spec.pluginWorkloadSdk:
#   - has its capability validated and projected to status.pluginWorkloadSdk,
#   - gets the SDK NetworkPolicies + token Secret + workload env injected,
#   - lets an allowed plugin workload call promptBridge + clientNotifications
#     through the recipe mcp-host's SDK server, authorized by control-api,
#   - records both calls in the invocation audit trail.
#
# Token-dependent steps (grant creation, SDK call assertions, audit query)
# use E2E_ADMIN_TOKEN when supplied, otherwise the gate acquires an ephemeral
# session from the seeded admin on an explicitly selected local minikube profile.
#
# Usage:
#   E2E_PLUGIN_SDK_WRITE_CONFIRM=1 KUBECONTEXT=clerum-test \
#     bash scripts/e2e/e2e-plugin-workload-sdk.sh
#   E2E_PLUGIN_SDK_WRITE_CONFIRM=1 bash scripts/e2e/e2e-plugin-workload-sdk.sh --cleanup-only

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

# Resolve the canonical repository environment before sourcing e2e-lib. A
# secondary worktree normally has no .env of its own; the seeded credentials
# and provider keys live in the primary checkout. Explicit process values are
# loaded for general E2E configuration. Admin credentials use the stricter
# shared resolver below so the primary checkout remains authoritative:
#
#   canonical root .env > process environment > safe local test fallback
#
# shellcheck source=scripts/e2e/load-dotenv.sh
source "${SCRIPT_DIR}/load-dotenv.sh"
dotenv_load_canonical_root "${REPO_ROOT}"
# shellcheck source=scripts/e2e/admin-credentials.sh
source "${SCRIPT_DIR}/admin-credentials.sh"
export RECIPE_NS="${RECIPE_NS:-sandbox-recipes}"
# shellcheck source=e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"
require_safe_kube_context

die() {
  printf '%s\n' "Plugin Workload SDK E2E error: $*" >&2
  exit 1
}

is_local_default_operator_context() {
  local context=${1:-}
  if [[ -z "$context" ]]; then
    context="$(current_e2e_context || true)"
  fi
  case "$context" in
    clerum-test) return 0 ;;
    clerum-codex-* | clerum-cursor-* | clerum-detached-*)
      is_branch_scoped_e2e_context "$context"
      ;;
    *) return 1 ;;
  esac
}

RECIPE_NAME="e2e-plugin-workload-sdk"
WORKLOAD_ID="sdk-caller"
EVENT_TYPE="e2e.test.notification"
E2E_DESKTOP_USER_EMAIL="${E2E_DESKTOP_USER_EMAIL:-${E2E_DEV_LOGIN_EMAIL:-test@clerum.io}}"

# The host-wide CLERUM_MODEL_PROVIDER may legitimately remain Z.AI for other
# local workloads, but this gate must not silently redirect a paid call. An
# explicit E2E provider/model is required when the root environment selects an
# unavailable provider; only an entirely absent provider uses the documented
# local OpenAI default.
E2E_WORKFLOW_MODEL_PROVIDER="${E2E_WORKFLOW_MODEL_PROVIDER:-}"
E2E_WORKFLOW_MODEL_NAME="${E2E_WORKFLOW_MODEL_NAME:-}"
if [ -z "$E2E_WORKFLOW_MODEL_PROVIDER" ]; then
  case "${CLERUM_MODEL_PROVIDER:-}" in
    openai|claude)
      E2E_WORKFLOW_MODEL_PROVIDER="$CLERUM_MODEL_PROVIDER"
      if [ -z "$E2E_WORKFLOW_MODEL_NAME" ] && [ -n "${CLERUM_MODEL_NAME:-}" ]; then
        E2E_WORKFLOW_MODEL_NAME="$CLERUM_MODEL_NAME"
      fi
      ;;
    '')
      E2E_WORKFLOW_MODEL_PROVIDER="openai"
      ;;
    *)
      die "CLERUM_MODEL_PROVIDER=${CLERUM_MODEL_PROVIDER} is not permitted for this gate while that provider is unavailable; set E2E_WORKFLOW_MODEL_PROVIDER and E2E_WORKFLOW_MODEL_NAME explicitly to OpenAI or Claude"
      ;;
  esac
fi
if [ -z "$E2E_WORKFLOW_MODEL_NAME" ]; then
  case "$E2E_WORKFLOW_MODEL_PROVIDER" in
    openai) E2E_WORKFLOW_MODEL_NAME="gpt-5.4-mini" ;;
    claude) E2E_WORKFLOW_MODEL_NAME="claude-sonnet-4-6" ;;
    *) E2E_WORKFLOW_MODEL_NAME="gpt-5.4-mini" ;;
  esac
fi
# promptBridge grants are an ordered, credential-bound policy. The target
# reference is not a credential value; operators can override both values for
# a non-default, provider-owned Secret data key without exposing its contents.
# The gate deliberately carries two distinct providers so selector authorization
# and ordered fallback are exercised independently of whether either paid
# provider is available in the local cluster.
E2E_PROMPT_TARGET_REF="${E2E_PROMPT_TARGET_REF:-primary}"
E2E_PROMPT_CREDENTIAL_SLOT="${E2E_PROMPT_CREDENTIAL_SLOT:-${E2E_WORKFLOW_CREDENTIAL_SLOT:-}}"
E2E_PROMPT_FALLBACK_PROVIDER="${E2E_PROMPT_FALLBACK_PROVIDER:-}"
E2E_PROMPT_FALLBACK_MODEL="${E2E_PROMPT_FALLBACK_MODEL:-}"
if [ -z "$E2E_PROMPT_FALLBACK_PROVIDER" ]; then
  if [ "$E2E_WORKFLOW_MODEL_PROVIDER" = "claude" ]; then
    E2E_PROMPT_FALLBACK_PROVIDER="openai"
  else
    E2E_PROMPT_FALLBACK_PROVIDER="claude"
  fi
fi
if [ -z "$E2E_PROMPT_FALLBACK_MODEL" ]; then
  case "$E2E_PROMPT_FALLBACK_PROVIDER" in
    openai) E2E_PROMPT_FALLBACK_MODEL="gpt-5.4-mini" ;;
    claude) E2E_PROMPT_FALLBACK_MODEL="claude-sonnet-4-6" ;;
    *) E2E_PROMPT_FALLBACK_MODEL="" ;;
  esac
fi
E2E_PROMPT_FALLBACK_TARGET_REF="${E2E_PROMPT_FALLBACK_TARGET_REF:-fallback-${E2E_PROMPT_FALLBACK_PROVIDER}}"
E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT="${E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT:-}"
E2E_ADMIN_TOKEN="${E2E_ADMIN_TOKEN:-}"

load_branch_profile_ports() {
  local ctx="${KUBECONTEXT:-${E2E_K8S_CONTEXT:-}}"
  local ports_env="${CLERUM_PROFILE_PORTS_ENV:-${E2E_PROFILE_PORTS_ENV:-}}"
  if [[ "$ctx" =~ ^clerum-(codex|detached)- ]] || [[ "$ctx" =~ ^clerum-.+-[0-9a-f]{7,8}$ ]]; then
    if [ -z "$ports_env" ]; then
      die "Branch-owned context ${ctx} requires ports.env generated by scripts/minikube-profiles/branch.mk; set CLERUM_PROFILE_PORTS_ENV or E2E_PROFILE_PORTS_ENV before running this gate"
    fi
  elif [ -z "$ports_env" ] && [ -n "$ctx" ]; then
    ports_env="${HOME}/.cache/clerum/minikube-profiles/${ctx}/ports.env"
  fi
  if [[ -n "$ports_env" ]]; then
    [[ -f "$ports_env" ]] || die "profile ports file not found: ${ports_env}"
    dotenv_load_file "$ports_env"
  fi
}
load_branch_profile_ports
# The recipe and Desktop fixture users are seeded from this same resolved
# credential. The known fallback is local-only and is selected only when none
# of the supported admin variables exists in the canonical .env or process.
LOCAL_ADMIN_PASSWORD_FALLBACK=""
if is_local_default_operator_context; then
  LOCAL_ADMIN_PASSWORD_FALLBACK="$(printf '%s%s' 'changeme123' '!')"
fi
RESOLVED_ADMIN_PASSWORD="$(e2e_resolve_admin_password "${REPO_ROOT}" "${LOCAL_ADMIN_PASSWORD_FALLBACK}" || true)"
E2E_DESKTOP_PASSWORD="${E2E_DESKTOP_PASSWORD:-${RESOLVED_ADMIN_PASSWORD}}"
E2E_CONTROL_API_URL="${E2E_CONTROL_API_URL:-${CONTROL_API_URL:-${CONTROL_API_BASE_URL:-http://127.0.0.1:8090}}}"
E2E_EXTERNAL_REST_API_URL="${E2E_EXTERNAL_REST_API_URL:-${EXTERNAL_REST_API_URL:-${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}}}"
E2E_RPC_PROXY_URL="${E2E_RPC_PROXY_URL:-${RPC_PROXY_URL:-${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}}}"
MCP_HOST_BASE_URL="${MCP_HOST_BASE_URL:-${MCP_HOST_RUNTIME_BASE_URL:-${MCP_HOST_RUNTIME_URL:-http://127.0.0.1:8080}}}"
export E2E_CONTROL_API_URL E2E_EXTERNAL_REST_API_URL E2E_RPC_PROXY_URL MCP_HOST_BASE_URL
FIXTURE="${SCRIPT_DIR}/../../tests/e2e/fixtures/plugin-workload-sdk-recipe.yaml"
E2E_CREATED_RECIPE=0
ADMIN_CURL_HTTP_STATUS=""
ADMIN_CURL_BODY=""
NOTIFICATION_PREF_SNAPSHOT_READY=0
NOTIFICATION_PREF_SNAPSHOT_EXISTS=0
NOTIFICATION_PREF_SNAPSHOT_MEDIUM=""
NOTIFICATION_PREF_SNAPSHOT_FALLBACK=""
NOTIFICATION_PREF_SNAPSHOT_ACCOUNT=""
# Hard ceiling for the entire gate (default 10m). Override with
# E2E_GATE_MAX_SECONDS only to a value no greater than the fail-closed ceiling.
E2E_GATE_MAX_SECONDS="${E2E_GATE_MAX_SECONDS:-600}"
E2E_GATE_STARTED_AT=$SECONDS
# Per-phase wait ceilings (seconds).
E2E_WAIT_STATUS_VALIDATED="${E2E_WAIT_STATUS_VALIDATED:-120}"
E2E_WAIT_CALLER_MARKER="${E2E_WAIT_CALLER_MARKER:-180}"
E2E_WAIT_CALLER_DONE="${E2E_WAIT_CALLER_DONE:-180}"
E2E_WAIT_NOTIFICATION_ROW="${E2E_WAIT_NOTIFICATION_ROW:-60}"
# issue #375 (P3): event-driven grant→WRC nudge target latency. A grant change
# must re-reconcile the recipe via Postgres LISTEN/NOTIFY in ≤10s WITHOUT a WRC
# restart (the ≤30s watchdog is the backstop, not the mechanism under test).
E2E_WAIT_NUDGE_VALIDATED="${E2E_WAIT_NUDGE_VALIDATED:-10}"

# Fail closed before creating any fixture if a caller or inherited shell
# environment asks this gate to wait for an unbounded or human-scale interval.
# These ceilings keep a broken reconcile/provider path actionable instead of
# leaving a T2 run apparently idle for tens of minutes.
E2E_HARD_MAX_GATE_SECONDS=600
E2E_HARD_MAX_PHASE_WAIT_SECONDS=180
E2E_HARD_MAX_POLL_INTERVAL_SECONDS=5

validate_bounded_seconds() {
  local name="$1" value="$2" ceiling="$3"
  if [[ ! "$value" =~ ^[0-9]+$ ]] || [ "$value" -le 0 ] || [ "$value" -gt "$ceiling" ]; then
    die "${name}=${value} exceeds the fail-closed ceiling of ${ceiling}s"
  fi
}

validate_bounded_seconds E2E_GATE_MAX_SECONDS "$E2E_GATE_MAX_SECONDS" "$E2E_HARD_MAX_GATE_SECONDS"
validate_bounded_seconds TIMEOUT_POD "$TIMEOUT_POD" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds TIMEOUT_DELETE "$TIMEOUT_DELETE" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds TIMEOUT_DISCOVERY "$TIMEOUT_DISCOVERY" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_STATUS_VALIDATED "$E2E_WAIT_STATUS_VALIDATED" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_CALLER_MARKER "$E2E_WAIT_CALLER_MARKER" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_CALLER_DONE "$E2E_WAIT_CALLER_DONE" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_NOTIFICATION_ROW "$E2E_WAIT_NOTIFICATION_ROW" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds E2E_WAIT_NUDGE_VALIDATED "$E2E_WAIT_NUDGE_VALIDATED" "$E2E_HARD_MAX_PHASE_WAIT_SECONDS"
validate_bounded_seconds POLL_INTERVAL "$POLL_INTERVAL" "$E2E_HARD_MAX_POLL_INTERVAL_SECONDS"

require_write_confirmation() {
  if [ "${E2E_PLUGIN_SDK_WRITE_CONFIRM:-}" != "1" ]; then
    printf '%s\n' \
      'Refusing Plugin Workload SDK E2E: this gate creates grants, calls providers, sends a notification, and changes test-user preferences.' \
      'Set E2E_PLUGIN_SDK_WRITE_CONFIRM=1 explicitly for the branch-owned local Minikube profile.' >&2
    return 1
  fi
}

for provider in "$E2E_WORKFLOW_MODEL_PROVIDER" "$E2E_PROMPT_FALLBACK_PROVIDER"; do
  case "$provider" in
    openai|claude) ;;
    *)
      printf 'Plugin Workload SDK E2E only permits OpenAI or Claude targets; got %q\n' "$provider" >&2
      exit 2
      ;;
  esac
done

default_prompt_credential_slot() {
  case "$1" in
    openai) printf '%s\n' 'openai-api-key' ;;
    claude) printf '%s\n' 'claude-api-key' ;;
    zai) printf '%s\n' 'zai-api-key' ;;
    bailian) printf '%s\n' 'bailian-api-key' ;;
    vertex) printf '%s\n' 'vertex-service-account-json' ;;
    bedrock) printf '%s\n' 'aws-access-key-id' ;;
    openrouter) printf '%s\n' 'openrouter-api-key' ;;
    gemini) printf '%s\n' 'gemini-api-key' ;;
    deepseek) printf '%s\n' 'deepseek-api-key' ;;
    groq) printf '%s\n' 'groq-api-key' ;;
    together) printf '%s\n' 'together-api-key' ;;
    fireworks) printf '%s\n' 'fireworks-api-key' ;;
    mistral) printf '%s\n' 'mistral-api-key' ;;
    xai) printf '%s\n' 'xai-api-key' ;;
    cerebras) printf '%s\n' 'cerebras-api-key' ;;
    deepinfra) printf '%s\n' 'deepinfra-api-key' ;;
    perplexity) printf '%s\n' 'perplexity-api-key' ;;
    moonshot) printf '%s\n' 'moonshot-api-key' ;;
    nebius) printf '%s\n' 'nebius-api-key' ;;
    novita) printf '%s\n' 'novita-api-key' ;;
    minimax) printf '%s\n' 'minimax-api-key' ;;
    azure) printf '%s\n' 'azure-openai-api-key' ;;
    *) return 1 ;;
  esac
}

if [ -z "$E2E_PROMPT_CREDENTIAL_SLOT" ]; then
  E2E_PROMPT_CREDENTIAL_SLOT="$(default_prompt_credential_slot "$E2E_WORKFLOW_MODEL_PROVIDER")" \
    || { printf 'Unsupported promptBridge provider %q; set E2E_PROMPT_CREDENTIAL_SLOT explicitly\n' "$E2E_WORKFLOW_MODEL_PROVIDER" >&2; exit 2; }
fi
if [ -z "$E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT" ]; then
  E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT="$(default_prompt_credential_slot "$E2E_PROMPT_FALLBACK_PROVIDER")" \
    || { printf 'Unsupported promptBridge fallback provider %q; set E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT explicitly\n' "$E2E_PROMPT_FALLBACK_PROVIDER" >&2; exit 2; }
fi
if [ "$E2E_WORKFLOW_MODEL_PROVIDER" = "$E2E_PROMPT_FALLBACK_PROVIDER" ]; then
  if [ "${E2E_PROMPT_ALLOW_SAME_PROVIDER:-0}" = "1" ]; then
    printf 'promptBridge E2E: primary and fallback share provider %q; proceeding because E2E_PROMPT_ALLOW_SAME_PROVIDER=1 (quota-limited local override — the ordered-fallback policy is still two ordered targets)\n' \
      "$E2E_WORKFLOW_MODEL_PROVIDER" >&2
  else
    printf 'promptBridge E2E requires two distinct providers; got %q for both primary and fallback\n' \
      "$E2E_WORKFLOW_MODEL_PROVIDER" >&2
    exit 2
  fi
fi

gate_assert_deadline() {
  local phase=${1:-gate}
  if [ $((SECONDS - E2E_GATE_STARTED_AT)) -ge "$E2E_GATE_MAX_SECONDS" ]; then
    fail "E2E gate exceeded ${E2E_GATE_MAX_SECONDS}s deadline during: ${phase}"
    print_results
    exit 1
  fi
}

stop_sdk_caller_fixture() {
  if ! kctl delete pod "$caller_pod" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1; then
    warn "could not delete sdk-caller pod before recycling it"
    return 1
  fi
  if wait_for_pod "$SANDBOX_NS" "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${WORKLOAD_ID}" "$TIMEOUT_POD"; then
    caller_pod="$(ready_pod_name "$SANDBOX_NS" "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${WORKLOAD_ID}")"
    return 0
  fi
  return 1
}

reset_sdk_runtime_state_for_happy_path() {
  local cleanup_status=0 count table query
  local -a delete_queries=(
    "DELETE FROM plugin_workload_sdk_quota_counters WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_provider_attempts WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_credential_ticket_jtis WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_invocation_attempts WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_invocations WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM notification_deliveries WHERE payload->>'recipeName'='${RECIPE_NAME}' AND payload->>'origin'='plugin_workload_sdk';"
  )
  for query in "${delete_queries[@]}"; do
    if ! psql_query "$query" >/dev/null; then
      cleanup_status=1
      warn "could not reset SDK runtime state before happy path"
    fi
  done
  local -a count_tables=(
    plugin_workload_sdk_quota_counters
    plugin_workload_sdk_provider_attempts
    plugin_workload_sdk_credential_ticket_jtis
    plugin_workload_sdk_invocation_attempts
    plugin_workload_sdk_invocations
  )
  for table in "${count_tables[@]}"; do
    query="SELECT count(*) FROM ${table} WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    if ! count="$(psql_query "$query" | tr -d '[:space:]')" || [ "$count" != "0" ]; then
      cleanup_status=1
      warn "SDK runtime reset postcondition failed for ${table}"
    fi
  done
  query="SELECT count(*) FROM notification_deliveries WHERE payload->>'recipeName'='${RECIPE_NAME}' AND payload->>'origin'='plugin_workload_sdk';"
  if ! count="$(psql_query "$query" | tr -d '[:space:]')" || [ "$count" != "0" ]; then
    cleanup_status=1
    warn "SDK notification delivery reset postcondition failed"
  fi
  return "$cleanup_status"
}

wait_for_caller_logs_matching() {
  local pattern=$1
  local timeout_sec=${2:-$E2E_WAIT_CALLER_MARKER}
  local deadline=$((SECONDS + timeout_sec))
  local logs=""
  while [ "$SECONDS" -lt "$deadline" ]; do
    gate_assert_deadline "waiting for sdk-caller log pattern: ${pattern}"
    logs="$(kctl logs "$caller_pod" -n "$SANDBOX_NS" --tail=120 2>/dev/null || true)"
    if printf "%s" "$logs" | grep -q "$pattern"; then
      printf '%s' "$logs"
      return 0
    fi
    sleep "$POLL_INTERVAL"
  done
  printf '%s' "$logs"
  return 1
}

wait_for_notification_delivery_row() {
  local notification_id=$1
  local timeout_sec=${2:-$E2E_WAIT_NOTIFICATION_ROW}
  local deadline=$((SECONDS + timeout_sec))
  local dedupe_key="${notification_id}:plugin_workload_sdk.notification"
  local row=""
  while [ "$SECONDS" -lt "$deadline" ]; do
    gate_assert_deadline "waiting for notification_deliveries row"
    row="$(psql_query "SELECT event_type || '|' || status || '|' || COALESCE(audience->>'userId','') FROM notification_deliveries WHERE dedupe_key='${dedupe_key}' OR payload->>'notificationId'='${notification_id}' LIMIT 1;" || true)"
    row="$(printf '%s' "$row" | tr -d '[:space:]')"
    if [ -n "$row" ] && printf '%s' "$row" | grep -q 'plugin_workload_sdk.notification|'; then
      printf '%s' "$row"
      return 0
    fi
    sleep "$POLL_INTERVAL"
  done
  printf '%s' "$row"
  return 1
}

notifications_desktop_first_enabled() {
  local value
  value="$(kctl get deploy control-api -n "$CONTROL_NS" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="NOTIFICATIONS_DESKTOP_FIRST_ENABLED")].value}' 2>/dev/null || true)"
  if [ -z "$value" ]; then
    printf 'true\n'
    return 0
  fi
  if [ "$value" = "false" ]; then
    printf 'false\n'
    return 1
  fi
  printf 'true\n'
}

# ─── cleanup ─────────────────────────────────────────────────────────────

cleanup_plugin_workload_sdk_db() {
  local cleanup_status=0 count query label
  local -a delete_queries=(
    "DELETE FROM plugin_workload_sdk_quota_counters WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_provider_attempts WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_credential_ticket_jtis WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_invocation_attempts WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_invocations WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM plugin_workload_sdk_grants WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    "DELETE FROM notification_deliveries WHERE payload->>'recipeName'='${RECIPE_NAME}' AND payload->>'origin'='plugin_workload_sdk';"
  )
  for query in "${delete_queries[@]}"; do
    if ! psql_query "$query" >/dev/null; then
      cleanup_status=1
      warn "SDK DB cleanup query failed (recipe=${RECIPE_NS}/${RECIPE_NAME}); refusing to claim a clean fixture"
    fi
  done

  local -a count_queries=(
    "plugin_workload_sdk_quota_counters"
    "plugin_workload_sdk_invocations"
    "plugin_workload_sdk_credential_ticket_jtis"
    "plugin_workload_sdk_invocation_attempts"
    "plugin_workload_sdk_provider_attempts"
    "plugin_workload_sdk_grants"
  )
  for label in "${count_queries[@]}"; do
    query="SELECT count(*) FROM ${label} WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}';"
    if ! count="$(psql_query "$query" | tr -d '[:space:]')"; then
      cleanup_status=1
      warn "SDK DB cleanup verification failed for ${label}; refusing to claim zero rows"
      continue
    fi
    if [ "$count" != "0" ]; then
      cleanup_status=1
      warn "SDK DB cleanup postcondition failed for ${label}: ${count} rows remain"
    fi
  done
  query="SELECT count(*) FROM notification_deliveries WHERE payload->>'recipeName'='${RECIPE_NAME}' AND payload->>'origin'='plugin_workload_sdk';"
  if ! count="$(psql_query "$query" | tr -d '[:space:]')"; then
    cleanup_status=1
    warn "notification delivery cleanup verification failed; refusing to claim zero rows"
  elif [ "$count" != "0" ]; then
    cleanup_status=1
    warn "notification delivery cleanup postcondition failed: ${count} rows remain"
  fi
  return "$cleanup_status"
}

prune_recipe_owned_resources() {
  local cleanup_status=0 ns kind
  # WRC owns the lifecycle sweep, but a failed/aborted reconcile can leave
  # recipe-labeled SDK NetworkPolicies behind after the WorkflowRecipe is
  # gone. The gate must not leak those resources into the next run, so prune
  # only the exact fixture label in the two namespaces it can touch.
  for ns in "$SANDBOX_NS" "$RECIPE_NS"; do
    for kind in deployment service secret configmap networkpolicy pod pvc; do
      kctl delete "$kind" -n "$ns" -l "clerum.io/recipe=${RECIPE_NAME}" \
        --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
    done
  done
  return "$cleanup_status"
}

cleanup_sdk_recipe() {
  local cleanup_status=0
  if ! cleanup_plugin_workload_sdk_db; then
    cleanup_status=1
  fi
  kctl delete workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
  wait_for_workflowrecipe_deleted "$RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" || cleanup_status=1
  if ! prune_recipe_owned_resources; then
    cleanup_status=1
  fi
  return "$cleanup_status"
}

# shellcheck disable=SC2329 # Registered below through the EXIT trap.
cleanup_on_exit() {
  local status=$?
  # A non-zero abort that no assertion recorded (a prerequisite helper that died
  # in a command-substitution subshell, a bare `exit` mid-phase, or a `set -e`
  # abort) would otherwise let print_results claim "All tests passed!" over a run
  # that skipped later phases. Record it as a real, countable failure first so
  # the summary — and the gate verdict — stay honest.
  if [ "$status" -ne 0 ] && [ "${e2e_fail:-0}" -eq 0 ]; then
    fail "run aborted before completion (exit ${status}) — later phases were skipped; see the log above for the failing step"
  fi
  if [ "${e2e_total:-0}" -gt 0 ] && [ "${E2E_SUPPRESS_RESULTS:-0}" != "1" ]; then
    print_results || true
  fi
  if ! restore_notification_preferences && [ "$status" -eq 0 ]; then
    fail "notification preference cleanup could not restore the seeded user state"
    status=1
  fi
  if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then exit "$status"; fi
  if [ "$E2E_CREATED_RECIPE" != "1" ]; then exit "$status"; fi
  if ! cleanup_sdk_recipe && [ "$status" -eq 0 ]; then
    fail "plugin workload SDK cleanup left E2E resources behind"
    exit 1
  fi
  exit "$status"
}

# ─── control-api admin helpers (exec curl inside the control-api pod) ─────

control_api_pod() {
  ready_pod_name "$CONTROL_NS" "app=control-api" 2>/dev/null \
    || ready_pod_name "$CONTROL_NS" "app.kubernetes.io/name=control-api" 2>/dev/null
}

operator_admin_password() {
  if [[ -n "${RESOLVED_ADMIN_PASSWORD:-}" ]]; then
    printf '%s' "${RESOLVED_ADMIN_PASSWORD}"
    return 0
  fi
  is_local_default_operator_context || return 1
  e2e_resolve_admin_password "${REPO_ROOT}" "$(printf '%s%s' 'changeme123' '!')"
}

ensure_operator_session() {
  if [[ -n "${E2E_ADMIN_TOKEN:-}" ]]; then
    return 0
  fi

  local admin_password admin_username pod response status session_cookie
  admin_password="$(operator_admin_password)" || return 1
  admin_username="${E2E_ADMIN_USERNAME:-admin}"
  pod="$(control_api_pod)" || return 1

  # NUL-delimited stdin keeps both credentials out of local and remote argv.
  response="$(
    e2e_write_nul_credentials "$admin_username" "$admin_password" \
      | kctl exec -i "$pod" -n "$CONTROL_NS" -- node -e '
    const chunks = []
    process.stdin.on("data", chunk => chunks.push(Buffer.from(chunk)))
    process.stdin.on("end", async () => {
      try {
        const input = Buffer.concat(chunks)
        const separator = input.indexOf(0)
        if (separator < 0) throw new Error("invalid login input")
        const username = input.subarray(0, separator).toString("utf8")
        const password = input.subarray(separator + 1).toString("utf8")
        const response = await fetch("http://localhost:8090/api/v1/admin/auth/login", {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ username, password }),
        })
        const cookies = typeof response.headers.getSetCookie === "function"
          ? response.headers.getSetCookie()
          : [response.headers.get("set-cookie")].filter(Boolean)
        const raw = cookies.map(String).find(cookie => cookie.startsWith("control_ui_admin_session=")) || ""
        const value = raw ? raw.split(";")[0].split("=").slice(1).join("=") : ""
        process.stdout.write(String(response.status) + "\t" + value)
      } catch (error) {
        process.stderr.write(error.message)
        process.exit(1)
      }
    })
  '
  )" || return 1
  status="$(printf '%s' "$response" | cut -f1)"
  session_cookie="$(printf '%s' "$response" | cut -f2)"
  [[ "$status" == "200" && -n "$session_cookie" ]] || return 1
  E2E_ADMIN_TOKEN="$session_cookie"
  return 0
}

operator_session_gate() {
  ensure_operator_session
}

# admin_curl METHOD PATH [JSON_BODY]
# Runs node fetch inside the control-api pod against localhost:8090 with the
# E2E_ADMIN_TOKEN control-ui session cookie. Sets ADMIN_CURL_HTTP_STATUS +
# ADMIN_CURL_BODY and echoes the body for callers that only need the payload.
admin_curl() {
  local method=$1 path=$2 body=${3:-} pod response
  pod="$(control_api_pod)" || return 1
  response="$(
    printf '%s\0%s\0%s\0%s' "$method" "$path" "$E2E_ADMIN_TOKEN" "$body" \
      | kctl exec -i "$pod" -n "$CONTROL_NS" -- node -e '
    const chunks = []
    process.stdin.on("data", chunk => chunks.push(Buffer.from(chunk)))
    process.stdin.on("end", async () => {
      try {
        const fields = Buffer.concat(chunks).toString("utf8").split("\0")
        if (fields.length < 4) throw new Error("invalid admin request input")
        const method = fields.shift()
        const path = fields.shift()
        const token = fields.shift()
        const body = fields.join("\0") || null
        const response = await fetch("http://localhost:8090" + path, {
          method,
          headers: {
            "Cookie": "control_ui_admin_session=" + token,
            "Content-Type": "application/json",
          },
          body,
        })
        const text = await response.text()
        process.stdout.write(String(response.status) + "\t" + text)
      } catch (error) {
        process.stderr.write(error.message)
        process.exit(1)
      }
    })
  '
  )"
  ADMIN_CURL_HTTP_STATUS="$(printf '%s' "$response" | cut -f1)"
  ADMIN_CURL_BODY="$(printf '%s' "$response" | cut -f2-)"
  printf '%s' "$ADMIN_CURL_BODY"
}

# session_curl METHOD PATH SESSION_TOKEN [JSON_BODY]
# User-session routes are exposed by external-rest-api (/api/v1/me/*) and forward
# to control-api with the service token. Match the desktop app transport via PF.
session_curl() {
  local method=$1 path=$2 session_token=$3 body=${4:-} api_path response curl_args
  if ! command -v curl >/dev/null 2>&1; then
    return 1
  fi
  api_path="${path#/external}"
  curl_args=(-sS -w '\n%{http_code}' -X "$method" \
    "${E2E_EXTERNAL_REST_API_URL}/api/v1${api_path}" \
    -H "Authorization: Bearer ${session_token}" \
    -H 'Content-Type: application/json')
  if [ -n "$body" ]; then
    curl_args+=(-d "$body")
  fi
  response="$(curl "${curl_args[@]}" 2>/dev/null || true)"
  ADMIN_CURL_HTTP_STATUS="$(printf '%s' "$response" | tail -n1)"
  ADMIN_CURL_BODY="$(printf '%s' "$response" | sed '$d')"
  printf '%s' "$ADMIN_CURL_BODY"
}

psql_query() {
  local sql=$1
  kctl exec deploy/control-postgres -n "$CONTROL_NS" -- \
    psql -v ON_ERROR_STOP=1 -U postgres -d profiles -tA \
    -c "$sql" 2>/dev/null
}

snapshot_notification_preferences() {
  local row
  if ! row="$(psql_query "SELECT CASE WHEN preferred_medium IS NULL THEN '<NULL>' ELSE preferred_medium END || '|' || channel_fallback_enabled::text || '|' || CASE WHEN preferred_account_id IS NULL THEN '<NULL>' ELSE preferred_account_id::text END FROM user_notification_preferences WHERE user_id='${USER_REF}'::uuid;")"; then
    return 1
  fi
  if [ -z "$row" ]; then
    NOTIFICATION_PREF_SNAPSHOT_EXISTS=0
    NOTIFICATION_PREF_SNAPSHOT_READY=1
    return 0
  fi
  IFS='|' read -r NOTIFICATION_PREF_SNAPSHOT_MEDIUM \
    NOTIFICATION_PREF_SNAPSHOT_FALLBACK \
    NOTIFICATION_PREF_SNAPSHOT_ACCOUNT <<< "$row"
  NOTIFICATION_PREF_SNAPSHOT_EXISTS=1
  NOTIFICATION_PREF_SNAPSHOT_READY=1
}

restore_notification_preferences() {
  if [ "${NOTIFICATION_PREF_SNAPSHOT_READY:-0}" != "1" ]; then
    return 0
  fi
  if [ "${NOTIFICATION_PREF_SNAPSHOT_EXISTS:-0}" != "1" ]; then
    psql_query "DELETE FROM user_notification_preferences WHERE user_id='${USER_REF}'::uuid;" >/dev/null 2>&1
    return $?
  fi

  local medium_sql account_sql
  if [ "$NOTIFICATION_PREF_SNAPSHOT_MEDIUM" = '<NULL>' ]; then
    medium_sql='NULL'
  else
    medium_sql="'${NOTIFICATION_PREF_SNAPSHOT_MEDIUM}'"
  fi
  if [ "$NOTIFICATION_PREF_SNAPSHOT_ACCOUNT" = '<NULL>' ]; then
    account_sql='NULL'
  else
    account_sql="'${NOTIFICATION_PREF_SNAPSHOT_ACCOUNT}'::uuid"
  fi
  psql_query "INSERT INTO user_notification_preferences (user_id,preferred_medium,channel_fallback_enabled,preferred_account_id) VALUES ('${USER_REF}'::uuid,${medium_sql},${NOTIFICATION_PREF_SNAPSHOT_FALLBACK},${account_sql}) ON CONFLICT (user_id) DO UPDATE SET preferred_medium=EXCLUDED.preferred_medium,channel_fallback_enabled=EXCLUDED.channel_fallback_enabled,preferred_account_id=EXCLUDED.preferred_account_id,updated_at=NOW();" >/dev/null 2>&1
}

if [ "${1:-}" = "--cleanup-only" ]; then
  require_write_confirmation || exit 2
  cleanup_sdk_recipe
  exit $?
fi

require_write_confirmation || exit 2
trap cleanup_on_exit EXIT

is_psql_true() {
  case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in
    t | true) return 0 ;;
    *) return 1 ;;
  esac
}

is_psql_false() {
  case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in
    f | false) return 0 ;;
    *) return 1 ;;
  esac
}

resolve_e2e_desktop_user_id() {
  local email=$1
  psql_query "SELECT id::text FROM users WHERE lower(email)=lower('${email}') LIMIT 1;" \
    | tr -d '[:space:]'
}

wait_for_sdk_protocol_ready() {
  local deadline=$((SECONDS + E2E_WAIT_STATUS_VALIDATED))
  local state=""
  while [ "$SECONDS" -lt "$deadline" ]; do
    gate_assert_deadline "waiting for Plugin Workload SDK identity bootstrap"
    state=$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" -o jsonpath='{.status.pluginWorkloadSdk.state}' 2>/dev/null || true)
    # A missing/unreviewed operator grant is an expected control-plane state,
    # not a provider outage. The eager host identity is usable enough for the
    # operator to create the grant; only the later validated state authorizes
    # prompt traffic.
    if [ "$state" = "awaiting_policy" ] || [ "$state" = "validated" ]; then
      printf '%s\n' "$state"
      return 0
    fi
    sleep "$POLL_INTERVAL"
  done
  printf '%s\n' "$state"
  return 1
}

wait_for_sdk_policy_validated() {
  local deadline=$((SECONDS + E2E_WAIT_STATUS_VALIDATED))
  local state=""
  while [ "$SECONDS" -lt "$deadline" ]; do
    gate_assert_deadline "waiting for status.pluginWorkloadSdk.state=validated after operator grant"
    state=$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" -o jsonpath='{.status.pluginWorkloadSdk.state}' 2>/dev/null || true)
    if [ "$state" = "validated" ]; then
      return 0
    fi
    sleep "$POLL_INTERVAL"
  done
  printf '%s\n' "$state"
  return 1
}

ensure_external_rest_api_reachable() {
  local url="${E2E_EXTERNAL_REST_API_URL}/health"
  if curl -sf -m 5 "$url" >/dev/null 2>&1; then
    return 0
  fi
  load_branch_profile_ports
  url="${E2E_EXTERNAL_REST_API_URL}/health"
  if curl -sf -m 5 "$url" >/dev/null 2>&1; then
    return 0
  fi
  # This helper runs inside obtain_desktop_session_token's command substitution,
  # so `exit 1` would only terminate that subshell (aborting the parent via
  # `set -e` with NO recorded failure) and a `fail` here would bump a counter the
  # parent shell never sees. Propagate a non-zero RETURN instead and let the
  # parent-shell caller record the countable, visible failure.
  printf 'external-rest-api not reachable at %s (run branch-profile-pf for %s)\n' \
    "${E2E_EXTERNAL_REST_API_URL}" "${KUBECONTEXT:-<profile>}" >&2
  return 1
}

obtain_desktop_session_token() {
  local email=$1 password=$2 response body token
  if ! command -v curl >/dev/null 2>&1; then
    return 1
  fi
  ensure_external_rest_api_reachable || return 1
  response="$(
    printf '%s\0%s' "$email" "$password" \
      | jq -Rs 'split("\u0000") | {email: .[0], password: .[1]}' \
      | curl -sS -w '\n%{http_code}' -X POST \
        "${E2E_EXTERNAL_REST_API_URL}/api/v1/auth/password-login" \
        -H 'Content-Type: application/json' \
        --data-binary @- \
        2>/dev/null || true
  )"
  ADMIN_CURL_HTTP_STATUS="$(printf '%s' "$response" | tail -n1)"
  body="$(printf '%s' "$response" | sed '$d')"
  token="$(printf '%s' "$body" | jq -r '.token // empty' 2>/dev/null || true)"
  ADMIN_CURL_BODY="$body"
  if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ] || [ -z "$token" ]; then
    return 1
  fi
  printf '%s' "$token"
}

create_grant() {
  local family=$1 payload
  if [ "$family" = "promptBridge" ]; then
    payload="$(prompt_grant_payload "$WORKLOAD_ID")"
  else
    # issue #348: active per-minute override (maxNotificationsPerMinute) drives the
    # rate-limit gate; the deprecated maxNotificationsPerRun stays on the wire to
    # exercise the admin validate-but-strip path (it is accepted then stripped).
    payload='{"recipeNamespace":"sandbox-recipes","recipeName":"'"$RECIPE_NAME"'","capabilityFamily":"clientNotifications","allowedEventTypes":["'"$EVENT_TYPE"'"],"allowedUserRefs":["'"$USER_REF"'"],"allowedCallers":["'"$WORKLOAD_ID"'"],"quotaLimits":{"maxNotificationsPerMinute":10,"maxNotificationsPerRun":10}}'
  fi
  admin_curl POST "/api/v1/admin/plugin-workload-sdk/grants" "$payload"
  # admin_curl only reflects transport success; the admin API returns 200 on
  # accept and 4xx on a rejected policy (e.g. a duplicate provider+model target).
  # Without this check a rejected grant looks "created" and the real failure only
  # surfaces far downstream. Fail loud with a grant-specific, HTTP-bearing reason.
  case "${ADMIN_CURL_HTTP_STATUS:-}" in
    2[0-9][0-9]) return 0 ;;
    *)
      printf 'grant creation (%s) rejected by admin API: HTTP %s\n' \
        "$family" "${ADMIN_CURL_HTTP_STATUS:-unknown}" >&2
      return 1
      ;;
  esac
}

# The admin API validates the policy; jq is used here solely to serialize E2E
# configuration safely, never to carry credential values. `allowedModels` is
# retained for the compatibility/bootstrap contract, while `promptTargets` is
# the authorization policy and `defaultTargetRef` must select its first target.
prompt_grant_payload() {
  local caller=$1
  jq -cn \
    --arg namespace "$RECIPE_NS" \
    --arg recipe "$RECIPE_NAME" \
    --arg provider "$E2E_WORKFLOW_MODEL_PROVIDER" \
    --arg model "$E2E_WORKFLOW_MODEL_NAME" \
    --arg targetRef "$E2E_PROMPT_TARGET_REF" \
    --arg credentialSlot "$E2E_PROMPT_CREDENTIAL_SLOT" \
    --arg fallbackProvider "$E2E_PROMPT_FALLBACK_PROVIDER" \
    --arg fallbackModel "$E2E_PROMPT_FALLBACK_MODEL" \
    --arg fallbackTargetRef "$E2E_PROMPT_FALLBACK_TARGET_REF" \
    --arg fallbackCredentialSlot "$E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT" \
    --arg caller "$caller" \
    '{recipeNamespace:$namespace,recipeName:$recipe,capabilityFamily:"promptBridge",provider:$provider,allowedModels:[$model,$fallbackModel],promptTargets:[{targetRef:$targetRef,provider:$provider,model:$model,credentialSlot:$credentialSlot},{targetRef:$fallbackTargetRef,provider:$fallbackProvider,model:$fallbackModel,credentialSlot:$fallbackCredentialSlot}],defaultTargetRef:$targetRef,allowedCallers:[$caller],quotaLimits:{maxInvocationsPerMinute:4,maxRequestsPerRun:4}}'
}

# ─── prerequisites ───────────────────────────────────────────────────────

header "Plugin Workload SDK E2E — prerequisites"

kctl cluster-info >/dev/null
kctl get ns "$SANDBOX_NS" >/dev/null
kctl get ns "$MCP_HOST_NS" >/dev/null
kctl get crd workflowrecipes.clerum.io >/dev/null
kctl get deploy workflow-recipes -n "$CONTROL_NS" >/dev/null
ok "runtime prerequisites available"

# The triple activation gate requires the flag on both control-api and the
# recipe mcp-host. Surface a clear failure if the feature is off.
if kctl get deploy control-api -n "$CONTROL_NS" -o jsonpath='{.spec.template.spec.containers[*].env[?(@.name=="PLUGIN_WORKLOAD_SDK_ENABLED")].value}' 2>/dev/null | grep -q true; then
  ok "control-api has PLUGIN_WORKLOAD_SDK_ENABLED=true"
else
  fail "control-api PLUGIN_WORKLOAD_SDK_ENABLED is not true — enable the feature flag (plan §9.2)"
  exit 1
fi

if ! cleanup_sdk_recipe; then
  fail "plugin workload SDK pre-run cleanup left E2E resources behind"
  exit 1
fi

# ─── apply fixture ───────────────────────────────────────────────────────

header "Apply the SDK recipe fixture"

USER_REF="${E2E_SDK_USER_REF:-$(resolve_e2e_desktop_user_id "$E2E_DESKTOP_USER_EMAIL")}"
if [ -z "$USER_REF" ]; then
  fail "could not resolve desktop user id for ${E2E_DESKTOP_USER_EMAIL} — seed test data first"
  exit 1
fi
ok "SDK clientNotifications userRef targets desktop user ${USER_REF}"

sed -e "s/PLACEHOLDER_PROVIDER/${E2E_WORKFLOW_MODEL_PROVIDER}/" \
    -e "s/PLACEHOLDER_MODEL/${E2E_WORKFLOW_MODEL_NAME}/" \
    -e "s/PLACEHOLDER_FALLBACK_TARGET_REF/${E2E_PROMPT_FALLBACK_TARGET_REF}/" \
    -e "s/PLACEHOLDER_RECIPE_NAME/${RECIPE_NAME}/" \
    -e "s/PLACEHOLDER_USER_REF/${USER_REF}/" \
    "$FIXTURE" | kctl apply -f -
E2E_CREATED_RECIPE=1
ok "applied WorkflowRecipe ${RECIPE_NAME}"

# ─── infra invariants (no admin token needed) ────────────────────────────

header "Infrastructure invariants"

# 1. Identity bootstrap is ready. A new recipe normally reports awaiting_policy
# until the operator grant journey below completes; it must not be forced to
# manufacture a grant before the eager host can be observed.
if state="$(wait_for_sdk_protocol_ready)"; then
  ok "Plugin Workload SDK identity bootstrap ready (state=${state})"
else
  fail "Plugin Workload SDK identity bootstrap did not reach awaiting_policy/validated (got '${state:-<empty>}')"
  exit 1
fi

# 2. mcp-host ready (hosts the SDK server).
if wait_for_pod "$SANDBOX_NS" "clerum.io/recipe=${RECIPE_NAME},clerum.io/component=workflow-mcp-host" 180; then
  ok "recipe mcp-host pod is ready"
else
  fail "recipe mcp-host pod never became ready"
  exit 1
fi

# 3. SDK token Secret created.
if kctl get secret "wf-${RECIPE_NAME}-plugin-workload-sdk-token" -n "$SANDBOX_NS" >/dev/null 2>&1; then
  ok "SDK workload token Secret exists"
else
  fail "SDK workload token Secret missing"
  exit 1
fi

# 4. SDK NetworkPolicies created (ingress + egress).
np_ok=1
for np in "${RECIPE_NAME}-workload-to-mcp-host-sdk-ingress" "${RECIPE_NAME}-workload-to-mcp-host-sdk-egress"; do
  kctl get networkpolicy "$np" -n "$SANDBOX_NS" >/dev/null 2>&1 || np_ok=0
done
if [ "$np_ok" = "1" ]; then
  ok "SDK ingress + egress NetworkPolicies exist"
else
  fail "one or both SDK NetworkPolicies are missing"
  exit 1
fi

# 5. sdk-caller pod ready + env injected.
if wait_for_pod "$SANDBOX_NS" "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${WORKLOAD_ID}" 180; then
  ok "sdk-caller workload pod is ready"
else
  fail "sdk-caller workload pod never became ready"
  exit 1
fi
caller_pod="$(ready_pod_name "$SANDBOX_NS" "clerum.io/recipe=${RECIPE_NAME},clerum.io/workload=${WORKLOAD_ID}")"
caller_env="$(kctl get pod "$caller_pod" -n "$SANDBOX_NS" -o jsonpath='{.spec.containers[0].env[*].name}' 2>/dev/null || true)"
if printf "%s" "$caller_env" | grep -q PLUGIN_WORKLOAD_SDK_ENDPOINT &&
   printf "%s" "$caller_env" | grep -q PLUGIN_WORKLOAD_SDK_TOKEN; then
  ok "sdk-caller received PLUGIN_WORKLOAD_SDK_ENDPOINT + PLUGIN_WORKLOAD_SDK_TOKEN"
else
  fail "sdk-caller missing SDK endpoint/token env"
  exit 1
fi

# Security invariant: no provider key env leaked into the workload.
if printf "%s" "$caller_env" | grep -Eq 'OPENAI_API_KEY|CLAUDE_API_KEY|ZAI_API_KEY|BAILIAN_API_KEY'; then
  fail "sdk-caller pod has a provider API key env — must never be exposed to workloads"
  exit 1
else
  ok "sdk-caller pod has no provider API key env"
fi

# ─── route guards (token-independent) ───────────────────────────────────

header "Route guards"

# 401: a request without the workload token is rejected by the SDK server.
# The caller image is node:24.22.1-alpine (no curl); use the in-pod node + fetch.
unauth_status="$(kctl exec "$caller_pod" -n "$SANDBOX_NS" -- node -e '
fetch(process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT + "/v1/prompt-bridge", {
  method: "POST",
  headers: { "Content-Type": "application/json", "x-clerum-caller-ref": "sdk-caller" },
  body: "{}",
}).then(r => { console.log(r.status); process.exit(0) })
  .catch(() => { console.log("ERR"); process.exit(0) })
' 2>/dev/null || true)"
if [ "$unauth_status" = "401" ]; then
  ok "SDK server returns 401 for a request without the workload token"
else
  fail "SDK server did not return 401 without a token (got '${unauth_status}')"
  exit 1
fi

# NetworkPolicy: a pod in a different namespace cannot reach the SDK port.
mcp_svc="$(kctl get svc -n "$SANDBOX_NS" -l "clerum.io/recipe=${RECIPE_NAME},clerum.io/component=workflow-mcp-host" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [ -n "$mcp_svc" ]; then
  # Do not use `kubectl run --rm -i` here.  The attached stdin can remain open
  # after the probe pod exits (notably with a branch-owned Minikube context),
  # leaving the whole gate blocked in a command substitution.  Create the pod
  # detached, observe its terminal phase with a hard deadline, then collect
  # logs and delete it explicitly.  This keeps the gate fail-closed without
  # weakening the NetworkPolicy assertion.
  probe_name="sdk-np-probe-$$"
  probe_deadline=$((SECONDS + 45))
  probe_phase=""
  probe_out=""
  kctl delete pod "$probe_name" -n default --ignore-not-found --wait=false >/dev/null 2>&1 || true
  if kctl run "$probe_name" -n default --restart=Never --image=busybox:1.36 \
      --request-timeout=15s --command -- \
      sh -c "nc -z -w5 ${mcp_svc}.${SANDBOX_NS}.svc.cluster.local 8099 && echo OPEN || echo BLOCKED" \
      >/dev/null 2>&1; then
    while [ "$SECONDS" -lt "$probe_deadline" ]; do
      probe_phase="$(kctl get pod "$probe_name" -n default -o jsonpath='{.status.phase}' 2>/dev/null || true)"
      case "$probe_phase" in
        Succeeded|Failed) break ;;
      esac
      sleep 1
    done
    probe_out="$(kctl logs "$probe_name" -n default --tail=20 2>/dev/null || true)"
  else
    probe_phase="CreateFailed"
    probe_out=""
  fi
  kctl delete pod "$probe_name" -n default --ignore-not-found --wait=false >/dev/null 2>&1 || true
  if [ "$probe_phase" = "Succeeded" ] && printf "%s" "$probe_out" | grep -q BLOCKED; then
    ok "SDK port 8099 is blocked from the 'default' namespace (NetworkPolicy enforced)"
  else
    fail "SDK cross-namespace NetworkPolicy probe failed (phase=${probe_phase:-Timeout}; output redacted)"
    exit 1
  fi
else
  warn "could not resolve the recipe mcp-host Service name; skipping cross-namespace NP probe"
fi

# ─── authorized happy path (requires admin token) ────────────────────────

if operator_session_gate; then
  ok "operator session available for authenticated Plugin Workload SDK checks"
else
  fail "operator session acquisition failed for required authenticated Plugin Workload SDK checks"
  exit 1
fi

header "Admin grant guardrails"

wildcard_payload="$(jq -cn \
  --arg namespace "$RECIPE_NS" \
  --arg recipe "$RECIPE_NAME" \
  --arg provider "$E2E_WORKFLOW_MODEL_PROVIDER" \
  --arg model "$E2E_WORKFLOW_MODEL_NAME" \
  --arg targetRef "$E2E_PROMPT_TARGET_REF" \
  --arg credentialSlot "$E2E_PROMPT_CREDENTIAL_SLOT" \
  --arg caller "$WORKLOAD_ID" \
  '{recipeNamespace:$namespace,recipeName:$recipe,capabilityFamily:"promptBridge",provider:$provider,allowedModels:["*"],promptTargets:[{targetRef:$targetRef,provider:$provider,model:$model,credentialSlot:$credentialSlot}],defaultTargetRef:$targetRef,allowedCallers:[$caller]}')"
admin_curl POST "/api/v1/admin/plugin-workload-sdk/grants" "$wildcard_payload" >/dev/null || true
wildcard_status="$ADMIN_CURL_HTTP_STATUS"
wildcard_body="$ADMIN_CURL_BODY"
if [ "$wildcard_status" = "400" ] && printf '%s' "$wildcard_body" | grep -q 'wildcard_not_allowed'; then
  ok "admin API rejects wildcard allowlists with wildcard_not_allowed"
else
  fail "wildcard grant was not rejected as expected (HTTP ${wildcard_status:-unknown}; response body redacted)"
  exit 1
fi

missing_policy_payload="$(jq -cn \
  --arg namespace "$RECIPE_NS" \
  --arg recipe "$RECIPE_NAME" \
  --arg provider "$E2E_WORKFLOW_MODEL_PROVIDER" \
  --arg model "$E2E_WORKFLOW_MODEL_NAME" \
  --arg caller "$WORKLOAD_ID" \
  '{recipeNamespace:$namespace,recipeName:$recipe,capabilityFamily:"promptBridge",provider:$provider,allowedModels:[$model],allowedCallers:[$caller]}')"
admin_curl POST "/api/v1/admin/plugin-workload-sdk/grants" "$missing_policy_payload" >/dev/null || true
missing_policy_status="$ADMIN_CURL_HTTP_STATUS"
missing_policy_body="$ADMIN_CURL_BODY"
if [ "$missing_policy_status" = "400" ] && printf '%s' "$missing_policy_body" | grep -q 'promptTargets must be a non-empty array'; then
  ok "admin API rejects promptBridge grants without ordered promptTargets"
else
  fail "promptBridge grant without promptTargets was not rejected as expected (HTTP ${missing_policy_status:-unknown}; response body redacted)"
  exit 1
fi

missing_default_payload="$(jq -cn \
  --arg namespace "$RECIPE_NS" \
  --arg recipe "$RECIPE_NAME" \
  --arg provider "$E2E_WORKFLOW_MODEL_PROVIDER" \
  --arg model "$E2E_WORKFLOW_MODEL_NAME" \
  --arg targetRef "$E2E_PROMPT_TARGET_REF" \
  --arg credentialSlot "$E2E_PROMPT_CREDENTIAL_SLOT" \
  --arg caller "$WORKLOAD_ID" \
  '{recipeNamespace:$namespace,recipeName:$recipe,capabilityFamily:"promptBridge",provider:$provider,allowedModels:[$model],promptTargets:[{targetRef:$targetRef,provider:$provider,model:$model,credentialSlot:$credentialSlot}],allowedCallers:[$caller]}')"
admin_curl POST "/api/v1/admin/plugin-workload-sdk/grants" "$missing_default_payload" >/dev/null || true
missing_default_status="$ADMIN_CURL_HTTP_STATUS"
missing_default_body="$ADMIN_CURL_BODY"
if [ "$missing_default_status" = "400" ] && printf '%s' "$missing_default_body" | grep -q 'defaultTargetRef is required'; then
  ok "admin API rejects promptBridge grants without defaultTargetRef"
else
  fail "promptBridge grant without defaultTargetRef was not rejected as expected (HTTP ${missing_default_status:-unknown}; response body redacted)"
  exit 1
fi

header "Grants + negative authorization"

E2E_PROBE_RUN_ID="${E2E_PROBE_RUN_ID:-run-$$-$(date +%s)}"
# Idempotency keys allow only [a-zA-Z0-9_-]; sanitize hostname-derived noise.
E2E_PROBE_RUN_ID="$(printf '%s' "$E2E_PROBE_RUN_ID" | tr -c 'a-zA-Z0-9_-' '_')"
if ! stop_sdk_caller_fixture; then
  fail "could not recycle sdk-caller pod before authorization probes"
  exit 1
fi
ok "paused sdk-caller fixture before authorization probes (run=${E2E_PROBE_RUN_ID})"

if create_grant promptBridge >/dev/null; then
  ok "created promptBridge grant"
else
  fail "promptBridge grant creation failed"
  exit 1
fi
# issue #348: prove the admin API accepted the deprecated per-run key on the wire
# (grant creation returned 200 above) but STRIPPED it, persisting only the active
# per-minute override. This is the deprecate-and-ignore contract exercised end-to-end.
persisted_pb_quota="$(psql_query "SELECT quota_limits::text FROM plugin_workload_sdk_grants WHERE recipe_namespace='${RECIPE_NS}' AND recipe_name='${RECIPE_NAME}' AND capability_family='promptBridge';")"
if printf '%s' "$persisted_pb_quota" | jq -e '(.maxInvocationsPerMinute == 4) and (has("maxRequestsPerRun") | not)' >/dev/null 2>&1; then
  ok "deprecate-and-strip verified: maxInvocationsPerMinute=4 persisted, deprecated maxRequestsPerRun stripped"
else
  fail "persisted promptBridge quota_limits violate deprecate-and-strip (got: ${persisted_pb_quota})"
  exit 1
fi
if create_grant clientNotifications >/dev/null; then
  ok "created clientNotifications grant"
else
  fail "clientNotifications grant creation failed"
  exit 1
fi

# The grant is now active and the WRC's next live capability probe must move
# the recipe from awaiting_policy to validated. This is the point at which the
# data plane is authorized; protocol readiness alone never authorizes a call.
if wait_for_sdk_policy_validated; then
  ok "status.pluginWorkloadSdk.state == validated after operator grants"
else
  fail "status.pluginWorkloadSdk.state did not reach 'validated' after operator grants (got '${state:-<empty>}')"
  exit 1
fi

# Read back only the persisted policy metadata. This proves the operator's
# two-provider order/default contract without ever logging a Secret value,
# bearer token, or credential ticket.
prompt_grant_readback="$(admin_curl GET "/api/v1/admin/plugin-workload-sdk/grants?recipeNamespace=${RECIPE_NS}&recipeName=${RECIPE_NAME}" || true)"
if [ "$ADMIN_CURL_HTTP_STATUS" = "200" ] &&
  printf '%s' "$prompt_grant_readback" | jq -e \
    --arg provider "$E2E_WORKFLOW_MODEL_PROVIDER" \
    --arg model "$E2E_WORKFLOW_MODEL_NAME" \
    --arg targetRef "$E2E_PROMPT_TARGET_REF" \
    --arg credentialSlot "$E2E_PROMPT_CREDENTIAL_SLOT" \
    --arg fallbackProvider "$E2E_PROMPT_FALLBACK_PROVIDER" \
    --arg fallbackModel "$E2E_PROMPT_FALLBACK_MODEL" \
    --arg fallbackTargetRef "$E2E_PROMPT_FALLBACK_TARGET_REF" \
    --arg fallbackCredentialSlot "$E2E_PROMPT_FALLBACK_CREDENTIAL_SLOT" \
    '(.items // []) | map(select(.capabilityFamily == "promptBridge")) | .[0] as $grant |
      ($grant.provider == $provider) and
      ($grant.defaultTargetRef == $targetRef) and
      (($grant.promptTargets // []) | length == 2) and
      ($grant.promptTargets[0] == {targetRef:$targetRef,provider:$provider,model:$model,credentialSlot:$credentialSlot}) and
      ($grant.promptTargets[1] == {targetRef:$fallbackTargetRef,provider:$fallbackProvider,model:$fallbackModel,credentialSlot:$fallbackCredentialSlot}) and
      ((($grant | tojson) | test("secret|token"; "i")) | not)' >/dev/null; then
  ok "persisted promptBridge policy contains ordered primary + fallback providers without credential material"
else
  fail "persisted promptBridge policy did not match the two-provider order/default contract (HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}; response body redacted)"
  exit 1
fi

# Recipients read endpoint (grant-driven picker): the clientNotifications grant
# just created carries allowedUserRefs=[USER_REF]. GET /v1/client-notifications/
# recipients must surface that userRef (resolved to a display label) so a recipe
# sandbox UI can populate its notify dropdown from the authoritative grant rather
# than a recipe-baked RECIPIENTS env. Read-only: it must NOT consume the quota.
recipients_out="$(kctl exec "$caller_pod" -n "$SANDBOX_NS" -- env "E2E_USER_REF=${USER_REF}" node -e '
fetch(process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT + "/v1/client-notifications/recipients", {
  headers: { Authorization: "Bearer " + process.env.PLUGIN_WORKLOAD_SDK_TOKEN },
}).then(async r => {
  const b = await r.json().catch(() => ({}));
  const recips = Array.isArray(b.recipients) ? b.recipients : [];
  const hit = recips.some(x => x && x.userRef === process.env.E2E_USER_REF && typeof x.displayName === "string");
  console.log(r.status + ":" + (hit ? "HIT" : "MISS") + ":" + recips.length);
  process.exit(0);
}).catch(() => { console.log("ERR"); process.exit(0) })
' 2>/dev/null || true)"
if printf "%s" "$recipients_out" | grep -q '^200:HIT:'; then
  ok "recipients endpoint surfaces the grant's allowedUserRefs (grant-driven picker)"
else
  fail "recipients endpoint did not surface the granted userRef (got '${recipients_out}')"
  exit 1
fi

# Probe denials on the infra-phase pod before restarting it for the happy-path fixture run.
# callerRef is token-bound on the SDK server — never from x-clerum-caller-ref.

deny_event="$(kctl exec "$caller_pod" -n "$SANDBOX_NS" -- env "E2E_USER_REF=${USER_REF}" "E2E_PROBE_RUN_ID=${E2E_PROBE_RUN_ID}" node -e '
fetch(process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT + "/v1/client-notifications", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: "Bearer " + process.env.PLUGIN_WORKLOAD_SDK_TOKEN,
  },
  body: JSON.stringify({
    eventType: "unknown.event",
    userRef: process.env.E2E_USER_REF,
    idempotencyKey: "e2e-deny-event-" + process.env.E2E_PROBE_RUN_ID,
    notification: { title: "x", body: "y" },
  }),
}).then(async r => { const b = await r.json().catch(() => ({})); console.log(r.status + ":" + (b.error || "")); process.exit(0) })
  .catch(() => { console.log("ERR"); process.exit(0) })
' 2>/dev/null || true)"
if printf "%s" "$deny_event" | grep -q 'event_type_not_allowed'; then
  ok "undeclared event type is denied with event_type_not_allowed"
else
  fail "undeclared event type was not denied as expected (got '${deny_event}')"
  exit 1
fi

unauthorized_target="$(kctl exec "$caller_pod" -n "$SANDBOX_NS" -- env "E2E_PROBE_RUN_ID=${E2E_PROBE_RUN_ID}" node -e '
fetch(process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT + "/v1/prompt-bridge", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: "Bearer " + process.env.PLUGIN_WORKLOAD_SDK_TOKEN,
  },
  body: JSON.stringify({
    purpose: "summarization",
    targetRef: "unauthorized-third-provider",
    idempotencyKey: "e2e-deny-target-" + process.env.E2E_PROBE_RUN_ID,
    messages: [{ role: "user", content: "x" }],
  }),
}).then(async r => { const b = await r.json().catch(() => ({})); console.log(r.status + ":" + (b.error || "") + ":" + (b.message || "")); process.exit(0) })
  .catch(() => { console.log("ERR"); process.exit(0) })
' 2>/dev/null || true)"
if printf "%s" "$unauthorized_target" | grep -q 'target_not_allowed'; then
  ok "selector outside the two-provider policy is denied with target_not_allowed"
else
  fail "unauthorized provider target was not denied as expected (got '${unauthorized_target}')"
  exit 1
fi

narrow_payload="$(prompt_grant_payload 'e2e-unlisted-caller')"
admin_curl POST "/api/v1/admin/plugin-workload-sdk/grants" "$narrow_payload" >/dev/null || true
if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then
  fail "failed to narrow promptBridge grant for caller_not_allowed probe (HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}; response body redacted)"
  exit 1
fi

deny_caller="$(kctl exec "$caller_pod" -n "$SANDBOX_NS" -- env "E2E_PROBE_RUN_ID=${E2E_PROBE_RUN_ID}" node -e '
fetch(process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT + "/v1/prompt-bridge", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: "Bearer " + process.env.PLUGIN_WORKLOAD_SDK_TOKEN,
  },
  body: JSON.stringify({
    purpose: "summarization",
    idempotencyKey: "e2e-deny-caller-" + process.env.E2E_PROBE_RUN_ID,
    messages: [{ role: "user", content: "x" }],
  }),
}).then(async r => { const b = await r.json().catch(() => ({})); console.log(r.status + ":" + (b.error || "")); process.exit(0) })
  .catch(() => { console.log("ERR"); process.exit(0) })
' 2>/dev/null || true)"
if printf "%s" "$deny_caller" | grep -q 'caller_not_allowed'; then
  ok "token-bound caller excluded from grant is denied with caller_not_allowed"
else
  fail "caller_not_allowed probe failed (got '${deny_caller}')"
  exit 1
fi

create_grant promptBridge >/dev/null || true
if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then
  fail "failed to restore promptBridge grant after caller_not_allowed probe (status=${ADMIN_CURL_HTTP_STATUS})"
  exit 1
fi
ok "restored promptBridge grant after caller_not_allowed probe"

header "Authorized happy path"

if ! reset_sdk_runtime_state_for_happy_path; then
  fail "could not prove a clean SDK runtime state before the happy-path fixture"
  exit 1
fi
ok "reset SDK invocation/quota/delivery state before happy-path fixture"

# Restart the caller so the fixture re-runs against the restored grants.
if ! stop_sdk_caller_fixture; then
  fail "sdk-caller did not come back ready after grants"
  exit 1
fi
ok "sdk-caller restarted after grants for happy-path fixture"

# The caller prints structured markers; wait for happy-path before idempotency/quota finish.
logs="$(wait_for_caller_logs_matching E2E_SDK_CLIENT_NOTIFICATION_OK "$E2E_WAIT_CALLER_MARKER" || true)"

if printf "%s" "$logs" | grep -q E2E_SDK_PROMPT_BRIDGE_OK; then
  ok "sdk-caller promptBridge call was authorized (invocationId returned)"
else
  markers="$(printf '%s' "$logs" | grep -E '^E2E_SDK_(PROMPT_BRIDGE|EXPLICIT_TARGET|CLIENT_NOTIFICATION|IDEMPOTENCY|QUOTA|DONE)_' || true)"
  fail "sdk-caller promptBridge call did not succeed within ${E2E_WAIT_CALLER_MARKER}s (markers=${markers:-<none>})"
  exit 1
fi

if printf "%s" "$logs" | grep -q E2E_SDK_CLIENT_NOTIFICATION_OK; then
  ok "sdk-caller clientNotifications call was accepted (notificationId returned)"
else
  markers="$(printf '%s' "$logs" | grep -E '^E2E_SDK_(PROMPT_BRIDGE|EXPLICIT_TARGET|CLIENT_NOTIFICATION|IDEMPOTENCY|QUOTA|DONE)_' || true)"
  fail "sdk-caller clientNotifications call did not succeed within ${E2E_WAIT_CALLER_MARKER}s (markers=${markers:-<none>})"
  exit 1
fi

# The fixture intentionally exercises the approved selector after the first
# notification, then idempotency and quota. Waiting for its terminal marker
# gives the process time to emit those later markers instead of evaluating a
# partial log snapshot immediately after clientNotifications succeeds.
logs="$(wait_for_caller_logs_matching E2E_SDK_DONE "$E2E_WAIT_CALLER_DONE" || true)"
if ! printf "%s" "$logs" | grep -q E2E_SDK_DONE; then
  fail "sdk-caller fixture did not finish within ${E2E_WAIT_CALLER_DONE}s"
  exit 1
fi

if printf "%s" "$logs" | grep -q E2E_SDK_EXPLICIT_TARGET_OK; then
  ok "sdk-caller explicit approved targetRef was served and returned non-empty content"
else
  markers="$(printf '%s' "$logs" | grep -E '^E2E_SDK_(PROMPT_BRIDGE|EXPLICIT_TARGET|CLIENT_NOTIFICATION|IDEMPOTENCY|QUOTA|DONE)_' || true)"
  fail "sdk-caller explicit approved targetRef did not return a served target/content marker (markers=${markers:-<none>})"
  exit 1
fi

NOTIFICATION_ID="$(printf '%s' "$logs" | sed -n 's/.*E2E_SDK_CLIENT_NOTIFICATION_OK=\([^[:space:]]*\).*/\1/p' | tail -1)"
if [ -z "$NOTIFICATION_ID" ]; then
  fail "could not parse notificationId from sdk-caller logs"
  exit 1
fi
NOTIFICATION_DEDUPE_KEY="${NOTIFICATION_ID}:plugin_workload_sdk.notification"

delivery_row="$(wait_for_notification_delivery_row "$NOTIFICATION_ID" "$E2E_WAIT_NOTIFICATION_ROW" || true)"
if printf "%s" "$delivery_row" | grep -Eq "plugin_workload_sdk.notification\|(queued|retrying)\|${USER_REF}"; then
  ok "notification_deliveries queued for desktop stream (audience.userId=${USER_REF})"
else
  fail "notification_deliveries row missing or mis-targeted within ${E2E_WAIT_NOTIFICATION_ROW}s (got '${delivery_row}')"
  exit 1
fi

header "Desktop grace + channel fallback eligibility"

if notifications_desktop_first_enabled >/dev/null; then
  # Assert the grace deferral relative to the row's own created_at, NOT to a live
  # NOW(). Both columns are stamped in the same INSERT statement (deferred branch:
  # next_attempt_at = NOW() + grace; ELSE branch: next_attempt_at = NOW()), so
  # next_attempt_at > created_at proves the desktop-first grace window was applied.
  # Comparing against a live NOW() here is racy: the caller fixture's preceding
  # promptBridge / idempotency / quota steps can consume more than the grace
  # window before this assertion runs, expiring it and masking a correct deferral.
  grace_active="$(psql_query "SELECT (next_attempt_at > created_at)::text FROM notification_deliveries WHERE dedupe_key='${NOTIFICATION_DEDUPE_KEY}';" \
    | tr -d '[:space:]')"
  if is_psql_true "$grace_active"; then
    ok "desktop-first grace window deferred channel fallback (next_attempt_at > created_at)"
  else
    fail "notification_deliveries next_attempt_at was not deferred for desktop grace (got '${grace_active}')"
    exit 1
  fi

  psql_query "UPDATE notification_deliveries SET next_attempt_at = NOW() - INTERVAL '10 seconds' WHERE dedupe_key='${NOTIFICATION_DEDUPE_KEY}';" >/dev/null 2>&1

  fallback_eligible="$(psql_query "SELECT (next_attempt_at <= NOW() AND status IN ('queued','retrying'))::text FROM notification_deliveries WHERE dedupe_key='${NOTIFICATION_DEDUPE_KEY}';" \
    | tr -d '[:space:]')"
  if is_psql_true "$fallback_eligible"; then
    ok "channel fallback claim window opened after grace expiry"
  else
    fail "notification remained ineligible for channel fallback after grace expiry (got '${fallback_eligible}')"
    exit 1
  fi
else
  warn "NOTIFICATIONS_DESKTOP_FIRST_ENABLED=false — skipping desktop grace + fallback eligibility checks"
fi

header "Notification preferences PUT contract"

desktop_session_token="$(obtain_desktop_session_token "$E2E_DESKTOP_USER_EMAIL" "$E2E_DESKTOP_PASSWORD" || true)"
if [ -z "$desktop_session_token" ]; then
  fail "could not obtain desktop session token for ${E2E_DESKTOP_USER_EMAIL} (HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown} via ${E2E_EXTERNAL_REST_API_URL}; response body redacted)"
  exit 1
fi
ok "obtained desktop session token for notification-preferences checks"

if ! snapshot_notification_preferences; then
  fail "could not snapshot notification preferences for ${USER_REF}"
  exit 1
fi
ok "snapshotted notification preferences before the contract PUT checks"

session_curl PUT "/external/me/notification-preferences" "$desktop_session_token" '{"preferredMedium":"telegram"}' >/dev/null || true
if [ "$ADMIN_CURL_HTTP_STATUS" = "400" ] &&
   printf '%s' "$ADMIN_CURL_BODY" | grep -q 'invalid_channel_fallback_enabled'; then
  ok "partial PUT without channelFallbackEnabled is rejected with invalid_channel_fallback_enabled"
else
  fail "partial notification-preferences PUT was not rejected as expected (HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}; response body redacted)"
  exit 1
fi

session_curl PUT "/external/me/notification-preferences" "$desktop_session_token" '{"channelFallbackEnabled":true}' >/dev/null || true
if [ "$ADMIN_CURL_HTTP_STATUS" = "400" ] &&
   printf '%s' "$ADMIN_CURL_BODY" | grep -q 'invalid_preferred_medium'; then
  ok "partial PUT without preferredMedium is rejected with invalid_preferred_medium"
else
  fail "partial notification-preferences PUT without preferredMedium was not rejected as expected (HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}; response body redacted)"
  exit 1
fi

session_curl PUT "/external/me/notification-preferences" "$desktop_session_token" '{"preferredMedium":null,"channelFallbackEnabled":false}' >/dev/null || true
if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then
  fail "full notification-preferences PUT expected HTTP 200, got ${ADMIN_CURL_HTTP_STATUS:-unknown} (response body redacted)"
  exit 1
fi

prefs_fallback="$(psql_query "SELECT channel_fallback_enabled::text FROM user_notification_preferences WHERE user_id='${USER_REF}'::uuid;" \
  | tr -d '[:space:]')"
if is_psql_false "$prefs_fallback"; then
  ok "user_notification_preferences.channel_fallback_enabled persisted as false after full PUT"
else
  fail "user_notification_preferences.channel_fallback_enabled was not persisted as false (got '${prefs_fallback}')"
  exit 1
fi

# Audit trail: both invocations recorded.
invocations="$(admin_curl GET "/api/v1/admin/plugin-workload-sdk/invocations?recipeName=${RECIPE_NAME}" || true)"
if printf "%s" "$invocations" | grep -q '"method":"promptBridge"' &&
   printf "%s" "$invocations" | grep -q '"method":"clientNotifications"'; then
  ok "invocation audit trail contains both promptBridge and clientNotifications records"
else
  fail "invocation audit trail is missing SDK records"
  exit 1
fi

# ─── idempotency + quota enforcement ─────────────────────────────────────

# Idempotency: a completed prompt replay is guarded before the provider seam.
# The current public SDK route returns idempotency_conflict because completion
# content is not persisted for replay; the audit record and quota reservation
# remain stable, and the fixture's subsequent quota phase proves no second
# provider charge was made.
if printf "%s" "$logs" | grep -q E2E_SDK_IDEMPOTENCY_REPLAY_GUARDED; then
  ok "idempotency replay was guarded without a second provider call or quota charge"
else
  fail "idempotency replay was not guarded as expected (fixture marker missing; caller logs redacted)"
  exit 1
fi

# Per-minute rate limit (issue #348): the 5th call is rejected once the grant's
# maxInvocationsPerMinute=4 window is exhausted. The wire error code stays
# `quota_exceeded` (HTTP 429), so the fixture marker names are unchanged.
if printf "%s" "$logs" | grep -q E2E_SDK_QUOTA_EXCEEDED_OK; then
  ok "per-minute rate limit rejected the 5th call after 4/4 invocations in the window"
else
  fail "per-minute rate limit did not reject the 5th call (fixture marker missing; caller logs redacted). If the caller logged E2E_SDK_QUOTA_EXCEEDED_FAIL=*:possible_window_ageout or *:window_overcount, this is the known per-minute 60s-window timing race (issue #348 fixture caveat), not a real enforcement regression — retry on a less loaded cluster or apply the coordinated gate+fixture reset fix."
  exit 1
fi

# ─── issue #375 (P3): event-driven grant→WRC nudge ───────────────────────
# The recipe is validated by now. Prove the grant→WRC nudge accelerates
# convergence: a grant change must re-reconcile the recipe (bump the published
# policyRevision, state stays validated) within ≤10s via Postgres LISTEN/NOTIFY,
# WITHOUT restarting the WRC (the ≤30s watchdog is the backstop, not the path
# under test). This is an ADDITIVE phase; it does not touch #351's phases.
header "Plugin Workload SDK grant→WRC event-driven nudge (issue #375 P3)"

# Capture the WRC pod set (by name) so we can assert NO restart drove convergence.
wrc_pods_before="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \
  | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"
if [ -z "$wrc_pods_before" ]; then
  fail "could not enumerate workflow-recipes pods before the nudge (selector empty)"
  exit 1
fi

nudge_rev_before="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
  -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"
nudge_rev_before="${nudge_rev_before:-0}"

# Re-upsert the promptBridge grant. The admin upsert increments policy_revision
# and (issue #375 P3) emits the transactional grant-update NOTIFY on COMMIT.
# issue #375 (R5): fail LOUD on a grant re-upsert failure. `|| true` here only
# guards `set -e` so we can read ADMIN_CURL_HTTP_STATUS; the explicit assert
# below is the real gate. Without it, an admin-API failure (expired token,
# control-api 5xx) would fall through to the poll and be misattributed to the
# nudge under test ("did not accelerate convergence"), hiding the real cause.
create_grant promptBridge >/dev/null 2>&1 || true
if [ "$ADMIN_CURL_HTTP_STATUS" != "200" ]; then
  fail "grant re-upsert failed (admin API HTTP ${ADMIN_CURL_HTTP_STATUS:-unknown}) — cannot test the event-driven nudge"
  exit 1
fi

nudge_found=0
nudge_state=""
nudge_rev_after=""
# issue #375 (R1-L4): measure the printed elapsed from the SAME origin as the
# deadline (post-upsert), so the cosmetic "~Ns (≤Xs)" line cannot exceed the cap.
nudge_started_at="$SECONDS"
nudge_deadline=$((SECONDS + E2E_WAIT_NUDGE_VALIDATED))
while [ "$SECONDS" -lt "$nudge_deadline" ]; do
  gate_assert_deadline "waiting for event-driven policyRevision bump (issue #375 P3)"
  nudge_state="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
    -o jsonpath='{.status.pluginWorkloadSdk.state}' 2>/dev/null || true)"
  nudge_rev_after="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$RECIPE_NS" \
    -o jsonpath='{.status.pluginWorkloadSdk.policyRevision}' 2>/dev/null || true)"
  if [ "$nudge_state" = "validated" ] && [ -n "$nudge_rev_after" ] &&
     [ "$nudge_rev_after" -gt "$nudge_rev_before" ] 2>/dev/null; then
    nudge_found=1
    break
  fi
  sleep "$POLL_INTERVAL"
done

nudge_elapsed=$((SECONDS - nudge_started_at))
if [ "$nudge_found" -eq 1 ]; then
  ok "grant change re-published (policyRevision ${nudge_rev_before}→${nudge_rev_after}, state=validated) in ~${nudge_elapsed}s via the event-driven nudge (≤${E2E_WAIT_NUDGE_VALIDATED}s)"
else
  fail "grant change was NOT re-published within ${E2E_WAIT_NUDGE_VALIDATED}s (state='${nudge_state:-<empty>}', policyRevision '${nudge_rev_before}'→'${nudge_rev_after:-<empty>}'); the LISTEN/NOTIFY nudge did not accelerate convergence"
  exit 1
fi

# The acceleration must come from the NOTIFY-driven reconcile, NOT a WRC restart.
wrc_pods_after="$(kctl get pods -n "$CONTROL_NS" -o name 2>/dev/null \
  | grep -E '^pod/workflow-recipes-' | sort | tr '\n' ',' || true)"
if [ "$wrc_pods_after" = "$wrc_pods_before" ]; then
  ok "WRC was NOT restarted during convergence (pod set unchanged) — convergence was event-driven"
else
  fail "WRC pod set changed during the nudge phase (before='${wrc_pods_before}' after='${wrc_pods_after}'); convergence must not depend on a restart"
  exit 1
fi

print_results
exit 0