Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.7 KB
Raw Blame
#!/usr/bin/env bash
# ═══════════════════════════════════════════════════════════════════════
# E2E — Figure D multi-bot per-CommunicationChannel delivery + cross-bot authz
# ═══════════════════════════════════════════════════════════════════════
#
# Companion gate for PR1 (.ralph/plans/figure-d-multibot-pr1.md). Proves, on a
# live cluster, what unit tests cannot:
#
#   B4 (always)      — migration 0039_wama_communication_channel_ref applied: communication_channel_ref column
#                      + idx_wama_channel_ref index + status CHECK admits
#                      skipped_no_bot + ONLY telegram NULL-ref rows force-disabled
#                      (Slack / other models untouched).
#   metric (always)  — control-api exposes workflow_approval_delivery_skipped_no_bot_total.
#   B1/B2/B3 (opt-in)— real two-bot delivery, cross-bot block, and skipped_no_bot,
#                      gated on E2E_FIGURE_D_BOT_A_TOKEN / _BOT_B_TOKEN. Without
#                      them the scenarios SKIP loudly (never silently pass).
#   B2b              — the deployed reader's decisionHandler actually parses a
#                      16-hex (64-bit) channelAlias. Guards against a stale reader
#                      image whose compact regex still expects 8 hex — which the
#                      consulta-only B2 cannot detect (it never hits the reader's
#                      /webhooks/telegram callback-parse path).
#   B5/B6            — non-regression of Figure C + Modo A is covered by
#                      scripts/e2e/e2e-workflow-approvals*.sh (run those too).
#
# Usage:
#   KUBECONTEXT=clerum-test bash scripts/e2e/e2e-figure-d-multibot.sh
#   # Optional two-bot scenarios:
#   E2E_FIGURE_D_BOT_A_TOKEN=... E2E_FIGURE_D_BOT_B_TOKEN=... \
#     KUBECONTEXT=clerum-test bash scripts/e2e/e2e-figure-d-multibot.sh
#
# ═══════════════════════════════════════════════════════════════════════

set -u
set -o pipefail

RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'

# KUBECONTEXT should be set explicitly to a branch/PR-owned profile. Defaulting
# to the shared `clerum-test` profile risks producing PR evidence against a stale
# shared cluster. We still allow it (it is a valid local profile per CLAUDE.md),
# but warn loudly + print evidence (HEAD/context) when it was not set explicitly.
KUBECONTEXT_WAS_SET=1; [ -z "${KUBECONTEXT:-}" ] && KUBECONTEXT_WAS_SET=0
KCTX="${KUBECONTEXT:-clerum-test}"
CONTROL_PLANE_NS="${CONTROL_PLANE_NS:-control-plane}"

pass=0; fail=0; skip=0; total=0
log()  { echo -e "${CYAN}[figure-d-multibot]${NC} $*"; }
ok()   { echo -e "  ${GREEN}PASS${NC} $*"; pass=$((pass+1)); total=$((total+1)); }
no()   { echo -e "  ${RED}FAIL${NC} $*"; fail=$((fail+1)); total=$((total+1)); }
skp()  { echo -e "  ${YELLOW}SKIP${NC} $*"; skip=$((skip+1)); total=$((total+1)); }

# ── Allowed-context guard (CLAUDE.md): never touch a non-Clerum cluster ──
case "$KCTX" in
  clerum-test|clerum-codex-*|clerum-detached-*|clerum-feat-*|clerum-fix-*|gke_your-gcp-project_us-central1-a_example-dev|gke_your-gcp-project_us-central1-a_clerum) ;;
  *) echo -e "${RED}Refusing to run against non-Clerum context: $KCTX${NC}"; exit 2 ;;
esac

kc() { kubectl --context="$KCTX" "$@"; }

psql_q() {
  local pg
  pg="$(kc -n "$CONTROL_PLANE_NS" get pod -l app=control-postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)"
  [ -z "$pg" ] && { echo "__NO_PG__"; return 1; }
  kc -n "$CONTROL_PLANE_NS" exec "$pg" -- psql -U postgres -d profiles -tA -c "$1" 2>/dev/null
}

log "Context: $KCTX"
if [ "$KUBECONTEXT_WAS_SET" = "0" ]; then
  log "${YELLOW}WARNING: KUBECONTEXT not set — defaulted to the SHARED 'clerum-test' profile.${NC}"
  log "${YELLOW}  For trustworthy PR/branch evidence, set KUBECONTEXT=clerum-feat-<topic>-<sha>.${NC}"
fi
if command -v git >/dev/null 2>&1; then
  log "Evidence: HEAD=$(git rev-parse --short HEAD 2>/dev/null || echo '?') context=$KCTX"
fi

# ─── B4: migration 0039_wama_communication_channel_ref applied ────────────────────────────────────────
log "B4 — migration 0039_wama_communication_channel_ref + schema"

ver="$(psql_q "SELECT version FROM schema_migrations WHERE version = '0039_wama_communication_channel_ref';")"
[ "$ver" = "0039_wama_communication_channel_ref" ] \
  && ok "schema_migrations records 0039" \
  || no "0039 not recorded in schema_migrations (got: '$ver')"

col="$(psql_q "SELECT column_name FROM information_schema.columns WHERE table_name='workflow_approval_medium_accounts' AND column_name='communication_channel_ref';")"
[ "$col" = "communication_channel_ref" ] \
  && ok "wama.communication_channel_ref column exists" \
  || no "communication_channel_ref column missing"

idx="$(psql_q "SELECT indexname FROM pg_indexes WHERE tablename='workflow_approval_medium_accounts' AND indexname='idx_wama_channel_ref';")"
[ "$idx" = "idx_wama_channel_ref" ] \
  && ok "idx_wama_channel_ref index exists" \
  || no "idx_wama_channel_ref index missing"

# CHECK admits skipped_no_bot: inspect the status CHECK constraint definition.
admit="$(psql_q "SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='notification_deliveries'::regclass AND contype='c' AND pg_get_constraintdef(oid) LIKE '%skipped_no_bot%';")"
[ -n "$admit" ] && ok "notification_deliveries.status CHECK admits skipped_no_bot" || no "status CHECK missing skipped_no_bot (constraint does not admit it)"

# Force-disable is telegram-only: no active telegram row may have a NULL ref.
orphan="$(psql_q "SELECT count(*) FROM workflow_approval_medium_accounts WHERE medium='telegram' AND communication_channel_ref IS NULL AND disabled_at IS NULL;")"
[ "$orphan" = "0" ] \
  && ok "no active telegram rows with NULL channel ref (force-disable applied)" \
  || no "found $orphan active telegram rows with NULL channel ref"

# ─── Metric exposed ────────────────────────────────────────────────────
log "metric — skipped_no_bot counter"
metric="$(kc -n "$CONTROL_PLANE_NS" exec deploy/control-api -- \
  sh -c 'wget -qO- http://localhost:8090/metrics 2>/dev/null || curl -s http://localhost:8090/metrics 2>/dev/null' 2>/dev/null \
  | grep -c 'workflow_approval_delivery_skipped_no_bot_total' || true)"
[ "${metric:-0}" -ge 1 ] 2>/dev/null \
  && ok "workflow_approval_delivery_skipped_no_bot_total exposed on /metrics" \
  || skp "metric not observed (control-api may gate /metrics; verify manually)"

# ─── B1/B2/B3: automated two-bot harness (fake-telegram-server) ────────
# Runs without real Telegram bots: the minikube overlay deploys
# tests/e2e/fixtures/fake-telegram-server and points the delivery worker at it
# (WORKFLOW_APPROVAL_TELEGRAM_API_ROOT override). The harness seeds DB rows the
# delivery worker + reader consulta consume, then asserts behaviour. Every
# seeded row / CR / Secret is cleaned up. Skips loudly when the fixture is absent
# (e.g. GKE), never silently passes.
log "B1/B2/B3 — two-bot delivery + cross-bot + no_bot (fake-telegram harness)"

CHANNELS_NS="${CHANNELS_NS:-channels}"
FD_FAKE_DEPLOY="${FD_FAKE_DEPLOY:-fake-telegram-server}"

# filtered psql: strip psql command-status tags (INSERT 0 1 …) so RETURNING
# captures are clean.
fd_q() { psql_q "$1" | grep -vE '^(INSERT|UPDATE|DELETE|SELECT|BEGIN|COMMIT|ROLLBACK)( [0-9]|$)'; }
fd_fake() { kc -n "$CHANNELS_NS" exec "$FD_FAKE_POD" -- node -e "$1" "${@:2}"; }
fd_ca()   { kc -n "$CONTROL_PLANE_NS" exec "$FD_CA_POD" -- node -e "$1" "${@:2}"; }
# 16 hex / 64-bit channelAlias — must match CHANNEL_ALIAS_LEN in control-api.
fd_channel_alias() { python3 -c "import hashlib,sys;print(hashlib.sha256(sys.argv[1].encode()).hexdigest()[:16])" "$1"; }
fd_metric() {
  kc -n "$CONTROL_PLANE_NS" exec deploy/control-api -- \
    sh -c 'wget -qO- http://localhost:8090/metrics 2>/dev/null || curl -s http://localhost:8090/metrics 2>/dev/null' 2>/dev/null \
    | awk '/^workflow_approval_delivery_skipped_no_bot_total\{medium="telegram"\}/{print $2}' | head -1
}

FD_FAKE_POD="$(kc -n "$CHANNELS_NS" get pod -l app="$FD_FAKE_DEPLOY" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
FD_CA_POD="$(kc -n "$CONTROL_PLANE_NS" get pod -l app=control-api -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"

if [ -z "$FD_FAKE_POD" ] || [ -z "$FD_CA_POD" ]; then
  skp "B1 multi-bot delivery — fake-telegram-server not deployed (minikube overlay only)"
  skp "B2 cross-bot block — requires fake-telegram harness + control-api"
  skp "B3 skipped_no_bot — requires fake-telegram harness"
  log "Deploy the minikube overlay (deploy/overlays/minikube) to run B1/B2/B3 automatically."
else
  TSX="$(date +%s)$$"
  mkchan() { # name secret token chat puid  (empty token => no telegram-bot-token key)
    if [ -n "$3" ]; then
      kc -n "$CHANNELS_NS" create secret generic "$2" --from-literal=telegram-bot-token="$3" >/dev/null 2>&1
    else
      kc -n "$CHANNELS_NS" create secret generic "$2" --from-literal=placeholder=x >/dev/null 2>&1
    fi
    printf 'apiVersion: clerum.io/v1alpha1\nkind: CommunicationChannel\nmetadata:\n  name: %s\n  namespace: %s\nspec:\n  hostRef: chatllm\n  credentialsSecretRef:\n    name: %s\n  telegram:\n    - channelId: "%s"\n      chatType: private\n      userIds: ["%s"]\n' \
      "$1" "$CHANNELS_NS" "$2" "$4" "$5" | kc apply -f - >/dev/null 2>&1
  }
  seed_delivery() { # email puid chat ccref idemp dedupe -> echoes uid
    local uid
    uid=$(fd_q "INSERT INTO users (email) VALUES ('$1') RETURNING id;")
    fd_q "INSERT INTO workflow_approval_medium_accounts (user_id,medium,provider_user_id,provider_channel_id,communication_channel_ref,verified_at) VALUES ('$uid','telegram','$2','$3','$4',NOW());" >/dev/null
    local war
    war=$(fd_q "INSERT INTO workflow_approval_requests (recipe_namespace,recipe_name,expires_at,status,target_user_id,payload,idempotency_key) VALUES ('sandbox-recipes','figd-e2e',NOW()+INTERVAL '1 hour','pending','$uid','{\"title\":\"Approve?\",\"actions\":[{\"label\":\"Approve\",\"id\":\"approve\"}]}'::jsonb,'$5') RETURNING id;")
    fd_q "INSERT INTO notification_deliveries (event_type,dedupe_key,audience,payload,status) VALUES ('approval.requested','$6','{\"userId\":\"$uid\"}'::jsonb,'{\"approvalRequestId\":\"$war\"}'::jsonb,'queued') RETURNING id;" >/dev/null
    echo "$uid"
  }
  wait_status() { # delivery_user_id expected_status -> echoes final status
    local u="$1" exp="$2" st=""
    for _ in 1 2 3 4 5 6 7 8; do
      sleep 3
      st=$(fd_q "SELECT status FROM notification_deliveries WHERE audience->>'userId'='$u';")
      [ "$st" = "$exp" ] && break
      case "$st" in sent|failed|skipped_no_bot) break;; esac
    done
    echo "$st"
  }
  cleanup_user() { # user_id
    fd_q "DELETE FROM notification_deliveries WHERE audience->>'userId'='$1';" >/dev/null
    fd_q "DELETE FROM workflow_approval_requests WHERE target_user_id='$1';" >/dev/null
    fd_q "DELETE FROM workflow_approval_medium_accounts WHERE user_id='$1';" >/dev/null
    fd_q "DELETE FROM users WHERE id='$1';" >/dev/null
  }
  del_chan() { kc -n "$CHANNELS_NS" delete communicationchannel "$1" >/dev/null 2>&1 || true; kc -n "$CHANNELS_NS" delete secret "$2" >/dev/null 2>&1 || true; }

  # ── B1: per-channel two-bot delivery (each delivery uses ITS channel's bot) ──
  fd_fake 'const h=require("http");const r=h.request({host:"127.0.0.1",port:8080,path:"/reset",method:"POST"},x=>x.resume());r.end()' >/dev/null 2>&1
  B1_TOKA="botA${TSX}"; B1_TOKB="botB${TSX}"; B1_CHATA="71${TSX}"; B1_CHATB="72${TSX}"
  mkchan "figd-bota-${TSX}" "figd-seca-${TSX}" "$B1_TOKA" "$B1_CHATA" "uidA${TSX}"
  mkchan "figd-botb-${TSX}" "figd-secb-${TSX}" "$B1_TOKB" "$B1_CHATB" "uidB${TSX}"
  B1_UA=$(seed_delivery "figd-b1a-${TSX}@clerum.io" "uidA${TSX}" "$B1_CHATA" "channels/figd-bota-${TSX}" "idmA${TSX}" "ddA${TSX}")
  B1_UB=$(seed_delivery "figd-b1b-${TSX}@clerum.io" "uidB${TSX}" "$B1_CHATB" "channels/figd-botb-${TSX}" "idmB${TSX}" "ddB${TSX}")
  B1_SA=$(wait_status "$B1_UA" sent); B1_SB=$(wait_status "$B1_UB" sent)
  B1_SENDS=$(fd_fake 'const h=require("http");h.get("http://127.0.0.1:8080/sends",x=>{let d="";x.on("data",c=>d+=c);x.on("end",()=>console.log(d))})' 2>/dev/null)
  B1_VERDICT=$(printf '%s' "$B1_SENDS" | python3 -c "
import sys,json
try: sends=json.load(sys.stdin).get('sends',[])
except Exception: sends=[]
ta,tb,ca,cb='$B1_TOKA','$B1_TOKB','$B1_CHATA','$B1_CHATB'
ok = any(s.get('token')==ta and str(s.get('chat_id'))==ca for s in sends) \
 and any(s.get('token')==tb and str(s.get('chat_id'))==cb for s in sends) \
 and not any(s.get('token')==ta and str(s.get('chat_id'))==cb for s in sends) \
 and not any(s.get('token')==tb and str(s.get('chat_id'))==ca for s in sends)
print('OK' if ok else 'BAD')
" 2>/dev/null)
  if [ "$B1_SA" = "sent" ] && [ "$B1_SB" = "sent" ] && [ "$B1_VERDICT" = "OK" ]; then
    ok "B1 per-channel two-bot delivery — A->botA, B->botB, no cross-bot leak"
  else
    no "B1 per-channel delivery (A=$B1_SA B=$B1_SB verdict=$B1_VERDICT)"
  fi
  cleanup_user "$B1_UA"; cleanup_user "$B1_UB"
  del_chan "figd-bota-${TSX}" "figd-seca-${TSX}"; del_chan "figd-botb-${TSX}" "figd-secb-${TSX}"

  # ── B2: cross-bot identity block (D1 STRICT) at the control-api CONSULTA ──
  # Scope note: this exercises the consulta endpoint DIRECTLY (the reader's
  # pre-check barrier #1). It does NOT replay a Telegram callback through the
  # reader's /webhooks/telegram → mcp-host → provider-decision path. The full
  # reader round-trip (forged callback -> 403 cross_bot_mismatch + WAR stays
  # pending; correct callback -> WAR approved -> step resumes) is validated by
  # the live weld in .local-notes/b7-*.sh, not by this gate. Do not read B2 as a
  # full end-to-end callback test.
  B2_TOKEN=$(kc -n "$CHANNELS_NS" get secret workflow-approval-request-reader-credentials -o jsonpath='{.data.control-api-token}' 2>/dev/null | base64 -d 2>/dev/null)
  if [ -z "$B2_TOKEN" ]; then
    no "B2 cross-bot block — reader control-api-token secret not found"
  else
    B2_PUID="figd-b2-${TSX}"; B2_CHANA="channels/figd-chanA-${TSX}"; B2_CHANB="channels/figd-chanB-${TSX}"
    B2_AA=$(fd_channel_alias "$B2_CHANA"); B2_AB=$(fd_channel_alias "$B2_CHANB")
    B2_UID=$(fd_q "INSERT INTO users (email) VALUES ('figd-b2-${TSX}@clerum.io') RETURNING id;")
    fd_q "INSERT INTO workflow_approval_medium_accounts (user_id,medium,provider_user_id,provider_channel_id,communication_channel_ref,verified_at) VALUES ('$B2_UID','telegram','$B2_PUID','73${TSX}','$B2_CHANA',NOW());" >/dev/null
    B2_WAR=$(fd_q "INSERT INTO workflow_approval_requests (recipe_namespace,recipe_name,expires_at,status,target_user_id,payload,idempotency_key) VALUES ('sandbox-recipes','figd-e2e-b2',NOW()+INTERVAL '1 hour','pending','$B2_UID','{}'::jsonb,'figd-b2-${TSX}') RETURNING id;")
    b2_consulta() { # alias providerUserId
      fd_ca 'const http=require("http");const [id,alias,puid,token]=process.argv.slice(1);const u="http://localhost:8090/api/v1/internal/workflow-approval-reader/approvals/"+id+"/can-approve?medium=telegram&providerUserId="+encodeURIComponent(puid)+"&channelAlias="+alias;http.get(u,{headers:{"x-service-token":"workflow-approval-reader","Authorization":"Bearer "+token}},r=>{let d="";r.on("data",c=>d+=c);r.on("end",()=>console.log(r.statusCode+" "+d))}).on("error",e=>console.log("ERR "+e.message));' "$B2_WAR" "$1" "$2" "$B2_TOKEN"
    }
    B2_OK=$(b2_consulta "$B2_AA" "$B2_PUID")
    B2_FORGE=$(b2_consulta "$B2_AB" "$B2_PUID")
    B2_NOACC=$(b2_consulta "$B2_AA" "nobody-${TSX}")
    if printf '%s' "$B2_OK" | grep -q '"canApprove":true' \
       && printf '%s' "$B2_FORGE" | grep -q '"reason":"cross_bot_mismatch"' \
       && printf '%s' "$B2_NOACC" | grep -q '"reason":"account_not_verified"'; then
      ok "B2 cross-bot block (consulta D1 STRICT, direct endpoint) — matching alias approves, forged -> cross_bot_mismatch, unknown user -> account_not_verified"
    else
      no "B2 cross-bot block (ok=$B2_OK forge=$B2_FORGE noacc=$B2_NOACC)"
    fi
    cleanup_user "$B2_UID"
  fi

  # ── B2b: reader decisionHandler parses a 16-hex channelAlias (stale-image guard) ──
  # B2 exercises ONLY the control-api consulta. It cannot catch a reader image whose
  # compact callback regex still expects an 8-hex channelAlias (the 64-bit widening,
  # PR1). Such a stale reader rejects every Figure-D callback with
  # invalid_decision_payload BEFORE any cross-bot logic runs, yet B2 still passes —
  # the gap a live weld (.local-notes/b7-*.sh) caught but the gate did not. This check
  # exercises the reader's OWN parse path: it runs the DEPLOYED reader's compiled
  # normalizeTelegramDecision against a compact 16-hex callback and asserts the
  # channelAlias round-trips. decisionHandler has no import side effects, so a bare
  # `node import` of the built module is safe.
  FD_READER_POD="$(kc -n "$CHANNELS_NS" get pod -l app=workflow-approval-request-reader -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
  if [ -z "$FD_READER_POD" ]; then
    skp "B2b reader decisionHandler parse — reader pod not found in $CHANNELS_NS"
  else
    B2B_ROUTE=$(fd_channel_alias "sandbox-recipes/figd-b2b-${TSX}")
    B2B_CHAN=$(fd_channel_alias "channels/figd-b2b-${TSX}")
    B2B_OUT=$(kc -n "$CHANNELS_NS" exec "$FD_READER_POD" -- node --input-type=module -e '
import { normalizeTelegramDecision } from "/app/dist/decisionHandler.js";
const [route, chan] = process.argv.slice(1);
const b64 = Buffer.from("99999999888877776666555555555555", "hex").toString("base64url");
const data = `a:${b64}:~${route}:${chan}`;
if (Buffer.byteLength(data, "utf8") > 64) { console.log("OVER64"); process.exit(0); }
const r = normalizeTelegramDecision({ callback_query: { id: "x", from: { id: 1 }, message: { chat: { id: 2, type: "private" } }, data } });
console.log(JSON.stringify({ alias: (r && r.channelAlias) || null, match: !!(r && r.channelAlias === chan) }));
' "$B2B_ROUTE" "$B2B_CHAN" 2>/dev/null)
    if printf '%s' "$B2B_OUT" | grep -q '"match":true'; then
      ok "B2b reader decisionHandler parses 16-hex channelAlias (deployed reader image not stale)"
    else
      no "B2b reader decisionHandler did NOT parse 16-hex channelAlias — stale reader image? (out=$B2B_OUT)"
    fi
  fi

  # ── B3: skipped_no_bot — channel whose Secret has no telegram-bot-token ──
  B3_M0=$(fd_metric); B3_M0=${B3_M0:-0}
  mkchan "figd-nobot-${TSX}" "figd-secnb-${TSX}" "" "74${TSX}" "uidNB${TSX}"
  B3_U=$(seed_delivery "figd-b3-${TSX}@clerum.io" "uidNB${TSX}" "74${TSX}" "channels/figd-nobot-${TSX}" "idmNB${TSX}" "ddNB${TSX}")
  B3_ST=$(wait_status "$B3_U" skipped_no_bot)
  B3_M1=$(fd_metric); B3_M1=${B3_M1:-0}
  if [ "$B3_ST" = "skipped_no_bot" ] && awk "BEGIN{exit !($B3_M1>$B3_M0)}"; then
    ok "B3 skipped_no_bot — no-token channel -> status skipped_no_bot + metric increment ($B3_M0->$B3_M1)"
  else
    no "B3 skipped_no_bot (status=$B3_ST metric=$B3_M0→$B3_M1)"
  fi
  cleanup_user "$B3_U"; del_chan "figd-nobot-${TSX}" "figd-secnb-${TSX}"
fi

# ─── B5/B6 pointers ────────────────────────────────────────────────────
log "B5/B6 — non-regression (run separately):"
log "  scripts/e2e/e2e-workflow-approvals.sh           (Figure C + token surface)"
log "  scripts/e2e/e2e-workflow-approvals-recovery.sh  (recovery/rotation)"

echo
log "Summary: ${GREEN}${pass} pass${NC} / ${RED}${fail} fail${NC} / ${YELLOW}${skip} skip${NC} (of $total)"
[ "$fail" -eq 0 ] || exit 1