Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

19.0 KB
Raw Blame
#!/usr/bin/env bash
#
# e2e-run-all-flows.sh — Master E2E orchestrator for all 66 WDIO specs.
#
# USAGE:
#   bash app/scripts/e2e-run-all-flows.sh [OPTIONS]
#
# OPTIONS:
#   --suite=SUITE     Run only one suite category. Valid values:
#                       auth, navigation, chat, skills, notifications,
#                       webhooks, providers, provider-web, payments, settings,
#                       system, journeys, all  (default: all)
#   --bail            Stop after the first spec failure (default: run all)
#   --skip-preflight  Skip the pre-flight environment check
#
# ENVIRONMENT:
#   E2E_ARTIFACTS_DIR  Directory where failure logs are copied.
#                      Default: app/test/e2e/artifacts/YYYYMMDD-HHMMSS
#
# REQUIREMENTS:
#   pnpm --filter openhuman-app test:e2e:build   (must be run first)
#
# Each spec runs to completion regardless of prior failures unless --bail is
# passed. A per-category mini-summary and a full summary are printed at the
# end. The script exits non-zero if any spec failed.
#
# (Previously `set -e` caused the first failure to abort the run and made
# the terminal appear to crash. `set -uo pipefail` preserves error detection
# without aborting mid-run.)
#
set -uo pipefail

APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO_DIR="$(cd "$APP_DIR/.." && pwd)"
cd "$APP_DIR" || {
  echo "[e2e-run-all-flows] Failed to cd into $APP_DIR" >&2
  exit 1
}

# ---------------------------------------------------------------------------
# Argument parsing
# ---------------------------------------------------------------------------
SUITE="all"
BAIL=0
SKIP_PREFLIGHT=0

for arg in "$@"; do
  case "$arg" in
    --suite=*)  SUITE="${arg#--suite=}" ;;
    --bail)     BAIL=1 ;;
    --skip-preflight) SKIP_PREFLIGHT=1 ;;
    *)
      echo "Unknown option: $arg" >&2
      echo "Usage: bash app/scripts/e2e-run-all-flows.sh [--suite=SUITE] [--bail] [--skip-preflight]" >&2
      exit 1
      ;;
  esac
done

VALID_SUITES="auth navigation chat skills notifications webhooks providers provider-web connectors payments settings system journeys all"

# Accept comma-separated suite lists, e.g. --suite=auth,navigation,system.
# CI sharding passes one such list per matrix shard so a few parallel jobs
# can cover the whole suite. `all` short-circuits to "everything".
IFS=',' read -r -a _REQUESTED_SUITES <<< "$SUITE"
for req in "${_REQUESTED_SUITES[@]}"; do
  match=0
  for s in $VALID_SUITES; do
    [[ "$req" == "$s" ]] && match=1 && break
  done
  if [[ $match -eq 0 ]]; then
    echo "Invalid suite: '$req'. Valid values: $VALID_SUITES" >&2
    exit 1
  fi
done

# ---------------------------------------------------------------------------
# Artifacts directory
# ---------------------------------------------------------------------------
E2E_ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-$APP_DIR/test/e2e/artifacts/$(date +%Y%m%d-%H%M%S)}"
export E2E_ARTIFACTS_DIR

# ---------------------------------------------------------------------------
# Spec collection: this script no longer invokes the runner once per spec.
# Instead `run()` accumulates spec paths into one list; at the very end we
# hand the whole list to `e2e-run-session.sh`, which launches the app +
# Appium + chromedriver ONCE and lets WDIO drive every spec inside a single
# shared session. The old per-spec orchestration paid CEF cold-start tax on
# every spec (~15-30s × 65 specs) and broke the contract in wdio.conf.ts
# ("WDIO creates ONE session per worker ... state from spec N flows into
# spec N+1"). Per-spec failure detail comes from WDIO's spec reporter now,
# not from a bash-side per-spec exit-code table.
#
# `--bail` is forwarded by env (E2E_BAIL_ON_FAILURE=1) so wdio.conf.ts can
# flip its `bail` count. Per-suite `--suite=` filtering is still honored at
# the run() call site.
# ---------------------------------------------------------------------------
_spec_paths=()    # collected spec paths, in declaration order
_spec_suites=()   # parallel array: suite name per collected spec
_spec_labels=()   # parallel array: human label per collected spec
_RUN_START_EPOCH=$(date +%s)

# ---------------------------------------------------------------------------
# run SPEC LABEL SUITE
#
# Appends the spec to the collected list; nothing runs yet. The actual WDIO
# invocation happens at the bottom of the script.
# ---------------------------------------------------------------------------
run() {
  local spec="$1"
  local label="${2:-$1}"
  local suite="${3:-unknown}"

  _spec_paths+=("$spec")
  _spec_suites+=("$suite")
  _spec_labels+=("$label")
}

# ---------------------------------------------------------------------------
# _copy_failure_logs
# Copies /tmp/openhuman-e2e-app-*.log files into E2E_ARTIFACTS_DIR once at
# end-of-run. With a single shared session there's now only one app log to
# capture (and Appium/chromedriver logs alongside).
# ---------------------------------------------------------------------------
_copy_failure_logs() {
  local logs driver_logs
  logs=$(ls /tmp/openhuman-e2e-app-*.log 2>/dev/null || true)
  driver_logs=$(ls "${RUNNER_TEMP:-${TMPDIR:-/tmp}}"/tauri-driver-*.log 2>/dev/null || true)
  if [[ -z "$logs" && -z "$driver_logs" ]]; then
    return
  fi
  mkdir -p "$E2E_ARTIFACTS_DIR"
  for f in $logs $driver_logs; do
    local dest="$E2E_ARTIFACTS_DIR/$(basename "$f" .log)-session.log"
    cp "$f" "$dest" 2>/dev/null || true
  done
  echo "[e2e-run-all-flows] Session logs copied to $E2E_ARTIFACTS_DIR"
}

# ---------------------------------------------------------------------------
# _mini_summary SUITE_NAME
# Print how many specs were collected for this suite (pre-run; WDIO will
# report per-spec pass/fail directly).
# ---------------------------------------------------------------------------
_mini_summary() {
  local suite="$1"
  local count=0
  for i in "${!_spec_labels[@]}"; do
    [[ "${_spec_suites[$i]}" == "$suite" ]] && (( count++ )) || true
  done
  printf "  [%s] %d spec(s) queued\n" "$suite" "$count"
}

# ---------------------------------------------------------------------------
# finish — print wall time + a markdown summary for CI job summary.
# Per-spec pass/fail comes from WDIO's spec reporter in the live output;
# the bash orchestrator no longer tracks per-spec exit codes.
# ---------------------------------------------------------------------------
_WDIO_EXIT_CODE=0
finish() {
  local t_end_epoch
  t_end_epoch=$(date +%s)
  local wall=$(( t_end_epoch - _RUN_START_EPOCH ))
  local wall_min=$(( wall / 60 ))
  local wall_sec=$(( wall % 60 ))
  local collected=${#_spec_paths[@]}

  echo ""
  echo "══════════════════════════════════════════════════════════════════"
  printf "  E2E run summary  ($(uname -s))  suite=%s\n" "$SUITE"
  echo "══════════════════════════════════════════════════════════════════"
  printf "  Specs queued: %d\n" "$collected"
  printf "  WDIO exit:    %d\n" "$_WDIO_EXIT_CODE"
  printf "  Wall time:    %dm %02ds\n" "$wall_min" "$wall_sec"
  echo "══════════════════════════════════════════════════════════════════"

  _copy_failure_logs

  {
    printf "## E2E Results ($(uname -s)) — suite=%s\n\n" "$SUITE"
    printf "| Field | Value |\n"
    printf "|-------|-------|\n"
    printf "| Specs queued | %d |\n" "$collected"
    printf "| WDIO exit code | %d |\n" "$_WDIO_EXIT_CODE"
    printf "| Wall time | %dm %02ds |\n" "$wall_min" "$wall_sec"
    printf "\nPer-spec pass/fail is in the WDIO spec-reporter output above.\n"
  } > /tmp/e2e-summary.txt

  if [[ $_WDIO_EXIT_CODE -ne 0 ]]; then
    exit "$_WDIO_EXIT_CODE"
  fi
}
trap finish EXIT

# ---------------------------------------------------------------------------
# Pre-flight check (unless --skip-preflight)
# ---------------------------------------------------------------------------
if [[ $SKIP_PREFLIGHT -eq 0 ]]; then
  if [[ -f "$APP_DIR/scripts/e2e-preflight.sh" ]]; then
    echo "[e2e-run-all-flows] Running pre-flight checks..."
    if ! bash "$APP_DIR/scripts/e2e-preflight.sh"; then
      echo "[e2e-run-all-flows] Pre-flight failed. Aborting." >&2
      exit 1
    fi
  else
    echo "[e2e-run-all-flows] Pre-flight script not found or not executable, skipping."
  fi
fi

# ---------------------------------------------------------------------------
# Helpers: should_run_suite SUITE_NAME
# Returns 0 (true) if this suite should run given --suite flag.
# ---------------------------------------------------------------------------
should_run_suite() {
  local want="$1"
  for req in "${_REQUESTED_SUITES[@]}"; do
    [[ "$req" == "all" || "$req" == "$want" ]] && return 0
  done
  return 1
}

# ---------------------------------------------------------------------------
# Auth & onboarding
# ---------------------------------------------------------------------------
if should_run_suite "auth"; then
  echo ""
  echo "## Running suite: auth"
  run "test/e2e/specs/smoke.spec.ts"                          "smoke"                     "auth"
  run "test/e2e/specs/login-flow.spec.ts"                     "login"                     "auth"
  run "test/e2e/specs/auth-access-control.spec.ts"            "auth"                      "auth"
  _mini_summary "auth"
fi

# ---------------------------------------------------------------------------
# Navigation & core UI
# ---------------------------------------------------------------------------
if should_run_suite "navigation"; then
  echo ""
  echo "## Running suite: navigation"
  run "test/e2e/specs/chip-tabs-keyboard.spec.ts"              "chip-tabs-keyboard"        "navigation"
  _mini_summary "navigation"
fi

# ---------------------------------------------------------------------------
# Chat & agent harness
# ---------------------------------------------------------------------------
if should_run_suite "chat"; then
  echo ""
  echo "## Running suite: chat"
  run "test/e2e/specs/chat-live-history-parity.spec.ts"       "chat-live-history-parity"  "chat"
  run "test/e2e/specs/chat-harness-subagent-continue.spec.ts" "chat-subagent-continue"    "chat"
  run "test/e2e/specs/chat-background-activity-panel.spec.ts" "chat-background-activity"  "chat"
  run "test/e2e/specs/chat-todos-goals.spec.ts"               "chat-todos-goals"          "chat"
  run "test/e2e/specs/agent-harness-behaviors.spec.ts"        "agent-harness-behaviors"   "chat"
  run "test/e2e/specs/file-drop-guard.spec.ts"                "file-drop-guard"           "chat"
  run "test/e2e/specs/chat-external-link.spec.ts"             "chat-external-link"        "chat"
  run "test/e2e/specs/chat-voice-note.spec.ts"                "chat-voice-note"           "chat"
  run "test/e2e/specs/agent-teams-live.spec.ts"               "agent-teams-live"          "chat"
  run "test/e2e/specs/mega-flow.spec.ts"                      "mega-flow"                 "chat"
  _mini_summary "chat"
fi

# ---------------------------------------------------------------------------
# Skills
# ---------------------------------------------------------------------------
if should_run_suite "skills"; then
  echo ""
  echo "## Running suite: skills"
  run "test/e2e/specs/skill-activate-invoke-chat.spec.ts"     "skill-activate-invoke"     "skills"
  run "test/e2e/specs/skill-activation-persistence.spec.ts"   "skill-activation-persist"  "skills"
  _mini_summary "skills"
fi

# ---------------------------------------------------------------------------
# Notifications, memory, cron
# ---------------------------------------------------------------------------
if should_run_suite "notifications"; then
  echo ""
  echo "## Running suite: notifications"
  run "test/e2e/specs/notifications.spec.ts"                  "notifications"             "notifications"
  _mini_summary "notifications"
fi

# ---------------------------------------------------------------------------
# Webhooks & tools
# ---------------------------------------------------------------------------
if should_run_suite "webhooks"; then
  echo ""
  echo "## Running suite: webhooks"
  _mini_summary "webhooks"
fi

# ---------------------------------------------------------------------------
# Provider flows
# ---------------------------------------------------------------------------
if should_run_suite "providers"; then
  echo ""
  echo "## Running suite: providers"
  run "test/e2e/specs/accounts-provider-modal.spec.ts"        "accounts-providers"        "providers"
  _mini_summary "providers"
fi

# Split browser-heavy provider flows out of the core provider shard. Telegram,
# Gmail, and accounts already consume several minutes of Appium/CEF runtime on
# Windows; keeping WhatsApp/conversations in the same shared session makes the
# later specs prone to "session is either terminated or not started" teardown
# failures before their own assertions run.
if should_run_suite "provider-web"; then
  echo ""
  echo "## Running suite: provider-web"
  # slack-flow currently crashes the CEF session mid-spec on Linux (#1850-style
  # state issue); skip until investigated rather than nuke the rest of the
  # provider suite.
  # run "test/e2e/specs/slack-flow.spec.ts"                   "slack"                     "provider-web"
  run "test/e2e/specs/whatsapp-flow.spec.ts"                  "whatsapp"                  "provider-web"
  # notion-flow.spec.ts was removed; skip to avoid "spec not found" failure.
  _mini_summary "provider-web"
fi

# ---------------------------------------------------------------------------
# Composio connector smoke specs.
#
# Split out of the `providers` suite into its own `connectors` shard so the
# 17 connector specs don't share a CEF session with the heavier provider
# flows (slack/whatsapp/etc.). The shared CEF process leaks resources over
# ~30+ specs and the second half of the suite hits 'A sessionId is
# required' / __simulateDeepLink-not-ready errors mid-run.
# ---------------------------------------------------------------------------
if should_run_suite "connectors"; then
  echo ""
  echo "## Running suite: connectors"
  # Table-driven contract spec covering the 11 formerly byte-identical
  # Composio connector flows (airtable/asana/clickup/confluence/gcal/gdrive/
  # gsheets/notion/slack/todoist/youtube) — see connector-contract.ts.
  run "test/e2e/specs/composio-cancel-pending.spec.ts"        "composio-cancel-pending"   "connectors"
  run "test/e2e/specs/connector-composio-contract.spec.ts"   "connector-composio-contract" "connectors"
  run "test/e2e/specs/connector-discord-composio.spec.ts"    "connector-discord"         "connectors"
  run "test/e2e/specs/connector-github.spec.ts"              "connector-github"          "connectors"
  run "test/e2e/specs/connector-jira.spec.ts"                "connector-jira"            "connectors"
  run "test/e2e/specs/connector-session-guard.spec.ts"       "connector-session-guard"   "connectors"
  _mini_summary "connectors"
fi

# ---------------------------------------------------------------------------
# Payments & rewards
# ---------------------------------------------------------------------------
if should_run_suite "payments"; then
  echo ""
  echo "## Running suite: payments"
  _mini_summary "payments"
fi

# ---------------------------------------------------------------------------
# Settings panels
# ---------------------------------------------------------------------------
if should_run_suite "settings"; then
  echo ""
  echo "## Running suite: settings"
  run "test/e2e/specs/settings-data-management.spec.ts"       "settings-data"             "settings"
  run "test/e2e/specs/memory-plan-required.spec.ts"           "memory-plan-required"      "settings"
  run "test/e2e/specs/settings-search.spec.ts"                "settings-search"           "settings"
  _mini_summary "settings"
fi

# ---------------------------------------------------------------------------
# System / AI / voice / Tauri
# ---------------------------------------------------------------------------
if should_run_suite "system"; then
  echo ""
  echo "## Running suite: system"
  run "test/e2e/specs/tauri-commands.spec.ts"                 "tauri-commands"            "system"
  # service-connectivity-flow tests the old sidecar service model removed in
  # PR #1061 (core is now in-process). Skip by not setting OPENHUMAN_SERVICE_MOCK=1.
  run "test/e2e/specs/service-connectivity-flow.spec.ts"    "service-connectivity"      "system"
  _mini_summary "system"
fi

# ---------------------------------------------------------------------------
# User journeys
# ---------------------------------------------------------------------------
if should_run_suite "journeys"; then
  echo ""
  echo "## Running suite: journeys"
  run "test/e2e/specs/flows.spec.ts"                               "flows"                 "journeys"
  _mini_summary "journeys"
fi

# ---------------------------------------------------------------------------
# Single shared WDIO session.
#
# All collected specs run inside one Appium/CEF session, restoring the
# contract in wdio.conf.ts. Per-spec pass/fail comes from WDIO's spec
# reporter (live stdout above). Exit code from e2e-run-session.sh is
# propagated to the `finish` summary trap.
#
# `--bail` is forwarded via E2E_BAIL_ON_FAILURE (wdio.conf.ts flips its
# `bail` count when this env is set).
# ---------------------------------------------------------------------------
if [[ ${#_spec_paths[@]} -eq 0 ]]; then
  echo "[e2e-run-all-flows] no specs matched suite=$SUITE — nothing to run." >&2
  exit 1
fi

echo ""
echo "──────────────────────────────────────────────────────────────────"
echo "  Launching WDIO for ${#_spec_paths[@]} spec(s)"
echo "──────────────────────────────────────────────────────────────────"

if [[ $BAIL -eq 1 ]]; then
  export E2E_BAIL_ON_FAILURE=1
fi

set +e
if [[ "$(uname -s)" == "Linux" && "${E2E_USE_TAURI_DRIVER:-}" == "1" ]]; then
  # tauri-driver creates a new WebDriver session for every spec, even when
  # WDIO serializes its workers. After a few sequential sessions the driver
  # can stop accepting POST /session, so give each Linux spec a clean driver
  # and app lifecycle. A retry has to restart that lifecycle too: retrying
  # within the same driver only repeats a hung POST /session request.
  for spec in "${_spec_paths[@]}"; do
    status=1
    for attempt in 1 2; do
      bash "$APP_DIR/scripts/e2e-run-session.sh" "$spec"
      status=$?
      [[ $status -eq 0 ]] && break
      if [[ $attempt -eq 1 ]]; then
        echo "[e2e-run-all-flows] retrying ${spec} with a fresh Linux driver/app lifecycle" >&2
      fi
    done
    if [[ $status -ne 0 ]]; then
      _WDIO_EXIT_CODE=$status
      [[ $BAIL -eq 1 ]] && break
    fi
  done
else
  bash "$APP_DIR/scripts/e2e-run-session.sh" "${_spec_paths[@]}"
  _WDIO_EXIT_CODE=$?
fi
set -e

# finish() trap will print the summary and exit with _WDIO_EXIT_CODE.