The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
chore(scripts): add keepgoing e2e test script
Steven Enamakelbab159a74ab2
diff --git a/scripts/keepgoing-e2e.sh b/scripts/keepgoing-e2e.shnew file mode 100644index 0000000000..9642b8558d--- /dev/null+++ b/scripts/keepgoing-e2e.sh@@ -0,0 +1,394 @@+#!/usr/bin/env bash+#+# Rust E2E suite — the cargo-test counterpart to the Tauri E2E specs.+#+# Boots the mock backend (same `scripts/mock-api-server.mjs` the Tauri+# E2E uses) on a fixed port and then runs each `tests/*_e2e.rs`+# integration test against it. Tests that don't currently consume the+# mock backend still run here so we keep one place to add new+# mock-driven integration tests over time.+#+# This is invoked from:+# - `pnpm test:rust:e2e` (local dev + Docker)+# - `.github/workflows/e2e.yml` (the `rust-e2e-linux` job)+#+# Usage:+# ./scripts/test-rust-e2e.sh # all default e2e tests+# ./scripts/test-rust-e2e.sh --suite json_rpc_e2e # one specific suite+# ./scripts/test-rust-e2e.sh -- --ignored # extra cargo-test args+#+# Env knobs:+# MOCK_API_PORT — mock backend port (default 18505).+# MOCK_LOG — path for mock server stdout/stderr.+#+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"++# The full set of `tests/*_e2e.rs` files. The default runner executes them+# serially so CI does not link several large integration binaries at once.+# Tests guarded by `#[ignore]` stay skipped unless the caller passes+# `-- --ignored`.+# The suite list is DERIVED from `tests/*_e2e.rs` plus the `in_process_all`+# aggregate (the former in-process router suites now live under+# `tests/in_process/`), not hand-maintained.+#+# It used to be a literal list of 20 names while 29 `tests/*_e2e.rs` targets+# existed, so nine were silently absent from this runner — including+# `transcript_search_e2e`, which a coverage matrix cited as existing coverage.+# CI never had the gap: `test-reusable.yml`, `scripts/ci/rust-coverage.sh` and+# `scripts/test-rust-with-mock.sh` all discover targets with the same `find`+# used below. Only the documented LOCAL command was short, which made a local+# green weaker than it read.+#+# Polarity matters here. An include list fails UNSAFE: a new target is omitted+# until someone remembers to add it. The exclude list below fails SAFE: a new+# target runs by default and must be deliberately opted out, with a cause. Keep+# it that way, and keep it empty if you can — a target that needs an external+# service should gate itself with `#[ignore]` or an env check, which costs+# nothing here because the target still runs and simply reports zero tests.+E2E_SUITE_EXCLUDE=()++_discover_e2e_suites() {+ local name+ while IFS= read -r name; do+ local skip=0+ local excluded+ for excluded in ${E2E_SUITE_EXCLUDE[@]+"${E2E_SUITE_EXCLUDE[@]}"}; do+ [ "$name" = "$excluded" ] && skip=1 && break+ done+ [ $skip -eq 0 ] && printf '%s\n' "$name"+ done < <(+ find "$REPO_ROOT/tests" -maxdepth 1 -type f \( -name '*_e2e.rs' -o -name 'in_process_all.rs' \) -print |+ sed -e 's#.*/##' -e 's#\.rs$##' |+ sort+ )+}++ALL_E2E_SUITES=()+while IFS= read -r _suite; do+ ALL_E2E_SUITES+=("$_suite")+done < <(_discover_e2e_suites)++# Refuse to run on an empty discovery rather than reporting a vacuous success.+# `SUITES` falling back to an empty `ALL_E2E_SUITES` would run nothing and exit+# 0, which reads exactly like a passing suite.+if [ "${#ALL_E2E_SUITES[@]}" -eq 0 ]; then+ echo "[rust-e2e] ERROR: discovered 0 e2e suites under $REPO_ROOT/tests." >&2+ echo " Expected tests/*_e2e.rs targets; refusing to report success." >&2+ exit 2+fi++# Parse args: --suite <name> can be passed multiple times to filter.+# Everything after `--` is forwarded to cargo test as test-binary args.+SUITES=()+EXTRA_ARGS=()+HAS_EXTRA_ARGS=0+while [ $# -gt 0 ]; do+ case "$1" in+ --suite)+ # Guard against `set -u` blowing up on `--suite` with no argument+ # — turning that into a clear usage error is friendlier than+ # the cryptic "$2: unbound variable" from bash.+ if [ $# -lt 2 ] || [ -z "${2:-}" ]; then+ echo "[rust-e2e] ERROR: --suite requires a test name (e.g. --suite json_rpc_e2e)" >&2+ exit 2+ fi+ SUITES+=("$2")+ shift 2+ ;;+ --)+ shift+ EXTRA_ARGS+=("$@")+ if [ "$#" -gt 0 ]; then+ HAS_EXTRA_ARGS=1+ fi+ break+ ;;+ *)+ EXTRA_ARGS+=("$1")+ HAS_EXTRA_ARGS=1+ shift+ ;;+ esac+done+if [ "${#SUITES[@]}" -eq 0 ]; then+ SUITES=("${ALL_E2E_SUITES[@]}")+fi++MOCK_API_PORT="${MOCK_API_PORT:-18505}"+MOCK_API_URL="http://127.0.0.1:${MOCK_API_PORT}"+MOCK_LOG="${MOCK_LOG:-/tmp/openhuman-rust-e2e-mock.log}"+MOCK_PID=""+CREATED_XDG_DATA_HOME=""+CREATED_XDG_CACHE_HOME=""+XDG_ROOT="${OPENHUMAN_XDG_ROOT:-${TMPDIR:-/tmp}}"++# TinyBus rejects module-search paths under a shared runner HOME mount. Keep+# module state created by this test runner in an owner-only scratch directory.+if [ -z "${XDG_DATA_HOME:-}" ]; then+ XDG_DATA_HOME="$(mktemp -d "${XDG_ROOT}/openhuman-rust-e2e-xdg.XXXXXX")"+ chmod 700 "$XDG_DATA_HOME"+ export XDG_DATA_HOME+ CREATED_XDG_DATA_HOME="$XDG_DATA_HOME"+fi++# dirs::cache_dir uses XDG_CACHE_HOME rather than XDG_DATA_HOME. Give it a+# separate private root: the release cache must not also be scanned as a+# TinyBus module search path.+if [ -z "${XDG_CACHE_HOME:-}" ]; then+ XDG_CACHE_HOME="$(mktemp -d "${XDG_ROOT}/openhuman-rust-e2e-cache.XXXXXX")"+ chmod 700 "$XDG_CACHE_HOME"+ CREATED_XDG_CACHE_HOME="$XDG_CACHE_HOME"+ export XDG_CACHE_HOME+fi++cleanup() {+ if [ -n "$MOCK_PID" ]; then+ kill "$MOCK_PID" 2>/dev/null || true+ wait "$MOCK_PID" 2>/dev/null || true+ fi+ if [ -n "$CREATED_XDG_DATA_HOME" ]; then+ rm -rf "$CREATED_XDG_DATA_HOME" 2>/dev/null || true+ fi+ if [ -n "$CREATED_XDG_CACHE_HOME" ]; then+ rm -rf "$CREATED_XDG_CACHE_HOME" 2>/dev/null || true+ fi+}+trap cleanup EXIT++echo "[rust-e2e] Starting mock API server on ${MOCK_API_URL} ..."+node "$SCRIPT_DIR/mock-api-server.mjs" --port "$MOCK_API_PORT" >"$MOCK_LOG" 2>&1 &+MOCK_PID=$!++for i in $(seq 1 30); do+ if curl -sf "${MOCK_API_URL}/__admin/health" >/dev/null 2>&1; then+ break+ fi+ if [ "$i" -eq 30 ]; then+ echo "[rust-e2e] ERROR: mock API server did not become healthy in time." >&2+ echo "[rust-e2e] See logs: $MOCK_LOG" >&2+ exit 1+ fi+ sleep 1+done+echo "[rust-e2e] Mock backend healthy."++export BACKEND_URL="$MOCK_API_URL"+export VITE_BACKEND_URL="$MOCK_API_URL"++# The Linux/macOS CI host may not provide a usable Secret Service. Supply a+# per-run encryption key so the core's encrypted storage backend can persist+# the E2E session without depending on the host keyring. Never print the key.+if [ -z "${OPENHUMAN_KEYRING_MASTER_KEY:-}" ] && [ -z "${OPENHUMAN_KEYRING_MASTER_KEY_FILE:-}" ]; then+ OPENHUMAN_KEYRING_MASTER_KEY="$(openssl rand -hex 32)"+ export OPENHUMAN_KEYRING_MASTER_KEY+fi++# The agent-harness E2E surface drives very large async futures in debug builds+# (the typed sub-agent runner + the full agentic brain turn exercised by+# json_rpc_meet_agent_session_lifecycle). The default Rust test-thread stack+# (2 MiB) overflows on that dispatch depth — a stack overflow in otherwise-correct+# tests, not a logic failure. Mirror scripts/test-rust-with-mock.sh and give the+# suite a larger stack unless the caller already pinned one explicitly.+export RUST_MIN_STACK="${RUST_MIN_STACK:-16777216}"++cd "$REPO_ROOT"+source "$HOME/.cargo/env" 2>/dev/null || true+RUSTC_BIN="$(command -v rustc)"+CARGO_BIN="${CARGO_BIN:-$(dirname "$RUSTC_BIN")/cargo}"+if [ ! -x "$CARGO_BIN" ]; then+ CARGO_BIN="$(command -v cargo)"+fi++# This is the product E2E runner, not the slim contributor build. Several+# suites below intentionally require product gates (for example `voice` for+# `json_rpc_e2e` and `memory-git` for memory artifact coverage), so invoking+# them with Cargo's default feature set makes the runner fail before a test can+# execute. Keep this list in the same canonical source as `pnpm test:rust`.+PRODUCT_FEATURES="$(bash "$REPO_ROOT/scripts/ci/product-features.sh")"+# scripts/e2e-core.sh adds a storage driver (storage-mongodb) the product set lacks.+if [ -n "${OPENHUMAN_E2E_EXTRA_FEATURES:-}" ]; then+ PRODUCT_FEATURES="$PRODUCT_FEATURES,$OPENHUMAN_E2E_EXTRA_FEATURES"+fi++module_target="$(bash "$SCRIPT_DIR/ci/native-test-module-platform.sh")"+IFS=$'\t' read -r module_ext module_platform <<< "$module_target"+# shellcheck source=modules/stage-local-modules.sh+source "$SCRIPT_DIR/modules/stage-local-modules.sh"+# With OPENHUMAN_MODULE_SOURCE=source the modules below are built from the pinned+# submodules (scripts/modules/build.mjs); otherwise TinyBox and TinyMCP are the+# registry-pinned releases, as in CI.+e2e_source_ids=()+if local_module_source_is_source; then+ e2e_source_ids+=(tinyconnectors tinysearch tinybox tinymcp)+elif [ "${OPENHUMAN_TEST_MODULES_STAGED:-}" = "1" ]; then+ echo "[rust-e2e] Using pre-staged native modules from OPENHUMAN_MODULE_PATH."+else+ # Shell command analysis and the storage URL E2E suites load TinyBox through+ # the native module path; the ignored MCP registry cases need TinyMCP. Both+ # come from the registry-pinned release, digest-verified before any+ # subprocess starts.+ for released_id in tinyconnectors tinyflows tinysearch tinybox tinymcp; do+ released_dir="$(stage_release_module "$released_id" "$module_platform" "$module_ext")"+ export OPENHUMAN_MODULE_PATH="$released_dir${OPENHUMAN_MODULE_PATH:+:$OPENHUMAN_MODULE_PATH}"+ done+fi++# Module-backed Composio coverage must use a staged artifact as well. Without+# it the core resolves TinyConnectors through release metadata at run time,+# turning an otherwise hermetic mock-backend suite into a network dependency+# and permanently faulting that process when the lookup fails. The isolated+# module-admission suite likewise needs a real native search module. The+# released archives are staged above (digest-verified); only the local+# OPENHUMAN_MODULE_SOURCE=source mode builds anything, through build.mjs.+if [ "${#e2e_source_ids[@]}" -gt 0 ]; then+ echo "[rust-e2e] Building pinned local test modules: ${e2e_source_ids[*]} ..."+ stage_source_modules "${e2e_source_ids[@]}"+fi+for source_id in ${e2e_source_ids[@]+"${e2e_source_ids[@]}"}; do+ export OPENHUMAN_MODULE_PATH="$REPO_ROOT/target/module-sources/$source_id${OPENHUMAN_MODULE_PATH:+:$OPENHUMAN_MODULE_PATH}"+done++echo "[rust-e2e] Running ${#SUITES[@]} suite(s) serially."++run_json_rpc_e2e_suite() {+ # JSON-RPC scenarios mutate process-global provider routes and runtime+ # configuration. Run every case in a fresh test process so a provider set by+ # one scenario cannot affect the routing assertions in another.+ #+ # Enumerate first, as its own checked step. The names used to be read straight+ # from a process substitution, whose exit status nothing sees: a target that+ # failed to compile listed nothing, the loop ran zero times, and the suite+ # reported success. A listing that fails part-way is refused whole, so a+ # partial list is never run as though it were complete.+ local listing test_names test_name+ if ! listing="$(+ "$CARGO_BIN" test --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test json_rpc_e2e -- --list+ )"; then+ echo "[rust-e2e] ERROR: could not enumerate json_rpc_e2e tests; refusing to run an empty or partial list." >&2+ return 1+ fi++ test_names="$(printf '%s\n' "$listing" | sed -n 's/: test$//p')"+ if [ -z "$test_names" ]; then+ echo "[rust-e2e] ERROR: json_rpc_e2e enumeration returned no tests; refusing to report success." >&2+ return 1+ fi++ while IFS= read -r test_name; do+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test json_rpc_e2e $test_name"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test json_rpc_e2e "$test_name" -- \+ --exact --test-threads=1 ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} || echo "[rust-e2e] FAILED-TEST: $test_name"+ done <<<"$test_names"+}++run_in_process_modules() {+ # `in_process_all` folds ~20 former `tests/*.rs` targets into one binary. Run+ # each module in its own process, as those targets did, so process globals+ # (the transport install, the RPC bearer, module singletons) stay per suite.+ local module+ while IFS= read -r module; do+ [ -n "$module" ] || continue+ echo "[rust-e2e] in_process module: ${module}"+ if [ "$module" = "config_auth_app_state_connectivity_e2e" ]; then+ # These RPC fixtures share process-global credential and listener state;+ # serialize them so one test cannot drop another test's ephemeral server.+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test in_process_all -- "${module}::" --test-threads=1 \+ ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ continue+ fi+ if [ "$module" = "domain_modules_e2e" ]; then+ # TinyFlows and TinyMCP use process-global module runtimes. A different+ # test's RPC host teardown can leave a later module call unavailable, so+ # run module-backed cases in fresh processes and keep the remaining+ # domain tests together in their normal harness.+ local -a isolated_module_tests=(+ domain_modules_e2e::cron_update_applies_a_partial_patch_without_clobbering_unset_fields+ domain_modules_e2e::mcp_audit_list_dispatches_internally_and_stays_out_of_the_schema_dump+ domain_modules_e2e::mcp_clients_read_paths_validate_before_reaching_outward+ domain_modules_e2e::mcp_tool_description_scan_flags_injection_and_spares_ordinary_prose+ )+ local -a skip_module_tests=()+ local test_name+ for test_name in "${isolated_module_tests[@]}"; do+ skip_module_tests+=(--skip "$test_name")+ done+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test in_process_all -- "${module}::" --test-threads=1 \+ "${skip_module_tests[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ for test_name in "${isolated_module_tests[@]}"; do+ echo "[rust-e2e] isolated test: ${test_name}"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test in_process_all -- "$test_name" --exact ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ done+ continue+ fi+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test in_process_all -- "${module}::" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ done < <(+ find "$REPO_ROOT/tests/in_process" -maxdepth 1 -type f -name '*.rs' -print |+ sed -e 's#.*/##' -e 's#\.rs$##' |+ sort+ )+}++for suite in "${SUITES[@]}"; do+ if [ "$suite" = "json_rpc_e2e" ]; then+ run_json_rpc_e2e_suite+ continue+ fi+ if [ "$suite" = "in_process_all" ]; then+ run_in_process_modules+ continue+ fi+ if [ "$suite" = "mcp_registry_e2e" ]; then+ # The registry tests open several process-global module clients across+ # workspaces. Parallel libtest threads can race their module lifecycle.+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test $suite (serial)"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test "$suite" -- --ignored --test-threads=1 ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ continue+ fi+ if [ "$suite" = "tinymcp_storage_smoke_e2e" ]; then+ # Released TinyMCP cdylib under the real host and its storage backend+ # (#7737). The staging above put it on OPENHUMAN_MODULE_PATH. It spawns+ # openhuman-core twice, so run it serially.+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test $suite (serial)"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES,e2e-test-support" \+ --test "$suite" -- --test-threads=1 ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ continue+ fi+ if [ "$suite" = "tinydocs_linked_e2e" ]; then+ # This local-only E2E target is outside the product feature set. Enable+ # its required linked implementation explicitly instead of failing Cargo's+ # required-features check.+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test $suite (linked-modules)"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES,linked-modules" \+ --test "$suite" -- ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}+ continue+ fi++ if [ "$HAS_EXTRA_ARGS" -eq 1 ]; then+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test $suite -- ${EXTRA_ARGS[*]}"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" \+ --test "$suite" -- "${EXTRA_ARGS[@]}"+ else+ echo "[rust-e2e] $CARGO_BIN test --manifest-path Cargo.toml --test $suite"+ bash "$SCRIPT_DIR/ci-cancel-aware.sh" "$CARGO_BIN" test \+ --manifest-path Cargo.toml --features "$PRODUCT_FEATURES" --test "$suite"+ fi+done