Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
E2E in CI: reusable lane workflow, PR fast lane, nightly runs
Petr Bauch9360a6decab8
diff --git a/.github/workflows/e2e-scheduled.yml b/.github/workflows/e2e-scheduled.ymlnew file mode 100644index 000000000..d923cdeea--- /dev/null+++ b/.github/workflows/e2e-scheduled.yml@@ -0,0 +1,77 @@+name: E2E scheduled++# When the end-to-end runs that are not PR checks happen. What a run does is+# e2e.yml's; this file only says when, and reports.+#+# after a merge to main the provider lane on a throwaway stack. Real model+# calls, so not on every PR, and never a required check+# nightly the provider lane again, and the fast lane against+# dev, then one summary notification for the night+#+# The fast lane on a throwaway stack is a PR check in pr-ci.yml.+#+# The notification goes to the incoming webhook in the repository secret+# E2E_NOTIFY_WEBHOOK_URL (Slack and Mattermost both accept the payload). Without+# it the run says so in a warning and sends nothing.++on:+ schedule:+ - cron: '41 3 * * *'+ push:+ branches: [main]+ paths-ignore:+ - 'docs/**'+ - '**/*.md'+ workflow_dispatch:++permissions:+ contents: read++jobs:+ provider:+ name: Provider lane+ uses: ./.github/workflows/e2e.yml+ with:+ lane: provider+ environment: ephemeral+ agent_type: claude-code++ fast-dev:+ name: Fast lane on dev+ if: ${{ github.event_name != 'push' }}+ uses: ./.github/workflows/e2e.yml+ with:+ lane: fast+ environment: dev+ agent_type: claude-code++ notify:+ name: Nightly summary+ needs: [provider, fast-dev]+ if: ${{ always() && github.event_name == 'schedule' }}+ runs-on: ubuntu-latest+ steps:+ - name: Send the summary+ env:+ WEBHOOK: ${{ secrets.E2E_NOTIFY_WEBHOOK_URL }}+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}+ PROVIDER_RESULT: ${{ needs.provider.result }}+ PROVIDER_SUMMARY: ${{ needs.provider.outputs.summary }}+ DEV_RESULT: ${{ needs.fast-dev.result }}+ DEV_SUMMARY: ${{ needs.fast-dev.outputs.summary }}+ run: |+ line() {+ local mark=❌+ [ "$2" = success ] && mark=✅+ printf '%s %s — %s\n' "$mark" "$1" "${3:-$2, no scenario summary}"+ }+ text="$(printf 'Switch E2E nightly\n%s\n%s\n%s' \+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \+ "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \+ "$RUN_URL")"+ echo "$text"+ if [ -z "$WEBHOOK" ]; then+ echo "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; the nightly summary was not sent anywhere."+ exit 0+ fi+ curl -fsS -H 'Content-Type: application/json' -d "$(jq -n --arg text "$text" '{text: $text}')" "$WEBHOOK" >/dev/nulldiff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlnew file mode 100644index 000000000..31a6eea3c--- /dev/null+++ b/.github/workflows/e2e.yml@@ -0,0 +1,226 @@+name: E2E++# The one definition of an end-to-end run: a lane of console/tooling-e2e against+# an environment. Called by pr-ci.yml (the fast lane on a throwaway stack, on+# every PR) and by e2e-scheduled.yml (the provider lane after each merge, the+# deployed environments nightly), and dispatchable by hand.+#+# lane fast provider-free scenarios over plain HTTP+# provider real agent sessions on a headless agent controller+# environment ephemeral a fresh stack on the runner, built from this checkout+# dev, staging an existing deployment, configured by the GitHub+# Environment of that name+#+# Fork safety: an ephemeral fast run generates every secret it uses on the+# runner and reads no repository or environment secret, so it runs the same+# from a fork PR's read-only token. Everything that needs a secret enters a+# GitHub Environment, whose protection rules decide which refs may run it:+#+# e2e-provider secret ANTHROPIC_API_KEY, for the controller's Claude Code+# dev, staging vars SWITCH_API_URL, MATTERMOST_URL, MATTERMOST_TEAM; secrets+# SWITCH_GATEWAY_ADMIN_EMAIL, SWITCH_GATEWAY_ADMIN_PASSWORD,+# SWITCH_AGENT_REGISTRATION_TOKEN, MATTERMOST_TOKEN+#+# Runs are strict (SWITCH_E2E_STRICT=1): a scenario that cannot run fails+# rather than skips, so a deployment without Mattermost configured fails the+# Mattermost scenario instead of passing without it.+#+# Artifacts are uploaded only after the output has been scanned for the run's+# own secret values and by gitleaks. A scan that fails or does not run means+# nothing is uploaded.++on:+ workflow_call:+ inputs:+ lane:+ type: string+ required: true+ environment:+ type: string+ required: true+ agent_type:+ description: Provider for the provider lane+ type: string+ required: true+ outputs:+ summary:+ description: One line counting the lane's scenarios by outcome+ value: ${{ jobs.run.outputs.summary }}+ workflow_dispatch:+ inputs:+ lane:+ type: choice+ options: [fast, provider]+ default: fast+ environment:+ type: choice+ options: [ephemeral, dev, staging]+ default: ephemeral+ agent_type:+ description: Provider for the provider lane+ type: choice+ options: [claude-code]+ default: claude-code++permissions:+ contents: read++jobs:+ run:+ name: ${{ inputs.lane }} lane on ${{ inputs.environment }}+ runs-on: ubuntu-latest+ timeout-minutes: 45+ environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || (inputs.lane == 'provider' && 'e2e-provider' || '') }}+ # One run at a time against a shared deployment, queued rather than+ # cancelled: a cancelled run leaves its agents and rooms behind. A throwaway+ # stack is the runner's own, so ephemeral runs never wait on each other.+ concurrency:+ group: ${{ inputs.environment == 'ephemeral' && format('e2e-ephemeral-{0}-{1}', inputs.lane, github.run_id) || format('e2e-{0}', inputs.environment) }}+ cancel-in-progress: false+ outputs:+ summary: ${{ steps.summary.outputs.line }}+ defaults:+ run:+ # Named explicitly for -o pipefail, which the implicit default lacks:+ # without it a failed suite piped through tee would pass.+ shell: bash+ env:+ OUT: ${{ github.workspace }}/build/e2e/out+ LANE: ${{ inputs.lane }}+ TARGET: ${{ inputs.environment }}+ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Check the inputs+ env:+ AGENT_TYPE: ${{ inputs.agent_type }}+ run: |+ case "$LANE" in fast|provider) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esac+ case "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esac+ if [ "$LANE" = provider ] && [ "$TARGET" != ephemeral ]; then+ echo "::error::The provider lane runs on an ephemeral stack only."; exit 1+ fi+ if [ "$LANE" = provider ] && [ "$AGENT_TYPE" != claude-code ]; then+ echo "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1+ fi++ - name: Set up uv+ if: ${{ inputs.environment == 'ephemeral' }}+ uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5+ with:+ python-version: '3.12'+ enable-cache: true++ - name: Set up pnpm+ uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4+ with:+ version: 11.5.3++ - name: Set up Node+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version-file: console/.nvmrc+ cache: pnpm+ cache-dependency-path: console/pnpm-lock.yaml++ - name: Install dependencies+ working-directory: console+ run: pnpm install --frozen-lockfile++ - name: Install Claude Code+ if: ${{ inputs.lane == 'provider' }}+ run: npm install -g @anthropic-ai/[email protected]++ # Into the controller's environment through --env-file; never into $OUT.+ - name: Write the provider key file+ if: ${{ inputs.lane == 'provider' }}+ env:+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ run: |+ if [ -z "$ANTHROPIC_API_KEY" ]; then+ echo "::error::The e2e-provider environment has no ANTHROPIC_API_KEY secret."; exit 1+ fi+ umask 077+ printf 'ANTHROPIC_API_KEY=%s\n' "$ANTHROPIC_API_KEY" >"$RUNNER_TEMP/provider.env"+ echo "SWITCH_E2E_PROVIDER_ENV_FILE=$RUNNER_TEMP/provider.env" >>"$GITHUB_ENV"++ - name: Run the lane on a throwaway stack+ if: ${{ inputs.environment == 'ephemeral' }}+ env:+ SWITCH_E2E_LANE: ${{ inputs.lane }}+ SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}+ E2E_OUT: ${{ env.OUT }}+ run: console/tooling-e2e/scripts/ephemeral-stack.sh++ - name: Run the lane against ${{ inputs.environment }}+ if: ${{ inputs.environment != 'ephemeral' }}+ working-directory: console/tooling-e2e+ env:+ SWITCH_E2E_LANE: ${{ inputs.lane }}+ SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}+ SWITCH_E2E_STRICT: '1'+ SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}+ SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}+ MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}+ MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}+ run: |+ mkdir -p "$OUT"+ pnpm exec vitest run --reporter=verbose --reporter=junit --outputFile.junit="$OUT/junit.xml" \+ 2>&1 | tee "$OUT/vitest.log"++ # What interrupted runs left behind. The floor keeps it off a run still in+ # progress, which is why it can run after every suite.+ - name: Remove stale harness artefacts from ${{ inputs.environment }}+ if: ${{ always() && inputs.environment != 'ephemeral' }}+ working-directory: console/tooling-e2e+ env:+ SWITCH_E2E_LANE: ${{ inputs.lane }}+ SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}+ SWITCH_E2E_STRICT: '1'+ SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}+ SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}+ MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}+ MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}+ run: pnpm cleanup --older-than 180++ - name: Summarise the run+ id: summary+ if: ${{ always() }}+ run: |+ node --experimental-strip-types console/tooling-e2e/src/ci-summary.ts \+ "$OUT/junit.xml" "$LANE" "$LANE lane on $TARGET" >>"$GITHUB_STEP_SUMMARY"++ - name: Scan the output for secrets+ id: scan+ if: ${{ always() }}+ env:+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}+ run: |+ [ -d "$OUT" ] || { echo "No output to scan."; exit 1; }+ if [ "$TARGET" = ephemeral ]; then+ # The stack script scans for the secrets it generated and the+ # provider key file, and leaves this file only when none turned up.+ [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }+ else+ printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN" \+ | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"+ fi+ docker run --rm -v "$OUT:/out:ro" zricethezav/gitleaks:v8.30.1 dir /out --redact --no-banner++ - name: Upload the output+ if: ${{ always() && steps.scan.outcome == 'success' }}+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: e2e-${{ inputs.lane }}-${{ inputs.environment }}+ path: ${{ env.OUT }}+ include-hidden-files: true+ retention-days: 14diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex 7021810e9..b514582c1 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -74,6 +74,7 @@ jobs:artifacts: ${{ steps.filter.outputs.artifacts }}migrations: ${{ steps.filter.outputs.migrations }}chart: ${{ steps.filter.outputs.chart }}+ e2e: ${{ steps.filter.outputs.e2e }}steps:- name: Checkoutuses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4@@ -105,6 +106,15 @@ jobs:gateway:- *shared- 'gateway/**'+ # The server, the stack the fast lane stands up, and the harness.+ e2e:+ - *shared+ - 'core/**'+ - 'console/tooling-e2e/**'+ - 'console/pnpm-lock.yaml'+ - 'deploy/local/**'+ - '.env.example'+ - '.github/workflows/e2e.yml'# Its own filter, narrower than `backend`, so the push-triggered# `migration-heads-on-merge` job below only runs when it can# actually find a new head — not on every backend-touching push.@@ -360,6 +370,19 @@ jobs:- name: Test (vitest)run: npm test+ # Real messages through a real server and Mattermost, on a stack the job+ # builds from this checkout. Needs no secret, so a fork PR runs it the same.+ # The provider lane and the deployed environments are e2e-scheduled.yml's.+ e2e-fast:+ name: E2E fast lane+ needs: changes+ if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}+ uses: ./.github/workflows/e2e.yml+ with:+ lane: fast+ environment: ephemeral+ agent_type: claude-code+migration-heads-on-merge:name: Migration heads on merge# Only meaningful right after a new migration lands on main — a PR stilldiff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex 643c1cc82..71b82c4b3 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -65,7 +65,9 @@ Typecheck: `pnpm typecheck`.| `src/fixtures/` | Throwaway agents and rooms for a scenario, the bridge-readiness probe, teardown || `src/fast/` | Fast-lane scenarios, one file per area, listed in `index.ts` || `src/provider/` | The provider lane: Console credential files, the shared agent and channel, the scenarios, `seed.ts` |-| `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast` |+| `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast` and `just e2e-provider` |+| `scripts/scan-artifacts.sh` | Fails if a run's output holds one of the secrets the run used |+| `src/ci-summary.ts` | The CI run summary, read from the JUnit report |## Scenarios and their headers@@ -208,6 +210,40 @@ Two things differ from a Console session:The controller's log goes to `build/e2e/ephemeral/controller.log`.+## In CI++`.github/workflows/e2e.yml` defines a run: one lane against one environment+(`ephemeral`, `dev` or `staging`). Other workflows call it, and it can also be+dispatched by hand.++| When | Lane | Environment | Required check |+| --- | --- | --- | --- |+| Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes |+| After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no |+| Nightly (`e2e-scheduled.yml`) | provider; fast | ephemeral; dev | no |++- **Secrets.** An ephemeral fast run generates every secret it uses on the+ runner, so it works the same on a fork PR. Every other run reads its secrets+ from a GitHub Environment:+ - `e2e-provider` holds `ANTHROPIC_API_KEY`;+ - `dev` and `staging` hold the deployment's URLs as variables and its+ credentials as secrets.++ The header of `e2e.yml` lists every name. Each environment's protection rules+ decide which branches may use it.+- **Concurrency.** Runs against a shared deployment queue behind each other.+ They are never cancelled, because a cancelled run leaves its artefacts+ behind. After every deployed run, `pnpm cleanup --older-than 180` removes+ what interrupted runs left.+- **Artifacts.** The output directory is uploaded only after it passes two+ scans: one for the exact secret values the run used (`scan-artifacts.sh`),+ and gitleaks. On an ephemeral stack, the env file, the controller's state and+ the agent's directory live outside the output directory, so the output holds+ only logs and the JUnit report.+- **Reporting.** Each run writes a scenario table to the job summary. A+ scheduled night sends one summary to the webhook in the repository secret+ `E2E_NOTIFY_WEBHOOK_URL`.+## With a Switch Console sessionThe provider lane can also run against a **Switch Console session for the agentdiff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.shindex 6629a587f..2c149d8df 100755--- a/console/tooling-e2e/scripts/ephemeral-stack.sh+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh@@ -19,7 +19,12 @@# secrets or pick up a stale value from them.## Ports 5432, 8000, 8065 and 8081 must be free: stop a local stack first.-# Everything the run writes goes to build/e2e/ephemeral/ (or $E2E_OUT).+# Logs and the JUnit report go to build/e2e/ephemeral/ (or $E2E_OUT); the env+# file, the controller's state and the agent's working directory go to a+# temporary directory, so the output holds nothing secret by construction.+# On exit the output is scanned for every secret the run used, and+# .secret-scan-passed is written there only if none turned up: CI uploads the+# output only when that file exists.set -euo pipefailREPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"@@ -48,17 +53,12 @@ for port in 5432 8000 8065 8081; dodonerm -rf "$OUT"; mkdir -p "$OUT"-ENV_FILE="$OUT/stack.env"-cp "$REPO/.env.example" "$ENV_FILE"-"$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e-if [ "$LANE" = provider ]; then- # Agent management, and the key controllers' access tokens are signed with.- sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \- -e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"- rm -f "$ENV_FILE.bak"-fi-chmod 600 "$ENV_FILE"-set -a; source "$ENV_FILE"; set +a+WORK="$(mktemp -d "${TMPDIR:-/tmp}/switch-e2e.XXXXXX")"+chmod 700 "$WORK"+ENV_FILE="$WORK/stack.env"+# Every secret the run uses, one per line, for the scan of $OUT on exit.+SECRETS="$WORK/secrets"+: >"$SECRETS"cd "$REPO"COMPOSE=(docker compose -p switch-e2e-ephemeral -f deploy/local/docker-compose.yml@@ -71,16 +71,49 @@ cleanup() {[ -n "$CONTROLLER_PID" ] && kill "$CONTROLLER_PID" 2>/dev/null && wait "$CONTROLLER_PID" 2>/dev/null[ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null && wait "$SERVER_PID" 2>/dev/null"${COMPOSE[@]}" down -v >/dev/null 2>&1 || true- rm -f "$ENV_FILE"elseecho "Left running: switch-core :8000 (pid $SERVER_PID), Mattermost :8065${CONTROLLER_PID:+, agent controller (pid $CONTROLLER_PID)}."- echo "Secrets for this stack: $ENV_FILE. Tear down with:"- echo " kill $SERVER_PID $CONTROLLER_PID; ${COMPOSE[*]} down -v"+ echo "Secrets and state for this stack: $WORK. Tear down with:"+ echo " kill $SERVER_PID $CONTROLLER_PID; ${COMPOSE[*]} down -v; rm -rf $WORK"fi+ if "$HARNESS/scripts/scan-artifacts.sh" "$OUT" <"$SECRETS"; then+ : >"$OUT/.secret-scan-passed"+ else+ status=1+ fi+ [ "$KEEP" = 0 ] && rm -rf "$WORK"exit "$status"}trap cleanup EXIT+cp "$REPO/.env.example" "$ENV_FILE"+"$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e+if [ "$LANE" = provider ]; then+ # Agent management, and the key controllers' access tokens are signed with.+ sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \+ -e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"+ rm -f "$ENV_FILE.bak"+fi+chmod 600 "$ENV_FILE"+set -a; source "$ENV_FILE"; set +a++# A secret's value, and each part of it, since SECRET_KEYS is `label:key,…`.+remember_secrets() {+ local value+ for value in "$@"; do+ printf '%s\n' "$value" >>"$SECRETS"+ tr ',:' '\n\n' <<<"$value" >>"$SECRETS"+ done+}+while IFS='=' read -r key value; do+ case "$key" in *SECRET*|*PASSWORD*|*TOKEN*|*_KEY|*_KEYS) remember_secrets "$value" ;; esac+done < <(grep -E '^[A-Z0-9_]+=' "$ENV_FILE")+if [ -n "${SWITCH_E2E_PROVIDER_ENV_FILE:-}" ] && [ -r "$SWITCH_E2E_PROVIDER_ENV_FILE" ]; then+ while IFS='=' read -r _ value; do remember_secrets "$value"; done \+ < <(grep -E '^[A-Za-z0-9_]+=' "$SWITCH_E2E_PROVIDER_ENV_FILE")+fi++echo "Starting a fresh stack (Postgres, Mattermost)…""${COMPOSE[@]}" down -v >/dev/null 2>&1 || true"${COMPOSE[@]}" up -d --build >"$OUT/compose-up.log" 2>&1 \@@ -115,6 +148,7 @@ MATTERMOST_TOKEN="$(curl -sf -D - -o /dev/null -H 'Content-Type: application/jso-d "$(jq -n --arg u "$MATTERMOST_USER" --arg p "$MATTERMOST_USER_PASSWORD" '{login_id: $u, password: $p}')" \http://localhost:8065/api/v4/users/login | tr -d '\r' | awk -F': ' 'tolower($1)=="token" {print $2}')"[ -n "$MATTERMOST_TOKEN" ] || { echo "could not log in to Mattermost as $MATTERMOST_USER" >&2; exit 1; }+remember_secrets "$MATTERMOST_TOKEN"PROVIDER_ENV=()if [ "$LANE" = provider ]; then@@ -122,7 +156,7 @@ if [ "$LANE" = provider ]; then(cd "$REPO/console" && pnpm -r --filter './packages/**' run build) >"$OUT/console-build.log" 2>&1 \|| { echo "console package build failed; see $OUT/console-build.log" >&2; exit 1; }- JAR="$OUT/gateway.cookies"+ JAR="$WORK/gateway.cookies"curl -sf -c "$JAR" -o /dev/null -H 'Content-Type: application/json' \-d "$(jq -n --arg e "$GATEWAY_ADMIN_EMAIL" --arg p "$GATEWAY_ADMIN_PASSWORD" '{email: $e, password: $p}')" \http://localhost:8000/gateway/auth/login \@@ -136,9 +170,10 @@ if [ "$LANE" = provider ]; then|| { echo "could not create an agent controller (is agent_management on?)" >&2; exit 1; }rm -f "$JAR"CONTROLLER_ID="$(jq -r .controller_id <<<"$CONTROLLER")"+ remember_secrets "$(jq -r .credential <<<"$CONTROLLER")"echo "Starting the agent controller…"- CONTROLLER_ARGS=(run --data-dir "$OUT/controller" --controller-id "$CONTROLLER_ID"+ CONTROLLER_ARGS=(run --data-dir "$WORK/controller" --controller-id "$CONTROLLER_ID"--server http://localhost:8000 --credential-stdin)if [ -n "${SWITCH_E2E_PROVIDER_ENV_FILE:-}" ]; thenCONTROLLER_ARGS+=(--env-file "$SWITCH_E2E_PROVIDER_ENV_FILE")@@ -163,8 +198,8 @@ if [ "$LANE" = provider ]; thenexit 1fi- mkdir -p "$OUT/agent-dir"- PROVIDER_ENV=(SWITCH_E2E_CONTROLLER_ID="$CONTROLLER_ID" SWITCH_E2E_AGENT_DIR="$OUT/agent-dir")+ mkdir -p "$WORK/agent-dir"+ PROVIDER_ENV=(SWITCH_E2E_CONTROLLER_ID="$CONTROLLER_ID" SWITCH_E2E_AGENT_DIR="$WORK/agent-dir")fiecho "Running the $LANE lane…"diff --git a/console/tooling-e2e/scripts/scan-artifacts.sh b/console/tooling-e2e/scripts/scan-artifacts.shnew file mode 100755index 000000000..8164c6499--- /dev/null+++ b/console/tooling-e2e/scripts/scan-artifacts.sh@@ -0,0 +1,36 @@+#!/usr/bin/env bash+# Fail if any file under a directory contains one of the secrets a run used.+#+# printf '%s\n' "$TOKEN" "$PASSWORD" | scan-artifacts.sh <dir>+#+# Reads the secret values on stdin, one per line, and lists every file that+# holds one, by name only: the point is to keep the values out of an uploaded+# artifact, so the report must not print them either. Exact values are what a+# run's own logs leak, and a pattern scanner cannot know them; run a generic+# scanner (gitleaks) as well for anything else.+#+# Values shorter than 8 characters are ignored: no generated secret is that+# short, and a short one would match ordinary text.+set -euo pipefail++DIR="${1:?usage: scan-artifacts.sh <dir> (secret values on stdin)}"+[ -d "$DIR" ] || { echo "scan-artifacts: $DIR is not a directory" >&2; exit 2; }++PATTERNS="$(mktemp)"+trap 'rm -f "$PATTERNS"' EXIT+chmod 600 "$PATTERNS"+while IFS= read -r value || [ -n "$value" ]; do+ [ "${#value}" -ge 8 ] && printf '%s\n' "$value" >>"$PATTERNS"+done++if [ ! -s "$PATTERNS" ]; then+ echo "scan-artifacts: no secret values were given to scan for" >&2+ exit 2+fi++if LEAKS="$(grep -rlF -f "$PATTERNS" -- "$DIR")"; then+ echo "scan-artifacts: these files contain a secret value from this run; nothing will be uploaded:" >&2+ sed 's/^/ /' <<<"$LEAKS" >&2+ exit 1+fi+echo "scan-artifacts: $(grep -c '' "$PATTERNS") secret values, none found under $DIR."diff --git a/console/tooling-e2e/src/ci-summary.test.ts b/console/tooling-e2e/src/ci-summary.test.tsnew file mode 100644index 000000000..33607ef20--- /dev/null+++ b/console/tooling-e2e/src/ci-summary.test.ts@@ -0,0 +1,46 @@+import { describe, expect, it } from 'vitest';+import { parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';++const REPORT = `<?xml version="1.0" encoding="UTF-8" ?>+<testsuites name="vitest tests" tests="4">+ <testsuite name="src/core/core.test.ts" tests="1">+ <testcase classname="src/core/core.test.ts" name="configuration > unit" time="0.001">+ </testcase>+ </testsuite>+ <testsuite name="src/provider/provider.e2e.test.ts" tests="1">+ <testcase classname="src/provider/provider.e2e.test.ts" name="provider lane > greet" time="0">+ <skipped/>+ </testcase>+ </testsuite>+ <testsuite name="src/fast/fast.e2e.test.ts" tests="3">+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane > addressed-delivery" time="1.25">+ </testcase>+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane > operations" time="0.5">+ <failure message="expected 200 to be 404 | "x" second line" type="AssertionError">stack</failure>+ </testcase>+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane > mattermost-roundtrip" time="0">+ <skipped/>+ </testcase>+ </testsuite>+</testsuites>`;++describe('CI run summary', () => {+ it("reads the lane's end-to-end cases, leaving out unit tests and the other lane", () => {+ const cases = parseScenarioCases(REPORT, 'fast');+ expect(cases.map((each) => [each.name, each.status])).toEqual([+ ['fast lane > addressed-delivery', 'pass'],+ ['fast lane > operations', 'fail'],+ ['fast lane > mattermost-roundtrip', 'skip'],+ ]);+ expect(cases[1]!.failure).toBe('expected 200 to be 404 | "x"');+ });++ it('counts each outcome, and says so when there were no scenarios', () => {+ const cases = parseScenarioCases(REPORT, 'fast');+ expect(summaryLine('fast on ephemeral', cases)).toBe(+ 'fast on ephemeral: 1 passed, 1 failed, 1 skipped'+ );+ expect(summaryMarkdown('fast on ephemeral', cases)).toContain('expected 200 to be 404 \\|');+ expect(summaryMarkdown('fast on dev', [])).toContain('holds no end-to-end scenarios');+ });+});diff --git a/console/tooling-e2e/src/ci-summary.ts b/console/tooling-e2e/src/ci-summary.tsnew file mode 100644index 000000000..a7c235591--- /dev/null+++ b/console/tooling-e2e/src/ci-summary.ts@@ -0,0 +1,121 @@+/**+ * Turn a run's JUnit report into the CI run summary: a Markdown table of the+ * end-to-end scenarios for the job page, and one line for the scheduled-run+ * notification.+ *+ * node --experimental-strip-types src/ci-summary.ts <junit.xml> <lane> <title>+ *+ * Prints the Markdown to stdout. Under GitHub Actions it also writes `line=` to+ * `$GITHUB_OUTPUT`. A missing report is an error: a lane that wrote none did not+ * get as far as its suite, and a summary that read "0 passed" would hide that.+ */+import { appendFileSync, readFileSync } from 'node:fs';+import type { Lane } from './core/env.ts';+import { message } from './core/suite.ts';++export type CaseStatus = 'pass' | 'fail' | 'skip';++export interface ScenarioCase {+ file: string;+ name: string;+ seconds: number;+ status: CaseStatus;+ /** The failure message, first line only; null unless the case failed. */+ failure: string | null;+}++const ENTITIES: Record<string, string> = {+ '<': '<',+ '>': '>',+ '"': '"',+ ''': "'",+ '&': '&',+};++function unescapeXml(text: string): string {+ return text.replace(/&(?:lt|gt|quot|apos|amp|#\d+|#x[0-9a-f]+);/gi, (entity) => {+ if (entity.startsWith('&#x')) return String.fromCodePoint(parseInt(entity.slice(3, -1), 16));+ if (entity.startsWith('&#')) return String.fromCodePoint(Number(entity.slice(2, -1)));+ return ENTITIES[entity] ?? entity;+ });+}++/**+ * One lane's end-to-end cases in a vitest JUnit report. Unit tests, and the+ * other lane's suite (which skips itself), are left out.+ */+export function parseScenarioCases(xml: string, lane: Lane): ScenarioCase[] {+ const cases: ScenarioCase[] = [];+ const pattern =+ /<testcase classname="([^"]*)" name="([^"]*)" time="([^"]*)"\s*(?:\/>|>([\s\S]*?)<\/testcase>)/g;+ for (const match of xml.matchAll(pattern)) {+ const file = unescapeXml(match[1] ?? '');+ if (!file.startsWith(`src/${lane}/`) || !file.includes('.e2e.test.')) continue;+ const body = match[4] ?? '';+ const failed = /<(failure|error)\b/.exec(body);+ const failureMessage = /<(?:failure|error)\b[^>]*\bmessage="([^"]*)"/.exec(body)?.[1];+ cases.push({+ file,+ name: unescapeXml(match[2] ?? ''),+ seconds: Number(match[3] ?? 0),+ status: failed ? 'fail' : /<skipped\b/.test(body) ? 'skip' : 'pass',+ failure: failed+ ? unescapeXml(failureMessage ?? 'failed')+ .split('\n')[0]!+ .slice(0, 200)+ : null,+ });+ }+ return cases;+}++export function summaryLine(title: string, cases: readonly ScenarioCase[]): string {+ const count = (status: CaseStatus) => cases.filter((each) => each.status === status).length;+ return `${title}: ${count('pass')} passed, ${count('fail')} failed, ${count('skip')} skipped`;+}++const MARK: Record<CaseStatus, string> = { pass: '✅', fail: '❌', skip: '⏭️' };++export function summaryMarkdown(title: string, cases: readonly ScenarioCase[]): string {+ const cell = (text: string) => text.replace(/\|/g, '\\|').replace(/\n/g, ' ');+ const rows = cases.map(+ (each) =>+ `| ${MARK[each.status]} | ${cell(each.name)} | ${each.seconds.toFixed(1)}s | ${cell(each.failure ?? '')} |`+ );+ return [+ `### ${summaryLine(title, cases)}`,+ '',+ ...(cases.length === 0+ ? ['The report holds no end-to-end scenarios.']+ : ['| | scenario | time | failure |', '|---|---|---|---|', ...rows]),+ '',+ ].join('\n');+}++function main(argv: readonly string[]): void {+ const [reportPath, lane, title] = argv;+ if (!reportPath || (lane !== 'fast' && lane !== 'provider') || !title) {+ throw new Error('Usage: ci-summary.ts <junit.xml> <fast|provider> <title>');+ }+ let xml: string;+ try {+ xml = readFileSync(reportPath, 'utf8');+ } catch (error) {+ throw new Error(+ `No JUnit report at ${reportPath} (${message(error)}): the lane stopped before its suite ran.`+ );+ }+ const cases = parseScenarioCases(xml, lane);+ process.stdout.write(summaryMarkdown(title, cases));+ const output = process.env.GITHUB_OUTPUT;+ if (output) appendFileSync(output, `line=${summaryLine(title, cases)}\n`);+}++if (import.meta.url === `file://${process.argv[1]}`) {+ try {+ main(process.argv.slice(2));+ } catch (error) {+ console.error(message(error));+ process.exitCode = 1;+ }+}