Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

E2E in CI: reusable lane workflow, PR fast lane, nightly runs

Petr Bauch9360a6decab8
diff --git a/.github/workflows/e2e-scheduled.yml b/.github/workflows/e2e-scheduled.yml
new file mode 100644
index 000000000..d923cdeea
--- /dev/null
+++ b/.github/workflows/e2e-scheduled.yml
@@ -0,0 +1,77 @@
+name: E2E scheduled
+
+# When the end-to-end runs that are not PR checks happen. What a run does is
+# e2e.yml's; this file only says when, and reports.
+#
+# after a merge to main the provider lane on a throwaway stack. Real model
+# calls, so not on every PR, and never a required check
+# nightly the provider lane again, and the fast lane against
+# dev, then one summary notification for the night
+#
+# The fast lane on a throwaway stack is a PR check in pr-ci.yml.
+#
+# The notification goes to the incoming webhook in the repository secret
+# E2E_NOTIFY_WEBHOOK_URL (Slack and Mattermost both accept the payload). Without
+# it the run says so in a warning and sends nothing.
+
+on:
+ schedule:
+ - cron: '41 3 * * *'
+ push:
+ branches: [main]
+ paths-ignore:
+ - 'docs/**'
+ - '**/*.md'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ provider:
+ name: Provider lane
+ uses: ./.github/workflows/e2e.yml
+ with:
+ lane: provider
+ environment: ephemeral
+ agent_type: claude-code
+
+ fast-dev:
+ name: Fast lane on dev
+ if: ${{ github.event_name != 'push' }}
+ uses: ./.github/workflows/e2e.yml
+ with:
+ lane: fast
+ environment: dev
+ agent_type: claude-code
+
+ notify:
+ name: Nightly summary
+ needs: [provider, fast-dev]
+ if: ${{ always() && github.event_name == 'schedule' }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Send the summary
+ env:
+ WEBHOOK: ${{ secrets.E2E_NOTIFY_WEBHOOK_URL }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ PROVIDER_RESULT: ${{ needs.provider.result }}
+ PROVIDER_SUMMARY: ${{ needs.provider.outputs.summary }}
+ DEV_RESULT: ${{ needs.fast-dev.result }}
+ DEV_SUMMARY: ${{ needs.fast-dev.outputs.summary }}
+ run: |
+ line() {
+ local mark=❌
+ [ "$2" = success ] && mark=✅
+ printf '%s %s — %s\n' "$mark" "$1" "${3:-$2, no scenario summary}"
+ }
+ text="$(printf 'Switch E2E nightly\n%s\n%s\n%s' \
+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \
+ "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \
+ "$RUN_URL")"
+ echo "$text"
+ if [ -z "$WEBHOOK" ]; then
+ echo "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; the nightly summary was not sent anywhere."
+ exit 0
+ fi
+ curl -fsS -H 'Content-Type: application/json' -d "$(jq -n --arg text "$text" '{text: $text}')" "$WEBHOOK" >/dev/null
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 000000000..31a6eea3c
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,226 @@
+name: E2E
+
+# The one definition of an end-to-end run: a lane of console/tooling-e2e against
+# an environment. Called by pr-ci.yml (the fast lane on a throwaway stack, on
+# every PR) and by e2e-scheduled.yml (the provider lane after each merge, the
+# deployed environments nightly), and dispatchable by hand.
+#
+# lane fast provider-free scenarios over plain HTTP
+# provider real agent sessions on a headless agent controller
+# environment ephemeral a fresh stack on the runner, built from this checkout
+# dev, staging an existing deployment, configured by the GitHub
+# Environment of that name
+#
+# Fork safety: an ephemeral fast run generates every secret it uses on the
+# runner and reads no repository or environment secret, so it runs the same
+# from a fork PR's read-only token. Everything that needs a secret enters a
+# GitHub Environment, whose protection rules decide which refs may run it:
+#
+# e2e-provider secret ANTHROPIC_API_KEY, for the controller's Claude Code
+# dev, staging vars SWITCH_API_URL, MATTERMOST_URL, MATTERMOST_TEAM; secrets
+# SWITCH_GATEWAY_ADMIN_EMAIL, SWITCH_GATEWAY_ADMIN_PASSWORD,
+# SWITCH_AGENT_REGISTRATION_TOKEN, MATTERMOST_TOKEN
+#
+# Runs are strict (SWITCH_E2E_STRICT=1): a scenario that cannot run fails
+# rather than skips, so a deployment without Mattermost configured fails the
+# Mattermost scenario instead of passing without it.
+#
+# Artifacts are uploaded only after the output has been scanned for the run's
+# own secret values and by gitleaks. A scan that fails or does not run means
+# nothing is uploaded.
+
+on:
+ workflow_call:
+ inputs:
+ lane:
+ type: string
+ required: true
+ environment:
+ type: string
+ required: true
+ agent_type:
+ description: Provider for the provider lane
+ type: string
+ required: true
+ outputs:
+ summary:
+ description: One line counting the lane's scenarios by outcome
+ value: ${{ jobs.run.outputs.summary }}
+ workflow_dispatch:
+ inputs:
+ lane:
+ type: choice
+ options: [fast, provider]
+ default: fast
+ environment:
+ type: choice
+ options: [ephemeral, dev, staging]
+ default: ephemeral
+ agent_type:
+ description: Provider for the provider lane
+ type: choice
+ options: [claude-code]
+ default: claude-code
+
+permissions:
+ contents: read
+
+jobs:
+ run:
+ name: ${{ inputs.lane }} lane on ${{ inputs.environment }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || (inputs.lane == 'provider' && 'e2e-provider' || '') }}
+ # One run at a time against a shared deployment, queued rather than
+ # cancelled: a cancelled run leaves its agents and rooms behind. A throwaway
+ # stack is the runner's own, so ephemeral runs never wait on each other.
+ concurrency:
+ group: ${{ inputs.environment == 'ephemeral' && format('e2e-ephemeral-{0}-{1}', inputs.lane, github.run_id) || format('e2e-{0}', inputs.environment) }}
+ cancel-in-progress: false
+ outputs:
+ summary: ${{ steps.summary.outputs.line }}
+ defaults:
+ run:
+ # Named explicitly for -o pipefail, which the implicit default lacks:
+ # without it a failed suite piped through tee would pass.
+ shell: bash
+ env:
+ OUT: ${{ github.workspace }}/build/e2e/out
+ LANE: ${{ inputs.lane }}
+ TARGET: ${{ inputs.environment }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Check the inputs
+ env:
+ AGENT_TYPE: ${{ inputs.agent_type }}
+ run: |
+ case "$LANE" in fast|provider) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esac
+ case "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esac
+ if [ "$LANE" = provider ] && [ "$TARGET" != ephemeral ]; then
+ echo "::error::The provider lane runs on an ephemeral stack only."; exit 1
+ fi
+ if [ "$LANE" = provider ] && [ "$AGENT_TYPE" != claude-code ]; then
+ echo "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1
+ fi
+
+ - name: Set up uv
+ if: ${{ inputs.environment == 'ephemeral' }}
+ uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
+ with:
+ python-version: '3.12'
+ enable-cache: true
+
+ - name: Set up pnpm
+ uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
+ with:
+ version: 11.5.3
+
+ - name: Set up Node
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version-file: console/.nvmrc
+ cache: pnpm
+ cache-dependency-path: console/pnpm-lock.yaml
+
+ - name: Install dependencies
+ working-directory: console
+ run: pnpm install --frozen-lockfile
+
+ - name: Install Claude Code
+ if: ${{ inputs.lane == 'provider' }}
+ run: npm install -g @anthropic-ai/[email protected]
+
+ # Into the controller's environment through --env-file; never into $OUT.
+ - name: Write the provider key file
+ if: ${{ inputs.lane == 'provider' }}
+ env:
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ run: |
+ if [ -z "$ANTHROPIC_API_KEY" ]; then
+ echo "::error::The e2e-provider environment has no ANTHROPIC_API_KEY secret."; exit 1
+ fi
+ umask 077
+ printf 'ANTHROPIC_API_KEY=%s\n' "$ANTHROPIC_API_KEY" >"$RUNNER_TEMP/provider.env"
+ echo "SWITCH_E2E_PROVIDER_ENV_FILE=$RUNNER_TEMP/provider.env" >>"$GITHUB_ENV"
+
+ - name: Run the lane on a throwaway stack
+ if: ${{ inputs.environment == 'ephemeral' }}
+ env:
+ SWITCH_E2E_LANE: ${{ inputs.lane }}
+ SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}
+ E2E_OUT: ${{ env.OUT }}
+ run: console/tooling-e2e/scripts/ephemeral-stack.sh
+
+ - name: Run the lane against ${{ inputs.environment }}
+ if: ${{ inputs.environment != 'ephemeral' }}
+ working-directory: console/tooling-e2e
+ env:
+ SWITCH_E2E_LANE: ${{ inputs.lane }}
+ SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}
+ SWITCH_E2E_STRICT: '1'
+ SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}
+ SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}
+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
+ MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}
+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
+ MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}
+ run: |
+ mkdir -p "$OUT"
+ pnpm exec vitest run --reporter=verbose --reporter=junit --outputFile.junit="$OUT/junit.xml" \
+ 2>&1 | tee "$OUT/vitest.log"
+
+ # What interrupted runs left behind. The floor keeps it off a run still in
+ # progress, which is why it can run after every suite.
+ - name: Remove stale harness artefacts from ${{ inputs.environment }}
+ if: ${{ always() && inputs.environment != 'ephemeral' }}
+ working-directory: console/tooling-e2e
+ env:
+ SWITCH_E2E_LANE: ${{ inputs.lane }}
+ SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}
+ SWITCH_E2E_STRICT: '1'
+ SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}
+ SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}
+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
+ MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}
+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
+ MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}
+ run: pnpm cleanup --older-than 180
+
+ - name: Summarise the run
+ id: summary
+ if: ${{ always() }}
+ run: |
+ node --experimental-strip-types console/tooling-e2e/src/ci-summary.ts \
+ "$OUT/junit.xml" "$LANE" "$LANE lane on $TARGET" >>"$GITHUB_STEP_SUMMARY"
+
+ - name: Scan the output for secrets
+ id: scan
+ if: ${{ always() }}
+ env:
+ SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
+ SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
+ MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
+ run: |
+ [ -d "$OUT" ] || { echo "No output to scan."; exit 1; }
+ if [ "$TARGET" = ephemeral ]; then
+ # The stack script scans for the secrets it generated and the
+ # provider key file, and leaves this file only when none turned up.
+ [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }
+ else
+ printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN" \
+ | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"
+ fi
+ docker run --rm -v "$OUT:/out:ro" zricethezav/gitleaks:v8.30.1 dir /out --redact --no-banner
+
+ - name: Upload the output
+ if: ${{ always() && steps.scan.outcome == 'success' }}
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: e2e-${{ inputs.lane }}-${{ inputs.environment }}
+ path: ${{ env.OUT }}
+ include-hidden-files: true
+ retention-days: 14
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
index 7021810e9..b514582c1 100644
--- a/.github/workflows/pr-ci.yml
+++ b/.github/workflows/pr-ci.yml
@@ -74,6 +74,7 @@ jobs:
artifacts: ${{ steps.filter.outputs.artifacts }}
migrations: ${{ steps.filter.outputs.migrations }}
chart: ${{ steps.filter.outputs.chart }}
+ e2e: ${{ steps.filter.outputs.e2e }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -105,6 +106,15 @@ jobs:
gateway:
- *shared
- 'gateway/**'
+ # The server, the stack the fast lane stands up, and the harness.
+ e2e:
+ - *shared
+ - 'core/**'
+ - 'console/tooling-e2e/**'
+ - 'console/pnpm-lock.yaml'
+ - 'deploy/local/**'
+ - '.env.example'
+ - '.github/workflows/e2e.yml'
# Its own filter, narrower than `backend`, so the push-triggered
# `migration-heads-on-merge` job below only runs when it can
# actually find a new head — not on every backend-touching push.
@@ -360,6 +370,19 @@ jobs:
- name: Test (vitest)
run: npm test
+ # Real messages through a real server and Mattermost, on a stack the job
+ # builds from this checkout. Needs no secret, so a fork PR runs it the same.
+ # The provider lane and the deployed environments are e2e-scheduled.yml's.
+ e2e-fast:
+ name: E2E fast lane
+ needs: changes
+ if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ uses: ./.github/workflows/e2e.yml
+ with:
+ lane: fast
+ environment: ephemeral
+ agent_type: claude-code
+
migration-heads-on-merge:
name: Migration heads on merge
# Only meaningful right after a new migration lands on main — a PR still
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index 643c1cc82..71b82c4b3 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -65,7 +65,9 @@ Typecheck: `pnpm typecheck`.
| `src/fixtures/` | Throwaway agents and rooms for a scenario, the bridge-readiness probe, teardown |
| `src/fast/` | Fast-lane scenarios, one file per area, listed in `index.ts` |
| `src/provider/` | The provider lane: Console credential files, the shared agent and channel, the scenarios, `seed.ts` |
-| `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast` |
+| `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast` and `just e2e-provider` |
+| `scripts/scan-artifacts.sh` | Fails if a run's output holds one of the secrets the run used |
+| `src/ci-summary.ts` | The CI run summary, read from the JUnit report |
## Scenarios and their headers
@@ -208,6 +210,40 @@ Two things differ from a Console session:
The controller's log goes to `build/e2e/ephemeral/controller.log`.
+## In CI
+
+`.github/workflows/e2e.yml` defines a run: one lane against one environment
+(`ephemeral`, `dev` or `staging`). Other workflows call it, and it can also be
+dispatched by hand.
+
+| When | Lane | Environment | Required check |
+| --- | --- | --- | --- |
+| Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes |
+| After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no |
+| Nightly (`e2e-scheduled.yml`) | provider; fast | ephemeral; dev | no |
+
+- **Secrets.** An ephemeral fast run generates every secret it uses on the
+ runner, so it works the same on a fork PR. Every other run reads its secrets
+ from a GitHub Environment:
+ - `e2e-provider` holds `ANTHROPIC_API_KEY`;
+ - `dev` and `staging` hold the deployment's URLs as variables and its
+ credentials as secrets.
+
+ The header of `e2e.yml` lists every name. Each environment's protection rules
+ decide which branches may use it.
+- **Concurrency.** Runs against a shared deployment queue behind each other.
+ They are never cancelled, because a cancelled run leaves its artefacts
+ behind. After every deployed run, `pnpm cleanup --older-than 180` removes
+ what interrupted runs left.
+- **Artifacts.** The output directory is uploaded only after it passes two
+ scans: one for the exact secret values the run used (`scan-artifacts.sh`),
+ and gitleaks. On an ephemeral stack, the env file, the controller's state and
+ the agent's directory live outside the output directory, so the output holds
+ only logs and the JUnit report.
+- **Reporting.** Each run writes a scenario table to the job summary. A
+ scheduled night sends one summary to the webhook in the repository secret
+ `E2E_NOTIFY_WEBHOOK_URL`.
+
## With a Switch Console session
The provider lane can also run against a **Switch Console session for the agent
diff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.sh
index 6629a587f..2c149d8df 100755
--- a/console/tooling-e2e/scripts/ephemeral-stack.sh
+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh
@@ -19,7 +19,12 @@
# secrets or pick up a stale value from them.
#
# Ports 5432, 8000, 8065 and 8081 must be free: stop a local stack first.
-# Everything the run writes goes to build/e2e/ephemeral/ (or $E2E_OUT).
+# Logs and the JUnit report go to build/e2e/ephemeral/ (or $E2E_OUT); the env
+# file, the controller's state and the agent's working directory go to a
+# temporary directory, so the output holds nothing secret by construction.
+# On exit the output is scanned for every secret the run used, and
+# .secret-scan-passed is written there only if none turned up: CI uploads the
+# output only when that file exists.
set -euo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
@@ -48,17 +53,12 @@ for port in 5432 8000 8065 8081; do
done
rm -rf "$OUT"; mkdir -p "$OUT"
-ENV_FILE="$OUT/stack.env"
-cp "$REPO/.env.example" "$ENV_FILE"
-"$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e
-if [ "$LANE" = provider ]; then
- # Agent management, and the key controllers' access tokens are signed with.
- sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \
- -e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"
- rm -f "$ENV_FILE.bak"
-fi
-chmod 600 "$ENV_FILE"
-set -a; source "$ENV_FILE"; set +a
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/switch-e2e.XXXXXX")"
+chmod 700 "$WORK"
+ENV_FILE="$WORK/stack.env"
+# Every secret the run uses, one per line, for the scan of $OUT on exit.
+SECRETS="$WORK/secrets"
+: >"$SECRETS"
cd "$REPO"
COMPOSE=(docker compose -p switch-e2e-ephemeral -f deploy/local/docker-compose.yml
@@ -71,16 +71,49 @@ cleanup() {
[ -n "$CONTROLLER_PID" ] && kill "$CONTROLLER_PID" 2>/dev/null && wait "$CONTROLLER_PID" 2>/dev/null
[ -n "$SERVER_PID" ] && kill "$SERVER_PID" 2>/dev/null && wait "$SERVER_PID" 2>/dev/null
"${COMPOSE[@]}" down -v >/dev/null 2>&1 || true
- rm -f "$ENV_FILE"
else
echo "Left running: switch-core :8000 (pid $SERVER_PID), Mattermost :8065${CONTROLLER_PID:+, agent controller (pid $CONTROLLER_PID)}."
- echo "Secrets for this stack: $ENV_FILE. Tear down with:"
- echo " kill $SERVER_PID $CONTROLLER_PID; ${COMPOSE[*]} down -v"
+ echo "Secrets and state for this stack: $WORK. Tear down with:"
+ echo " kill $SERVER_PID $CONTROLLER_PID; ${COMPOSE[*]} down -v; rm -rf $WORK"
fi
+ if "$HARNESS/scripts/scan-artifacts.sh" "$OUT" <"$SECRETS"; then
+ : >"$OUT/.secret-scan-passed"
+ else
+ status=1
+ fi
+ [ "$KEEP" = 0 ] && rm -rf "$WORK"
exit "$status"
}
trap cleanup EXIT
+cp "$REPO/.env.example" "$ENV_FILE"
+"$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e
+if [ "$LANE" = provider ]; then
+ # Agent management, and the key controllers' access tokens are signed with.
+ sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \
+ -e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"
+ rm -f "$ENV_FILE.bak"
+fi
+chmod 600 "$ENV_FILE"
+set -a; source "$ENV_FILE"; set +a
+
+# A secret's value, and each part of it, since SECRET_KEYS is `label:key,…`.
+remember_secrets() {
+ local value
+ for value in "$@"; do
+ printf '%s\n' "$value" >>"$SECRETS"
+ tr ',:' '\n\n' <<<"$value" >>"$SECRETS"
+ done
+}
+while IFS='=' read -r key value; do
+ case "$key" in *SECRET*|*PASSWORD*|*TOKEN*|*_KEY|*_KEYS) remember_secrets "$value" ;; esac
+done < <(grep -E '^[A-Z0-9_]+=' "$ENV_FILE")
+if [ -n "${SWITCH_E2E_PROVIDER_ENV_FILE:-}" ] && [ -r "$SWITCH_E2E_PROVIDER_ENV_FILE" ]; then
+ while IFS='=' read -r _ value; do remember_secrets "$value"; done \
+ < <(grep -E '^[A-Za-z0-9_]+=' "$SWITCH_E2E_PROVIDER_ENV_FILE")
+fi
+
+
echo "Starting a fresh stack (Postgres, Mattermost)…"
"${COMPOSE[@]}" down -v >/dev/null 2>&1 || true
"${COMPOSE[@]}" up -d --build >"$OUT/compose-up.log" 2>&1 \
@@ -115,6 +148,7 @@ MATTERMOST_TOKEN="$(curl -sf -D - -o /dev/null -H 'Content-Type: application/jso
-d "$(jq -n --arg u "$MATTERMOST_USER" --arg p "$MATTERMOST_USER_PASSWORD" '{login_id: $u, password: $p}')" \
http://localhost:8065/api/v4/users/login | tr -d '\r' | awk -F': ' 'tolower($1)=="token" {print $2}')"
[ -n "$MATTERMOST_TOKEN" ] || { echo "could not log in to Mattermost as $MATTERMOST_USER" >&2; exit 1; }
+remember_secrets "$MATTERMOST_TOKEN"
PROVIDER_ENV=()
if [ "$LANE" = provider ]; then
@@ -122,7 +156,7 @@ if [ "$LANE" = provider ]; then
(cd "$REPO/console" && pnpm -r --filter './packages/**' run build) >"$OUT/console-build.log" 2>&1 \
|| { echo "console package build failed; see $OUT/console-build.log" >&2; exit 1; }
- JAR="$OUT/gateway.cookies"
+ JAR="$WORK/gateway.cookies"
curl -sf -c "$JAR" -o /dev/null -H 'Content-Type: application/json' \
-d "$(jq -n --arg e "$GATEWAY_ADMIN_EMAIL" --arg p "$GATEWAY_ADMIN_PASSWORD" '{email: $e, password: $p}')" \
http://localhost:8000/gateway/auth/login \
@@ -136,9 +170,10 @@ if [ "$LANE" = provider ]; then
|| { echo "could not create an agent controller (is agent_management on?)" >&2; exit 1; }
rm -f "$JAR"
CONTROLLER_ID="$(jq -r .controller_id <<<"$CONTROLLER")"
+ remember_secrets "$(jq -r .credential <<<"$CONTROLLER")"
echo "Starting the agent controller…"
- CONTROLLER_ARGS=(run --data-dir "$OUT/controller" --controller-id "$CONTROLLER_ID"
+ CONTROLLER_ARGS=(run --data-dir "$WORK/controller" --controller-id "$CONTROLLER_ID"
--server http://localhost:8000 --credential-stdin)
if [ -n "${SWITCH_E2E_PROVIDER_ENV_FILE:-}" ]; then
CONTROLLER_ARGS+=(--env-file "$SWITCH_E2E_PROVIDER_ENV_FILE")
@@ -163,8 +198,8 @@ if [ "$LANE" = provider ]; then
exit 1
fi
- mkdir -p "$OUT/agent-dir"
- PROVIDER_ENV=(SWITCH_E2E_CONTROLLER_ID="$CONTROLLER_ID" SWITCH_E2E_AGENT_DIR="$OUT/agent-dir")
+ mkdir -p "$WORK/agent-dir"
+ PROVIDER_ENV=(SWITCH_E2E_CONTROLLER_ID="$CONTROLLER_ID" SWITCH_E2E_AGENT_DIR="$WORK/agent-dir")
fi
echo "Running the $LANE lane…"
diff --git a/console/tooling-e2e/scripts/scan-artifacts.sh b/console/tooling-e2e/scripts/scan-artifacts.sh
new file mode 100755
index 000000000..8164c6499
--- /dev/null
+++ b/console/tooling-e2e/scripts/scan-artifacts.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Fail if any file under a directory contains one of the secrets a run used.
+#
+# printf '%s\n' "$TOKEN" "$PASSWORD" | scan-artifacts.sh <dir>
+#
+# Reads the secret values on stdin, one per line, and lists every file that
+# holds one, by name only: the point is to keep the values out of an uploaded
+# artifact, so the report must not print them either. Exact values are what a
+# run's own logs leak, and a pattern scanner cannot know them; run a generic
+# scanner (gitleaks) as well for anything else.
+#
+# Values shorter than 8 characters are ignored: no generated secret is that
+# short, and a short one would match ordinary text.
+set -euo pipefail
+
+DIR="${1:?usage: scan-artifacts.sh <dir> (secret values on stdin)}"
+[ -d "$DIR" ] || { echo "scan-artifacts: $DIR is not a directory" >&2; exit 2; }
+
+PATTERNS="$(mktemp)"
+trap 'rm -f "$PATTERNS"' EXIT
+chmod 600 "$PATTERNS"
+while IFS= read -r value || [ -n "$value" ]; do
+ [ "${#value}" -ge 8 ] && printf '%s\n' "$value" >>"$PATTERNS"
+done
+
+if [ ! -s "$PATTERNS" ]; then
+ echo "scan-artifacts: no secret values were given to scan for" >&2
+ exit 2
+fi
+
+if LEAKS="$(grep -rlF -f "$PATTERNS" -- "$DIR")"; then
+ echo "scan-artifacts: these files contain a secret value from this run; nothing will be uploaded:" >&2
+ sed 's/^/ /' <<<"$LEAKS" >&2
+ exit 1
+fi
+echo "scan-artifacts: $(grep -c '' "$PATTERNS") secret values, none found under $DIR."
diff --git a/console/tooling-e2e/src/ci-summary.test.ts b/console/tooling-e2e/src/ci-summary.test.ts
new file mode 100644
index 000000000..33607ef20
--- /dev/null
+++ b/console/tooling-e2e/src/ci-summary.test.ts
@@ -0,0 +1,46 @@
+import { describe, expect, it } from 'vitest';
+import { parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';
+
+const REPORT = `<?xml version="1.0" encoding="UTF-8" ?>
+<testsuites name="vitest tests" tests="4">
+ <testsuite name="src/core/core.test.ts" tests="1">
+ <testcase classname="src/core/core.test.ts" name="configuration &gt; unit" time="0.001">
+ </testcase>
+ </testsuite>
+ <testsuite name="src/provider/provider.e2e.test.ts" tests="1">
+ <testcase classname="src/provider/provider.e2e.test.ts" name="provider lane &gt; greet" time="0">
+ <skipped/>
+ </testcase>
+ </testsuite>
+ <testsuite name="src/fast/fast.e2e.test.ts" tests="3">
+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane &gt; addressed-delivery" time="1.25">
+ </testcase>
+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane &gt; operations" time="0.5">
+ <failure message="expected 200 to be 404 | &quot;x&quot;&#10;second line" type="AssertionError">stack</failure>
+ </testcase>
+ <testcase classname="src/fast/fast.e2e.test.ts" name="fast lane &gt; mattermost-roundtrip" time="0">
+ <skipped/>
+ </testcase>
+ </testsuite>
+</testsuites>`;
+
+describe('CI run summary', () => {
+ it("reads the lane's end-to-end cases, leaving out unit tests and the other lane", () => {
+ const cases = parseScenarioCases(REPORT, 'fast');
+ expect(cases.map((each) => [each.name, each.status])).toEqual([
+ ['fast lane > addressed-delivery', 'pass'],
+ ['fast lane > operations', 'fail'],
+ ['fast lane > mattermost-roundtrip', 'skip'],
+ ]);
+ expect(cases[1]!.failure).toBe('expected 200 to be 404 | "x"');
+ });
+
+ it('counts each outcome, and says so when there were no scenarios', () => {
+ const cases = parseScenarioCases(REPORT, 'fast');
+ expect(summaryLine('fast on ephemeral', cases)).toBe(
+ 'fast on ephemeral: 1 passed, 1 failed, 1 skipped'
+ );
+ expect(summaryMarkdown('fast on ephemeral', cases)).toContain('expected 200 to be 404 \\|');
+ expect(summaryMarkdown('fast on dev', [])).toContain('holds no end-to-end scenarios');
+ });
+});
diff --git a/console/tooling-e2e/src/ci-summary.ts b/console/tooling-e2e/src/ci-summary.ts
new file mode 100644
index 000000000..a7c235591
--- /dev/null
+++ b/console/tooling-e2e/src/ci-summary.ts
@@ -0,0 +1,121 @@
+/**
+ * Turn a run's JUnit report into the CI run summary: a Markdown table of the
+ * end-to-end scenarios for the job page, and one line for the scheduled-run
+ * notification.
+ *
+ * node --experimental-strip-types src/ci-summary.ts <junit.xml> <lane> <title>
+ *
+ * Prints the Markdown to stdout. Under GitHub Actions it also writes `line=` to
+ * `$GITHUB_OUTPUT`. A missing report is an error: a lane that wrote none did not
+ * get as far as its suite, and a summary that read "0 passed" would hide that.
+ */
+import { appendFileSync, readFileSync } from 'node:fs';
+import type { Lane } from './core/env.ts';
+import { message } from './core/suite.ts';
+
+export type CaseStatus = 'pass' | 'fail' | 'skip';
+
+export interface ScenarioCase {
+ file: string;
+ name: string;
+ seconds: number;
+ status: CaseStatus;
+ /** The failure message, first line only; null unless the case failed. */
+ failure: string | null;
+}
+
+const ENTITIES: Record<string, string> = {
+ '&lt;': '<',
+ '&gt;': '>',
+ '&quot;': '"',
+ '&apos;': "'",
+ '&amp;': '&',
+};
+
+function unescapeXml(text: string): string {
+ return text.replace(/&(?:lt|gt|quot|apos|amp|#\d+|#x[0-9a-f]+);/gi, (entity) => {
+ if (entity.startsWith('&#x')) return String.fromCodePoint(parseInt(entity.slice(3, -1), 16));
+ if (entity.startsWith('&#')) return String.fromCodePoint(Number(entity.slice(2, -1)));
+ return ENTITIES[entity] ?? entity;
+ });
+}
+
+/**
+ * One lane's end-to-end cases in a vitest JUnit report. Unit tests, and the
+ * other lane's suite (which skips itself), are left out.
+ */
+export function parseScenarioCases(xml: string, lane: Lane): ScenarioCase[] {
+ const cases: ScenarioCase[] = [];
+ const pattern =
+ /<testcase classname="([^"]*)" name="([^"]*)" time="([^"]*)"\s*(?:\/>|>([\s\S]*?)<\/testcase>)/g;
+ for (const match of xml.matchAll(pattern)) {
+ const file = unescapeXml(match[1] ?? '');
+ if (!file.startsWith(`src/${lane}/`) || !file.includes('.e2e.test.')) continue;
+ const body = match[4] ?? '';
+ const failed = /<(failure|error)\b/.exec(body);
+ const failureMessage = /<(?:failure|error)\b[^>]*\bmessage="([^"]*)"/.exec(body)?.[1];
+ cases.push({
+ file,
+ name: unescapeXml(match[2] ?? ''),
+ seconds: Number(match[3] ?? 0),
+ status: failed ? 'fail' : /<skipped\b/.test(body) ? 'skip' : 'pass',
+ failure: failed
+ ? unescapeXml(failureMessage ?? 'failed')
+ .split('\n')[0]!
+ .slice(0, 200)
+ : null,
+ });
+ }
+ return cases;
+}
+
+export function summaryLine(title: string, cases: readonly ScenarioCase[]): string {
+ const count = (status: CaseStatus) => cases.filter((each) => each.status === status).length;
+ return `${title}: ${count('pass')} passed, ${count('fail')} failed, ${count('skip')} skipped`;
+}
+
+const MARK: Record<CaseStatus, string> = { pass: '✅', fail: '❌', skip: '⏭️' };
+
+export function summaryMarkdown(title: string, cases: readonly ScenarioCase[]): string {
+ const cell = (text: string) => text.replace(/\|/g, '\\|').replace(/\n/g, ' ');
+ const rows = cases.map(
+ (each) =>
+ `| ${MARK[each.status]} | ${cell(each.name)} | ${each.seconds.toFixed(1)}s | ${cell(each.failure ?? '')} |`
+ );
+ return [
+ `### ${summaryLine(title, cases)}`,
+ '',
+ ...(cases.length === 0
+ ? ['The report holds no end-to-end scenarios.']
+ : ['| | scenario | time | failure |', '|---|---|---|---|', ...rows]),
+ '',
+ ].join('\n');
+}
+
+function main(argv: readonly string[]): void {
+ const [reportPath, lane, title] = argv;
+ if (!reportPath || (lane !== 'fast' && lane !== 'provider') || !title) {
+ throw new Error('Usage: ci-summary.ts <junit.xml> <fast|provider> <title>');
+ }
+ let xml: string;
+ try {
+ xml = readFileSync(reportPath, 'utf8');
+ } catch (error) {
+ throw new Error(
+ `No JUnit report at ${reportPath} (${message(error)}): the lane stopped before its suite ran.`
+ );
+ }
+ const cases = parseScenarioCases(xml, lane);
+ process.stdout.write(summaryMarkdown(title, cases));
+ const output = process.env.GITHUB_OUTPUT;
+ if (output) appendFileSync(output, `line=${summaryLine(title, cases)}\n`);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+ try {
+ main(process.argv.slice(2));
+ } catch (error) {
+ console.error(message(error));
+ process.exitCode = 1;
+ }
+}