Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
E2E: leave runs against the deployed environments to the deployment config
Petr Bauch1c00e767e340
diff --git a/.github/workflows/e2e-scheduled.yml b/.github/workflows/e2e-scheduled.ymlindex f2c18e345..55a1dcf92 100644--- a/.github/workflows/e2e-scheduled.yml+++ b/.github/workflows/e2e-scheduled.yml@@ -5,14 +5,15 @@ name: E2E scheduled## after a merge to main the provider lane on a throwaway stack. Real model# calls, so not on every PR, and never a required check-# nightly the provider lane again, the fast and browser lanes-# against dev, then one summary notification for the-# night+# nightly the provider lane again, then a notification## A failed run after a merge is reported too, since the lane is not a required# check and nothing else would surface it before the night; a passing one is# not, to keep the channel to one message a night.#+# Runs against the deployed environments live with the deployment+# configuration, not here.+## The fast lane on a throwaway stack is a PR check in pr-ci.yml.## Notifications go to the incoming webhook in the repository secret@@ -43,30 +44,12 @@ jobs:uses: ./.github/workflows/e2e.ymlwith:lane: provider- environment: ephemeral- agent_type: claude-code-- fast-dev:- name: Fast lane on dev- if: ${{ github.event_name != 'push' }}- uses: ./.github/workflows/e2e.yml- with:- lane: fast- environment: devagent_type: claude-code- browser-dev:- name: Browser lane on dev- if: ${{ github.event_name != 'push' }}- uses: ./.github/workflows/e2e.yml- with:- lane: browser- environment: dev- agent_type: claude-codenotify:name: Report- needs: [provider, fast-dev, browser-dev]+ needs: [provider]if: ${{ always() && github.event_name != 'workflow_dispatch' }}runs-on: ubuntu-lateststeps:@@ -76,10 +59,6 @@ jobs:RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}PROVIDER_RESULT: ${{ needs.provider.result }}PROVIDER_SUMMARY: ${{ needs.provider.outputs.summary }}- DEV_RESULT: ${{ needs.fast-dev.result }}- DEV_SUMMARY: ${{ needs.fast-dev.outputs.summary }}- BROWSER_RESULT: ${{ needs.browser-dev.result }}- BROWSER_SUMMARY: ${{ needs.browser-dev.outputs.summary }}run: |if [ "$GITHUB_EVENT_NAME" = push ] && [ "$PROVIDER_RESULT" = success ]; thenecho "The provider lane passed after the merge; nothing to report."@@ -91,16 +70,13 @@ jobs:printf '%s %s — %s\n' "$mark" "$1" "${3:-$2, no scenario summary}"}if [ "$GITHUB_EVENT_NAME" = push ]; then- text="$(printf 'Switch E2E after merging %s\n%s\n%s' "${GITHUB_SHA:0:9}" \- "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \- "$RUN_URL")"+ title="Switch E2E after merging ${GITHUB_SHA:0:9}"else- text="$(printf 'Switch E2E nightly\n%s\n%s\n%s\n%s' \- "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \- "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \- "$(line 'Browser lane (dev)' "$BROWSER_RESULT" "$BROWSER_SUMMARY")" \- "$RUN_URL")"+ title='Switch E2E nightly'fi+ text="$(printf '%s\n%s\n%s' "$title" \+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \+ "$RUN_URL")"echo "$text"if [ -z "$WEBHOOK" ]; thenecho "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; this report was not sent anywhere."diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex 0d25b0d74..6326f05ff 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -1,19 +1,20 @@name: E2E-# The one definition of an end-to-end run: a lane of console/tooling-e2e against-# an environment. Called by pr-ci.yml (the fast lane on a throwaway stack, on-# every PR) and by e2e-scheduled.yml (the provider lane after each merge, the-# deployed environments nightly), and dispatchable by hand.+# The one definition of an end-to-end run: a lane of console/tooling-e2e on a+# throwaway stack built from this checkout. Called by pr-ci.yml (the fast lane,+# on every PR) and by e2e-scheduled.yml (the provider lane after each merge and+# nightly), and dispatchable by hand.## lane fast provider-free scenarios over plain HTTP# provider real agent sessions on a headless agent controller# agentic the provider lane's setup, driven by scenario files: a# model plays the human, another judges the result-# browser the gateway UI in headless Chromium; on ephemeral the-# UI is built from this checkout-# environment ephemeral a fresh stack on the runner, built from this checkout-# dev, staging an existing deployment, configured by the GitHub-# Environment of that name+# browser the gateway UI in headless Chromium, built from this+# checkout+#+# Runs against the deployed environments are not here: they live with the+# deployment configuration, which checks out this harness at the commit the+# environment runs and drives it with SWITCH_E2E_ENVIRONMENT set.## Fork safety: an ephemeral fast run generates every secret it uses on the# runner and reads no repository or environment secret, so it runs the same@@ -21,14 +22,9 @@ name: E2E# GitHub Environment, whose protection rules decide which refs may run it:## e2e-provider secret ANTHROPIC_API_KEY, for the controller's Claude Code-# dev, staging vars SWITCH_API_URL, MATTERMOST_URL, MATTERMOST_TEAM, and-# SWITCH_GATEWAY_UI_URL for the browser lane; secrets-# SWITCH_GATEWAY_ADMIN_EMAIL, SWITCH_GATEWAY_ADMIN_PASSWORD,-# SWITCH_AGENT_REGISTRATION_TOKEN, MATTERMOST_TOKEN## Runs are strict (SWITCH_E2E_STRICT=1): a scenario that cannot run fails-# rather than skips, so a deployment without Mattermost configured fails the-# Mattermost scenario instead of passing without it.+# rather than skips.## Artifacts are uploaded only after the output has been scanned for the run's# own secret values and by gitleaks. A scan that fails or does not run means@@ -40,9 +36,6 @@ on:lane:type: stringrequired: true- environment:- type: string- required: trueagent_type:description: Provider for the provider lanetype: string@@ -57,10 +50,6 @@ on:type: choiceoptions: [fast, provider, browser, agentic]default: fast- environment:- type: choice- options: [ephemeral, dev, staging]- default: ephemeralagent_type:description: Provider for the provider lanetype: choice@@ -72,16 +61,10 @@ permissions:jobs:run:- name: ${{ inputs.lane }} lane on ${{ inputs.environment }}+ name: ${{ inputs.lane }} laneruns-on: ubuntu-latesttimeout-minutes: 45- environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || ((inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '') }}- # One run at a time against a shared deployment, queued rather than- # cancelled: a cancelled run leaves its agents and rooms behind. A throwaway- # stack is the runner's own, so ephemeral runs never wait on each other.- concurrency:- group: ${{ inputs.environment == 'ephemeral' && format('e2e-ephemeral-{0}-{1}', inputs.lane, github.run_id) || format('e2e-{0}', inputs.environment) }}- cancel-in-progress: false+ environment: ${{ (inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '' }}outputs:summary: ${{ steps.summary.outputs.line }}defaults:@@ -92,7 +75,6 @@ jobs:env:OUT: ${{ github.workspace }}/build/e2e/outLANE: ${{ inputs.lane }}- TARGET: ${{ inputs.environment }}steps:- name: Checkoutuses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4@@ -102,17 +84,12 @@ jobs:AGENT_TYPE: ${{ inputs.agent_type }}run: |case "$LANE" in fast|provider|browser|agentic) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esac- case "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esaccase "$LANE" in provider|agentic) REAL_SESSION=1 ;; *) REAL_SESSION=0 ;; esac- if [ "$REAL_SESSION" = 1 ] && [ "$TARGET" != ephemeral ]; then- echo "::error::The $LANE lane runs on an ephemeral stack only."; exit 1- fiif [ "$REAL_SESSION" = 1 ] && [ "$AGENT_TYPE" != claude-code ]; thenecho "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1fi- name: Set up uv- if: ${{ inputs.environment == 'ephemeral' }}uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5with:python-version: '3.12'@@ -157,7 +134,6 @@ jobs:echo "SWITCH_E2E_PROVIDER_ENV_FILE=$RUNNER_TEMP/provider.env" >>"$GITHUB_ENV"- name: Run the lane on a throwaway stack- if: ${{ inputs.environment == 'ephemeral' }}env:SWITCH_E2E_LANE: ${{ inputs.lane }}SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}@@ -170,85 +146,29 @@ jobs:SWITCH_E2E_AGENTIC_BUDGET_USD: '2'run: console/tooling-e2e/scripts/ephemeral-stack.sh- - name: Run the lane against ${{ inputs.environment }}- if: ${{ inputs.environment != 'ephemeral' }}- working-directory: console/tooling-e2e- env:- SWITCH_E2E_LANE: ${{ inputs.lane }}- SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}- SWITCH_E2E_STRICT: '1'- SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}- SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}- MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}- MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}- SWITCH_GATEWAY_UI_URL: ${{ vars.SWITCH_GATEWAY_UI_URL }}- SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secrets- SWITCH_E2E_OUTPUT_DIR: ${{ env.OUT }}- run: |- mkdir -p "$OUT"- pnpm exec vitest run --reporter=verbose --reporter=junit --outputFile.junit="$OUT/junit.xml" \- 2>&1 | tee "$OUT/vitest.log"-- # What interrupted runs left behind, whichever lane made it; cleanup- # resolves as the fast lane, which needs nothing a lane adds. The floor- # keeps it off a run still in progress, which is why it can run after- # every suite.- - name: Remove stale harness artefacts from ${{ inputs.environment }}- if: ${{ always() && inputs.environment != 'ephemeral' }}- working-directory: console/tooling-e2e- env:- SWITCH_E2E_LANE: fast- SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}- SWITCH_E2E_STRICT: '1'- SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}- SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}- MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}- MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}- SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secrets- run: pnpm cleanup --older-than 180-- name: Summarise the runid: summaryif: ${{ always() }}run: |node --experimental-strip-types console/tooling-e2e/src/ci-summary.ts \- "$OUT/junit.xml" "$LANE" "$LANE lane on $TARGET" >>"$GITHUB_STEP_SUMMARY"+ "$OUT/junit.xml" "$LANE" "$LANE lane" >>"$GITHUB_STEP_SUMMARY"- name: Scan the output for secretsid: scanif: ${{ always() }}- env:- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}run: |[ -d "$OUT" ] || { echo "No output to scan."; exit 1; }- if [ "$TARGET" = ephemeral ]; then- # The stack script scans for the secrets it generated, the provider- # key file and every credential the server issued the harness, and- # leaves this file only when none turned up.- [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }- else- # The environment's own secrets, and every credential the server- # issued the harness during the run (agent keys, gateway sessions),- # which stay valid on a shared deployment after the run ends.- { printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN"- cat "$RUNNER_TEMP/issued-secrets" 2>/dev/null || true- } | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"- fi+ # The stack script scans for the secrets it generated, the provider+ # key file and every credential the server issued the harness, and+ # leaves this file only when none turned up.+ [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }docker run --rm -v "$OUT:/out:ro" zricethezav/gitleaks:v8.30.1 dir /out --redact --no-banner- name: Upload the outputif: ${{ always() && steps.scan.outcome == 'success' }}uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2with:- name: e2e-${{ inputs.lane }}-${{ inputs.environment }}+ name: e2e-${{ inputs.lane }}path: ${{ env.OUT }}include-hidden-files: trueretention-days: 14diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex cf38cf5ab..a004cc339 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -383,7 +383,7 @@ jobs:# Real messages through a real server and Mattermost, on a stack the job# builds from this checkout. Needs no secret, so a fork PR runs it the same.- # The provider lane and the deployed environments are e2e-scheduled.yml's.+ # The provider lane is e2e-scheduled.yml's.e2e-fast:name: E2E fast laneneeds: changes@@ -391,7 +391,6 @@ jobs:uses: ./.github/workflows/e2e.ymlwith:lane: fast- environment: ephemeralagent_type: claude-codemigration-heads-on-merge:diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex 563159ceb..0e6c34751 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -115,7 +115,9 @@ in `environments`, and its `requires` are configured. Otherwise it shows as**skipped, with the reason**, never as passed. Under `SWITCH_E2E_STRICT=1` (how`just e2e-fast` and CI run) a missing prerequisite, missing configuration or anunreachable server **fails** the run instead, so a job that ran nothing cannot-go green.+go green. The one exception is an environment that deploys no Mattermost on+purpose and says so with `SWITCH_E2E_NO_MATTERMOST=1`: its Mattermost+scenarios skip even under strict mode, and the reason names the setting.A scenario that changes server-global state (a feature flag, say) lists only`ephemeral`. One that writes anything never lists `prod`.@@ -193,11 +195,12 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.| `MATTERMOST_TOKEN` | for Mattermost scenarios | Token of the **human** the harness posts as (a personal access token, or a login session token). || `MATTERMOST_TEAM` | for Mattermost scenarios | Team **slug** (e.g. `switch`), not its display name. || `SWITCH_E2E_STRICT` | no | `1` turns every "cannot run" into a failure. CI sets it. |+| `SWITCH_E2E_NO_MATTERMOST` | no | `1` declares that the environment deploys no Mattermost, so its Mattermost scenarios skip even under strict mode. Refused alongside `MATTERMOST_*`. || `SWITCH_E2E_KEEP` | no | `1` leaves agents, rooms and channels behind for inspection. || `SWITCH_E2E_SERVER_LOG` | no | The server's log, written with `LOG_FORMAT=json`. When set, a scenario fails if the server logged an error while it ran. The stack script sets it. |-| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. |+| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script sets it. || `SWITCH_GATEWAY_UI_URL` | browser lane | Where the gateway UI is served. Its `/gateway` requests must reach `SWITCH_API_URL`. `just e2e-browser` sets it. |-| `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script and `e2e.yml` set it. |+| `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script sets it. || `SWITCH_E2E_DRIVER_MODEL` | agentic lane | Exact model id that plays the human, e.g. `claude-sonnet-5`. || `SWITCH_E2E_JUDGE_MODEL` | agentic lane | Exact model id that grades the conversation. || `SWITCH_E2E_AGENTIC_BUDGET_USD` | agentic lane | What one run may spend, driver, judge and agent together. The run stops once it is reached. |@@ -405,30 +408,21 @@ on every PR with no server.## In CI-`.github/workflows/e2e.yml` defines a run: one lane against one environment-(`ephemeral`, `dev` or `staging`). Other workflows call it, and it can also be-dispatched by hand.+`.github/workflows/e2e.yml` defines a run: one lane on a throwaway stack built+from the checkout. Other workflows call it, and it can also be dispatched by+hand.| When | Lane | Environment | Required check || --- | --- | --- | --- || Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes || After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no |-| Nightly (`e2e-scheduled.yml`) | provider; fast; browser | ephemeral; dev; dev | no |+| Nightly (`e2e-scheduled.yml`) | provider | ephemeral | no || By hand only, for now (`e2e.yml`, dispatch) | agentic | ephemeral | no |- **Secrets.** An ephemeral fast run generates every secret it uses on therunner, so it works the same on a fork PR. Every other run reads its secrets- from a GitHub Environment:- - `e2e-provider` holds `ANTHROPIC_API_KEY`;- - `dev` and `staging` hold the deployment's URLs as variables (the browser- lane also needs `SWITCH_GATEWAY_UI_URL`) and its credentials as secrets.-- The header of `e2e.yml` lists every name. Each environment's protection rules- decide which branches may use it.-- **Concurrency.** Runs against a shared deployment queue behind each other.- They are never cancelled, because a cancelled run leaves its artefacts- behind. After every deployed run, `pnpm cleanup --older-than 180` removes- what interrupted runs left.+ from the GitHub Environment `e2e-provider`, which holds `ANTHROPIC_API_KEY`+ and whose protection rules decide which branches may use it.- **Artifacts.** The output directory is uploaded only after it passes twoscans. The first (`scan-artifacts.sh`) looks for the exact secret values therun used: the ones it was configured with, plus every credential the server@@ -442,6 +436,28 @@ dispatched by hand.- one summary each night;- a report when the provider lane fails after a merge.+## Against a deployed environment++Runs against the shared deployments are driven from the deployment+configuration, not from this repository's CI: it knows which commit each+environment runs, checks this harness out at that commit, and holds the+environment's addresses and credentials. To run a lane there:++```bash+SWITCH_E2E_LANE=fast SWITCH_E2E_ENVIRONMENT=dev SWITCH_E2E_STRICT=1 \+SWITCH_API_URL=… SWITCH_GATEWAY_ADMIN_EMAIL=… SWITCH_GATEWAY_ADMIN_PASSWORD=… \+SWITCH_AGENT_REGISTRATION_TOKEN=… SWITCH_E2E_SECRETS_FILE=… pnpm test+```++- Give either the three `MATTERMOST_*` variables or `SWITCH_E2E_NO_MATTERMOST=1`.+- The browser lane also needs `SWITCH_GATEWAY_UI_URL`.+- Run one lane at a time per environment and never cancel a run: a cancelled+ run leaves its agents and rooms behind. After each run,+ `pnpm cleanup --older-than 180` removes what interrupted runs left.+- Scan the output for the run's secrets before keeping it: the configured+ ones, and every credential listed in `SWITCH_E2E_SECRETS_FILE`+ (`scripts/scan-artifacts.sh` takes them on stdin).+## With a Switch Console sessionThe provider lane can also run against a **Switch Console session for the agentdiff --git a/console/tooling-e2e/src/core/core.test.ts b/console/tooling-e2e/src/core/core.test.tsindex 76447934a..9a0e56173 100644--- a/console/tooling-e2e/src/core/core.test.ts+++ b/console/tooling-e2e/src/core/core.test.ts@@ -44,6 +44,23 @@ describe('configuration', () => {);});+ it('refuses a Mattermost configuration beside a declaration that there is none', () => {+ expect(loadEnv('fast', { ...BASE, SWITCH_E2E_NO_MATTERMOST: '1' }, {}).noMattermost).toBe(true);+ expect(() =>+ loadEnv(+ 'fast',+ {+ ...BASE,+ SWITCH_E2E_NO_MATTERMOST: '1',+ MATTERMOST_URL: 'http://mm.invalid',+ MATTERMOST_TOKEN: 'mm-token-placeholder',+ MATTERMOST_TEAM: 'team',+ },+ {}+ )+ ).toThrow(/SWITCH_E2E_NO_MATTERMOST/);+ });+it('refuses an unknown environment or lane', () => {expect(() => loadEnv('fast', { ...BASE, SWITCH_E2E_ENVIRONMENT: 'qa' }, {})).toThrow(/qa/);expect(() => laneFrom({ SWITCH_E2E_LANE: 'slow' }, {})).toThrow(/slow/);@@ -169,6 +186,19 @@ describe('admission', () => {reason: expect.stringContaining('MATTERMOST_URL'),});});++ it('skips without failing, even when strict, on a Mattermost the environment does not deploy', () => {+ const needsMattermost = header({ requires: ['mattermost'] });+ expect(admit(needsMattermost, env({ strict: true, noMattermost: true }))).toMatchObject({+ run: false,+ failure: false,+ reason: expect.stringContaining('does not deploy'),+ });+ const needsBoth = header({ requires: ['mattermost', 'gateway-ui'] });+ expect(+ admit(needsBoth, env({ strict: true, noMattermost: true, gatewayUiUrl: null }))+ ).toMatchObject({ run: false, failure: true });+ });});describe('cleanup age floor', () => {diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.tsindex 826064f35..158e99390 100644--- a/console/tooling-e2e/src/core/env.ts+++ b/console/tooling-e2e/src/core/env.ts@@ -140,6 +140,11 @@ export interface HarnessEnv {switch: SwitchEnv;/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */mattermost: MattermostEnv | null;+ /**+ * The environment deploys no Mattermost, by design (`SWITCH_E2E_NO_MATTERMOST=1`).+ * Scenarios that need one skip even in strict mode, and say why.+ */+ noMattermost: boolean;provider: ProviderEnv;agentic: AgenticEnv;}@@ -267,6 +272,12 @@ export function loadEnv(.filter(([, value]) => value === undefined).map(([key]) => key);if (missing.length > 0) throw new MissingConfigError(missing);+ const noMattermost = get('SWITCH_E2E_NO_MATTERMOST') === '1';+ if (noMattermost && mattermostGiven) {+ throw new Error(+ 'SWITCH_E2E_NO_MATTERMOST=1 says this environment has no Mattermost, but MATTERMOST_* is set too. Unset one.'+ );+ }return {lane,@@ -290,6 +301,7 @@ export function loadEnv(team: mattermost.MATTERMOST_TEAM as string,}: null,+ noMattermost,provider: {agentRepoDir: get('SWITCH_E2E_AGENT_DIR') ?? null,manifestPath: get('SWITCH_E2E_MANIFEST') ?? null,diff --git a/console/tooling-e2e/src/core/scenario.ts b/console/tooling-e2e/src/core/scenario.tsindex 1ee572a3d..e33ee7fa0 100644--- a/console/tooling-e2e/src/core/scenario.ts+++ b/console/tooling-e2e/src/core/scenario.ts@@ -88,6 +88,8 @@ export function assertValidHeaders(headers: readonly ScenarioHeader[]): void {* is simply not this run's business. A missing prerequisite is different: the* scenario belongs here and cannot run, which skips on a developer machine and* fails in strict mode — CI must not report a scenario that never ran as green.+ * The exception is a prerequisite the environment declares it lacks by design+ * (`SWITCH_E2E_NO_MATTERMOST`): that skips everywhere, with the reason given.*/export type Admission = { run: true } | { run: false; reason: string; failure: boolean };@@ -104,15 +106,22 @@ export function admit(header: ScenarioHeader, env: HarnessEnv): Admission {}const missing = header.requires.filter((prerequisite) => !satisfied(prerequisite, env));if (missing.length > 0) {+ const byDesign = missing.every((prerequisite) => absentByDesign(prerequisite, env));return {run: false,- reason: `needs ${missing.map(describePrerequisite).join(' and ')}`,- failure: env.strict,+ reason:+ `needs ${missing.map(describePrerequisite).join(' and ')}` ++ (byDesign ? `, which '${env.environment}' does not deploy (SWITCH_E2E_NO_MATTERMOST)` : ''),+ failure: env.strict && !byDesign,};}return { run: true };}+function absentByDesign(prerequisite: Prerequisite, env: HarnessEnv): boolean {+ return prerequisite === 'mattermost' && env.noMattermost;+}+function satisfied(prerequisite: Prerequisite, env: HarnessEnv): boolean {switch (prerequisite) {case 'mattermost':