Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

E2E: leave runs against the deployed environments to the deployment config

Petr Bauch1c00e767e340
diff --git a/.github/workflows/e2e-scheduled.yml b/.github/workflows/e2e-scheduled.yml
index f2c18e345..55a1dcf92 100644
--- a/.github/workflows/e2e-scheduled.yml
+++ b/.github/workflows/e2e-scheduled.yml
@@ -5,14 +5,15 @@ name: E2E scheduled
#
# after a merge to main the provider lane on a throwaway stack. Real model
# calls, so not on every PR, and never a required check
-# nightly the provider lane again, the fast and browser lanes
-# against dev, then one summary notification for the
-# night
+# nightly the provider lane again, then a notification
#
# A failed run after a merge is reported too, since the lane is not a required
# check and nothing else would surface it before the night; a passing one is
# not, to keep the channel to one message a night.
#
+# Runs against the deployed environments live with the deployment
+# configuration, not here.
+#
# The fast lane on a throwaway stack is a PR check in pr-ci.yml.
#
# Notifications go to the incoming webhook in the repository secret
@@ -43,30 +44,12 @@ jobs:
uses: ./.github/workflows/e2e.yml
with:
lane: provider
- environment: ephemeral
- agent_type: claude-code
-
- fast-dev:
- name: Fast lane on dev
- if: ${{ github.event_name != 'push' }}
- uses: ./.github/workflows/e2e.yml
- with:
- lane: fast
- environment: dev
agent_type: claude-code
- browser-dev:
- name: Browser lane on dev
- if: ${{ github.event_name != 'push' }}
- uses: ./.github/workflows/e2e.yml
- with:
- lane: browser
- environment: dev
- agent_type: claude-code
notify:
name: Report
- needs: [provider, fast-dev, browser-dev]
+ needs: [provider]
if: ${{ always() && github.event_name != 'workflow_dispatch' }}
runs-on: ubuntu-latest
steps:
@@ -76,10 +59,6 @@ jobs:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
PROVIDER_RESULT: ${{ needs.provider.result }}
PROVIDER_SUMMARY: ${{ needs.provider.outputs.summary }}
- DEV_RESULT: ${{ needs.fast-dev.result }}
- DEV_SUMMARY: ${{ needs.fast-dev.outputs.summary }}
- BROWSER_RESULT: ${{ needs.browser-dev.result }}
- BROWSER_SUMMARY: ${{ needs.browser-dev.outputs.summary }}
run: |
if [ "$GITHUB_EVENT_NAME" = push ] && [ "$PROVIDER_RESULT" = success ]; then
echo "The provider lane passed after the merge; nothing to report."
@@ -91,16 +70,13 @@ jobs:
printf '%s %s — %s\n' "$mark" "$1" "${3:-$2, no scenario summary}"
}
if [ "$GITHUB_EVENT_NAME" = push ]; then
- text="$(printf 'Switch E2E after merging %s\n%s\n%s' "${GITHUB_SHA:0:9}" \
- "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \
- "$RUN_URL")"
+ title="Switch E2E after merging ${GITHUB_SHA:0:9}"
else
- text="$(printf 'Switch E2E nightly\n%s\n%s\n%s\n%s' \
- "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \
- "$(line 'Fast lane (dev)' "$DEV_RESULT" "$DEV_SUMMARY")" \
- "$(line 'Browser lane (dev)' "$BROWSER_RESULT" "$BROWSER_SUMMARY")" \
- "$RUN_URL")"
+ title='Switch E2E nightly'
fi
+ text="$(printf '%s\n%s\n%s' "$title" \
+ "$(line 'Provider lane (ephemeral)' "$PROVIDER_RESULT" "$PROVIDER_SUMMARY")" \
+ "$RUN_URL")"
echo "$text"
if [ -z "$WEBHOOK" ]; then
echo "::warning::E2E_NOTIFY_WEBHOOK_URL is not set; this report was not sent anywhere."
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 0d25b0d74..6326f05ff 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -1,19 +1,20 @@
name: E2E
-# The one definition of an end-to-end run: a lane of console/tooling-e2e against
-# an environment. Called by pr-ci.yml (the fast lane on a throwaway stack, on
-# every PR) and by e2e-scheduled.yml (the provider lane after each merge, the
-# deployed environments nightly), and dispatchable by hand.
+# The one definition of an end-to-end run: a lane of console/tooling-e2e on a
+# throwaway stack built from this checkout. Called by pr-ci.yml (the fast lane,
+# on every PR) and by e2e-scheduled.yml (the provider lane after each merge and
+# nightly), and dispatchable by hand.
#
# lane fast provider-free scenarios over plain HTTP
# provider real agent sessions on a headless agent controller
# agentic the provider lane's setup, driven by scenario files: a
# model plays the human, another judges the result
-# browser the gateway UI in headless Chromium; on ephemeral the
-# UI is built from this checkout
-# environment ephemeral a fresh stack on the runner, built from this checkout
-# dev, staging an existing deployment, configured by the GitHub
-# Environment of that name
+# browser the gateway UI in headless Chromium, built from this
+# checkout
+#
+# Runs against the deployed environments are not here: they live with the
+# deployment configuration, which checks out this harness at the commit the
+# environment runs and drives it with SWITCH_E2E_ENVIRONMENT set.
#
# Fork safety: an ephemeral fast run generates every secret it uses on the
# runner and reads no repository or environment secret, so it runs the same
@@ -21,14 +22,9 @@ name: E2E
# GitHub Environment, whose protection rules decide which refs may run it:
#
# e2e-provider secret ANTHROPIC_API_KEY, for the controller's Claude Code
-# dev, staging vars SWITCH_API_URL, MATTERMOST_URL, MATTERMOST_TEAM, and
-# SWITCH_GATEWAY_UI_URL for the browser lane; secrets
-# SWITCH_GATEWAY_ADMIN_EMAIL, SWITCH_GATEWAY_ADMIN_PASSWORD,
-# SWITCH_AGENT_REGISTRATION_TOKEN, MATTERMOST_TOKEN
#
# Runs are strict (SWITCH_E2E_STRICT=1): a scenario that cannot run fails
-# rather than skips, so a deployment without Mattermost configured fails the
-# Mattermost scenario instead of passing without it.
+# rather than skips.
#
# Artifacts are uploaded only after the output has been scanned for the run's
# own secret values and by gitleaks. A scan that fails or does not run means
@@ -40,9 +36,6 @@ on:
lane:
type: string
required: true
- environment:
- type: string
- required: true
agent_type:
description: Provider for the provider lane
type: string
@@ -57,10 +50,6 @@ on:
type: choice
options: [fast, provider, browser, agentic]
default: fast
- environment:
- type: choice
- options: [ephemeral, dev, staging]
- default: ephemeral
agent_type:
description: Provider for the provider lane
type: choice
@@ -72,16 +61,10 @@ permissions:
jobs:
run:
- name: ${{ inputs.lane }} lane on ${{ inputs.environment }}
+ name: ${{ inputs.lane }} lane
runs-on: ubuntu-latest
timeout-minutes: 45
- environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || ((inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '') }}
- # One run at a time against a shared deployment, queued rather than
- # cancelled: a cancelled run leaves its agents and rooms behind. A throwaway
- # stack is the runner's own, so ephemeral runs never wait on each other.
- concurrency:
- group: ${{ inputs.environment == 'ephemeral' && format('e2e-ephemeral-{0}-{1}', inputs.lane, github.run_id) || format('e2e-{0}', inputs.environment) }}
- cancel-in-progress: false
+ environment: ${{ (inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '' }}
outputs:
summary: ${{ steps.summary.outputs.line }}
defaults:
@@ -92,7 +75,6 @@ jobs:
env:
OUT: ${{ github.workspace }}/build/e2e/out
LANE: ${{ inputs.lane }}
- TARGET: ${{ inputs.environment }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -102,17 +84,12 @@ jobs:
AGENT_TYPE: ${{ inputs.agent_type }}
run: |
case "$LANE" in fast|provider|browser|agentic) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esac
- case "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esac
case "$LANE" in provider|agentic) REAL_SESSION=1 ;; *) REAL_SESSION=0 ;; esac
- if [ "$REAL_SESSION" = 1 ] && [ "$TARGET" != ephemeral ]; then
- echo "::error::The $LANE lane runs on an ephemeral stack only."; exit 1
- fi
if [ "$REAL_SESSION" = 1 ] && [ "$AGENT_TYPE" != claude-code ]; then
echo "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1
fi
- name: Set up uv
- if: ${{ inputs.environment == 'ephemeral' }}
uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
with:
python-version: '3.12'
@@ -157,7 +134,6 @@ jobs:
echo "SWITCH_E2E_PROVIDER_ENV_FILE=$RUNNER_TEMP/provider.env" >>"$GITHUB_ENV"
- name: Run the lane on a throwaway stack
- if: ${{ inputs.environment == 'ephemeral' }}
env:
SWITCH_E2E_LANE: ${{ inputs.lane }}
SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}
@@ -170,85 +146,29 @@ jobs:
SWITCH_E2E_AGENTIC_BUDGET_USD: '2'
run: console/tooling-e2e/scripts/ephemeral-stack.sh
- - name: Run the lane against ${{ inputs.environment }}
- if: ${{ inputs.environment != 'ephemeral' }}
- working-directory: console/tooling-e2e
- env:
- SWITCH_E2E_LANE: ${{ inputs.lane }}
- SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}
- SWITCH_E2E_STRICT: '1'
- SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}
- SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}
- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
- MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}
- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
- MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}
- SWITCH_GATEWAY_UI_URL: ${{ vars.SWITCH_GATEWAY_UI_URL }}
- SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secrets
- SWITCH_E2E_OUTPUT_DIR: ${{ env.OUT }}
- run: |
- mkdir -p "$OUT"
- pnpm exec vitest run --reporter=verbose --reporter=junit --outputFile.junit="$OUT/junit.xml" \
- 2>&1 | tee "$OUT/vitest.log"
-
- # What interrupted runs left behind, whichever lane made it; cleanup
- # resolves as the fast lane, which needs nothing a lane adds. The floor
- # keeps it off a run still in progress, which is why it can run after
- # every suite.
- - name: Remove stale harness artefacts from ${{ inputs.environment }}
- if: ${{ always() && inputs.environment != 'ephemeral' }}
- working-directory: console/tooling-e2e
- env:
- SWITCH_E2E_LANE: fast
- SWITCH_E2E_ENVIRONMENT: ${{ inputs.environment }}
- SWITCH_E2E_STRICT: '1'
- SWITCH_API_URL: ${{ vars.SWITCH_API_URL }}
- SWITCH_GATEWAY_ADMIN_EMAIL: ${{ secrets.SWITCH_GATEWAY_ADMIN_EMAIL }}
- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
- MATTERMOST_URL: ${{ vars.MATTERMOST_URL }}
- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
- MATTERMOST_TEAM: ${{ vars.MATTERMOST_TEAM }}
- SWITCH_E2E_SECRETS_FILE: ${{ runner.temp }}/issued-secrets
- run: pnpm cleanup --older-than 180
-
- name: Summarise the run
id: summary
if: ${{ always() }}
run: |
node --experimental-strip-types console/tooling-e2e/src/ci-summary.ts \
- "$OUT/junit.xml" "$LANE" "$LANE lane on $TARGET" >>"$GITHUB_STEP_SUMMARY"
+ "$OUT/junit.xml" "$LANE" "$LANE lane" >>"$GITHUB_STEP_SUMMARY"
- name: Scan the output for secrets
id: scan
if: ${{ always() }}
- env:
- SWITCH_GATEWAY_ADMIN_PASSWORD: ${{ secrets.SWITCH_GATEWAY_ADMIN_PASSWORD }}
- SWITCH_AGENT_REGISTRATION_TOKEN: ${{ secrets.SWITCH_AGENT_REGISTRATION_TOKEN }}
- MATTERMOST_TOKEN: ${{ secrets.MATTERMOST_TOKEN }}
run: |
[ -d "$OUT" ] || { echo "No output to scan."; exit 1; }
- if [ "$TARGET" = ephemeral ]; then
- # The stack script scans for the secrets it generated, the provider
- # key file and every credential the server issued the harness, and
- # leaves this file only when none turned up.
- [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }
- else
- # The environment's own secrets, and every credential the server
- # issued the harness during the run (agent keys, gateway sessions),
- # which stay valid on a shared deployment after the run ends.
- { printf '%s\n' "$SWITCH_GATEWAY_ADMIN_PASSWORD" "$SWITCH_AGENT_REGISTRATION_TOKEN" "$MATTERMOST_TOKEN"
- cat "$RUNNER_TEMP/issued-secrets" 2>/dev/null || true
- } | console/tooling-e2e/scripts/scan-artifacts.sh "$OUT"
- fi
+ # The stack script scans for the secrets it generated, the provider
+ # key file and every credential the server issued the harness, and
+ # leaves this file only when none turned up.
+ [ -e "$OUT/.secret-scan-passed" ] || { echo "::error::The stack's own secret scan did not pass."; exit 1; }
docker run --rm -v "$OUT:/out:ro" zricethezav/gitleaks:v8.30.1 dir /out --redact --no-banner
- name: Upload the output
if: ${{ always() && steps.scan.outcome == 'success' }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
- name: e2e-${{ inputs.lane }}-${{ inputs.environment }}
+ name: e2e-${{ inputs.lane }}
path: ${{ env.OUT }}
include-hidden-files: true
retention-days: 14
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml
index cf38cf5ab..a004cc339 100644
--- a/.github/workflows/pr-ci.yml
+++ b/.github/workflows/pr-ci.yml
@@ -383,7 +383,7 @@ jobs:
# Real messages through a real server and Mattermost, on a stack the job
# builds from this checkout. Needs no secret, so a fork PR runs it the same.
- # The provider lane and the deployed environments are e2e-scheduled.yml's.
+ # The provider lane is e2e-scheduled.yml's.
e2e-fast:
name: E2E fast lane
needs: changes
@@ -391,7 +391,6 @@ jobs:
uses: ./.github/workflows/e2e.yml
with:
lane: fast
- environment: ephemeral
agent_type: claude-code
migration-heads-on-merge:
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index 563159ceb..0e6c34751 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -115,7 +115,9 @@ in `environments`, and its `requires` are configured. Otherwise it shows as
**skipped, with the reason**, never as passed. Under `SWITCH_E2E_STRICT=1` (how
`just e2e-fast` and CI run) a missing prerequisite, missing configuration or an
unreachable server **fails** the run instead, so a job that ran nothing cannot
-go green.
+go green. The one exception is an environment that deploys no Mattermost on
+purpose and says so with `SWITCH_E2E_NO_MATTERMOST=1`: its Mattermost
+scenarios skip even under strict mode, and the reason names the setting.
A scenario that changes server-global state (a feature flag, say) lists only
`ephemeral`. One that writes anything never lists `prod`.
@@ -193,11 +195,12 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.
| `MATTERMOST_TOKEN` | for Mattermost scenarios | Token of the **human** the harness posts as (a personal access token, or a login session token). |
| `MATTERMOST_TEAM` | for Mattermost scenarios | Team **slug** (e.g. `switch`), not its display name. |
| `SWITCH_E2E_STRICT` | no | `1` turns every "cannot run" into a failure. CI sets it. |
+| `SWITCH_E2E_NO_MATTERMOST` | no | `1` declares that the environment deploys no Mattermost, so its Mattermost scenarios skip even under strict mode. Refused alongside `MATTERMOST_*`. |
| `SWITCH_E2E_KEEP` | no | `1` leaves agents, rooms and channels behind for inspection. |
| `SWITCH_E2E_SERVER_LOG` | no | The server's log, written with `LOG_FORMAT=json`. When set, a scenario fails if the server logged an error while it ran. The stack script sets it. |
-| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. |
+| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script sets it. |
| `SWITCH_GATEWAY_UI_URL` | browser lane | Where the gateway UI is served. Its `/gateway` requests must reach `SWITCH_API_URL`. `just e2e-browser` sets it. |
-| `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script and `e2e.yml` set it. |
+| `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script sets it. |
| `SWITCH_E2E_DRIVER_MODEL` | agentic lane | Exact model id that plays the human, e.g. `claude-sonnet-5`. |
| `SWITCH_E2E_JUDGE_MODEL` | agentic lane | Exact model id that grades the conversation. |
| `SWITCH_E2E_AGENTIC_BUDGET_USD` | agentic lane | What one run may spend, driver, judge and agent together. The run stops once it is reached. |
@@ -405,30 +408,21 @@ on every PR with no server.
## In CI
-`.github/workflows/e2e.yml` defines a run: one lane against one environment
-(`ephemeral`, `dev` or `staging`). Other workflows call it, and it can also be
-dispatched by hand.
+`.github/workflows/e2e.yml` defines a run: one lane on a throwaway stack built
+from the checkout. Other workflows call it, and it can also be dispatched by
+hand.
| When | Lane | Environment | Required check |
| --- | --- | --- | --- |
| Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes |
| After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no |
-| Nightly (`e2e-scheduled.yml`) | provider; fast; browser | ephemeral; dev; dev | no |
+| Nightly (`e2e-scheduled.yml`) | provider | ephemeral | no |
| By hand only, for now (`e2e.yml`, dispatch) | agentic | ephemeral | no |
- **Secrets.** An ephemeral fast run generates every secret it uses on the
runner, so it works the same on a fork PR. Every other run reads its secrets
- from a GitHub Environment:
- - `e2e-provider` holds `ANTHROPIC_API_KEY`;
- - `dev` and `staging` hold the deployment's URLs as variables (the browser
- lane also needs `SWITCH_GATEWAY_UI_URL`) and its credentials as secrets.
-
- The header of `e2e.yml` lists every name. Each environment's protection rules
- decide which branches may use it.
-- **Concurrency.** Runs against a shared deployment queue behind each other.
- They are never cancelled, because a cancelled run leaves its artefacts
- behind. After every deployed run, `pnpm cleanup --older-than 180` removes
- what interrupted runs left.
+ from the GitHub Environment `e2e-provider`, which holds `ANTHROPIC_API_KEY`
+ and whose protection rules decide which branches may use it.
- **Artifacts.** The output directory is uploaded only after it passes two
scans. The first (`scan-artifacts.sh`) looks for the exact secret values the
run used: the ones it was configured with, plus every credential the server
@@ -442,6 +436,28 @@ dispatched by hand.
- one summary each night;
- a report when the provider lane fails after a merge.
+## Against a deployed environment
+
+Runs against the shared deployments are driven from the deployment
+configuration, not from this repository's CI: it knows which commit each
+environment runs, checks this harness out at that commit, and holds the
+environment's addresses and credentials. To run a lane there:
+
+```bash
+SWITCH_E2E_LANE=fast SWITCH_E2E_ENVIRONMENT=dev SWITCH_E2E_STRICT=1 \
+SWITCH_API_URL=… SWITCH_GATEWAY_ADMIN_EMAIL=… SWITCH_GATEWAY_ADMIN_PASSWORD=… \
+SWITCH_AGENT_REGISTRATION_TOKEN=… SWITCH_E2E_SECRETS_FILE=… pnpm test
+```
+
+- Give either the three `MATTERMOST_*` variables or `SWITCH_E2E_NO_MATTERMOST=1`.
+- The browser lane also needs `SWITCH_GATEWAY_UI_URL`.
+- Run one lane at a time per environment and never cancel a run: a cancelled
+ run leaves its agents and rooms behind. After each run,
+ `pnpm cleanup --older-than 180` removes what interrupted runs left.
+- Scan the output for the run's secrets before keeping it: the configured
+ ones, and every credential listed in `SWITCH_E2E_SECRETS_FILE`
+ (`scripts/scan-artifacts.sh` takes them on stdin).
+
## With a Switch Console session
The provider lane can also run against a **Switch Console session for the agent
diff --git a/console/tooling-e2e/src/core/core.test.ts b/console/tooling-e2e/src/core/core.test.ts
index 76447934a..9a0e56173 100644
--- a/console/tooling-e2e/src/core/core.test.ts
+++ b/console/tooling-e2e/src/core/core.test.ts
@@ -44,6 +44,23 @@ describe('configuration', () => {
);
});
+ it('refuses a Mattermost configuration beside a declaration that there is none', () => {
+ expect(loadEnv('fast', { ...BASE, SWITCH_E2E_NO_MATTERMOST: '1' }, {}).noMattermost).toBe(true);
+ expect(() =>
+ loadEnv(
+ 'fast',
+ {
+ ...BASE,
+ SWITCH_E2E_NO_MATTERMOST: '1',
+ MATTERMOST_URL: 'http://mm.invalid',
+ MATTERMOST_TOKEN: 'mm-token-placeholder',
+ MATTERMOST_TEAM: 'team',
+ },
+ {}
+ )
+ ).toThrow(/SWITCH_E2E_NO_MATTERMOST/);
+ });
+
it('refuses an unknown environment or lane', () => {
expect(() => loadEnv('fast', { ...BASE, SWITCH_E2E_ENVIRONMENT: 'qa' }, {})).toThrow(/qa/);
expect(() => laneFrom({ SWITCH_E2E_LANE: 'slow' }, {})).toThrow(/slow/);
@@ -169,6 +186,19 @@ describe('admission', () => {
reason: expect.stringContaining('MATTERMOST_URL'),
});
});
+
+ it('skips without failing, even when strict, on a Mattermost the environment does not deploy', () => {
+ const needsMattermost = header({ requires: ['mattermost'] });
+ expect(admit(needsMattermost, env({ strict: true, noMattermost: true }))).toMatchObject({
+ run: false,
+ failure: false,
+ reason: expect.stringContaining('does not deploy'),
+ });
+ const needsBoth = header({ requires: ['mattermost', 'gateway-ui'] });
+ expect(
+ admit(needsBoth, env({ strict: true, noMattermost: true, gatewayUiUrl: null }))
+ ).toMatchObject({ run: false, failure: true });
+ });
});
describe('cleanup age floor', () => {
diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.ts
index 826064f35..158e99390 100644
--- a/console/tooling-e2e/src/core/env.ts
+++ b/console/tooling-e2e/src/core/env.ts
@@ -140,6 +140,11 @@ export interface HarnessEnv {
switch: SwitchEnv;
/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */
mattermost: MattermostEnv | null;
+ /**
+ * The environment deploys no Mattermost, by design (`SWITCH_E2E_NO_MATTERMOST=1`).
+ * Scenarios that need one skip even in strict mode, and say why.
+ */
+ noMattermost: boolean;
provider: ProviderEnv;
agentic: AgenticEnv;
}
@@ -267,6 +272,12 @@ export function loadEnv(
.filter(([, value]) => value === undefined)
.map(([key]) => key);
if (missing.length > 0) throw new MissingConfigError(missing);
+ const noMattermost = get('SWITCH_E2E_NO_MATTERMOST') === '1';
+ if (noMattermost && mattermostGiven) {
+ throw new Error(
+ 'SWITCH_E2E_NO_MATTERMOST=1 says this environment has no Mattermost, but MATTERMOST_* is set too. Unset one.'
+ );
+ }
return {
lane,
@@ -290,6 +301,7 @@ export function loadEnv(
team: mattermost.MATTERMOST_TEAM as string,
}
: null,
+ noMattermost,
provider: {
agentRepoDir: get('SWITCH_E2E_AGENT_DIR') ?? null,
manifestPath: get('SWITCH_E2E_MANIFEST') ?? null,
diff --git a/console/tooling-e2e/src/core/scenario.ts b/console/tooling-e2e/src/core/scenario.ts
index 1ee572a3d..e33ee7fa0 100644
--- a/console/tooling-e2e/src/core/scenario.ts
+++ b/console/tooling-e2e/src/core/scenario.ts
@@ -88,6 +88,8 @@ export function assertValidHeaders(headers: readonly ScenarioHeader[]): void {
* is simply not this run's business. A missing prerequisite is different: the
* scenario belongs here and cannot run, which skips on a developer machine and
* fails in strict mode — CI must not report a scenario that never ran as green.
+ * The exception is a prerequisite the environment declares it lacks by design
+ * (`SWITCH_E2E_NO_MATTERMOST`): that skips everywhere, with the reason given.
*/
export type Admission = { run: true } | { run: false; reason: string; failure: boolean };
@@ -104,15 +106,22 @@ export function admit(header: ScenarioHeader, env: HarnessEnv): Admission {
}
const missing = header.requires.filter((prerequisite) => !satisfied(prerequisite, env));
if (missing.length > 0) {
+ const byDesign = missing.every((prerequisite) => absentByDesign(prerequisite, env));
return {
run: false,
- reason: `needs ${missing.map(describePrerequisite).join(' and ')}`,
- failure: env.strict,
+ reason:
+ `needs ${missing.map(describePrerequisite).join(' and ')}` +
+ (byDesign ? `, which '${env.environment}' does not deploy (SWITCH_E2E_NO_MATTERMOST)` : ''),
+ failure: env.strict && !byDesign,
};
}
return { run: true };
}
+function absentByDesign(prerequisite: Prerequisite, env: HarnessEnv): boolean {
+ return prerequisite === 'mattermost' && env.noMattermost;
+}
+
function satisfied(prerequisite: Prerequisite, env: HarnessEnv): boolean {
switch (prerequisite) {
case 'mattermost':