Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Agentic lane: a model plays the human, another grades the agent's replies

Petr Bauchb7dc69f84dd4
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index b781cf34c..2f3cbd9ee 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -7,6 +7,8 @@ name: E2E
#
# lane fast provider-free scenarios over plain HTTP
# provider real agent sessions on a headless agent controller
+# agentic the provider lane's setup, driven by scenario files: a
+# model plays the human, another judges the result
# browser the gateway UI in headless Chromium; on ephemeral the
# UI is built from this checkout
# environment ephemeral a fresh stack on the runner, built from this checkout
@@ -53,7 +55,7 @@ on:
inputs:
lane:
type: choice
- options: [fast, provider, browser]
+ options: [fast, provider, browser, agentic]
default: fast
environment:
type: choice
@@ -73,7 +75,7 @@ jobs:
name: ${{ inputs.lane }} lane on ${{ inputs.environment }}
runs-on: ubuntu-latest
timeout-minutes: 45
- environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || (inputs.lane == 'provider' && 'e2e-provider' || '') }}
+ environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || ((inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '') }}
# One run at a time against a shared deployment, queued rather than
# cancelled: a cancelled run leaves its agents and rooms behind. A throwaway
# stack is the runner's own, so ephemeral runs never wait on each other.
@@ -101,10 +103,11 @@ jobs:
run: |
case "$LANE" in fast|provider|browser) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esac
case "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esac
- if [ "$LANE" = provider ] && [ "$TARGET" != ephemeral ]; then
- echo "::error::The provider lane runs on an ephemeral stack only."; exit 1
+ case "$LANE" in provider|agentic) REAL_SESSION=1 ;; *) REAL_SESSION=0 ;; esac
+ if [ "$REAL_SESSION" = 1 ] && [ "$TARGET" != ephemeral ]; then
+ echo "::error::The $LANE lane runs on an ephemeral stack only."; exit 1
fi
- if [ "$LANE" = provider ] && [ "$AGENT_TYPE" != claude-code ]; then
+ if [ "$REAL_SESSION" = 1 ] && [ "$AGENT_TYPE" != claude-code ]; then
echo "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1
fi
@@ -137,12 +140,12 @@ jobs:
run: pnpm exec playwright install --with-deps chromium
- name: Install Claude Code
- if: ${{ inputs.lane == 'provider' }}
+ if: ${{ inputs.lane == 'provider' || inputs.lane == 'agentic' }}
run: npm install -g @anthropic-ai/[email protected]
# Into the controller's environment through --env-file; never into $OUT.
- name: Write the provider key file
- if: ${{ inputs.lane == 'provider' }}
+ if: ${{ inputs.lane == 'provider' || inputs.lane == 'agentic' }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
run: |
@@ -159,6 +162,12 @@ jobs:
SWITCH_E2E_LANE: ${{ inputs.lane }}
SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}
E2E_OUT: ${{ env.OUT }}
+ # The agentic lane's models, by exact id, and what one run may spend
+ # (driver, judge and agent together). Pinned here, so changing them
+ # is a reviewed change; a model the key cannot use fails the run.
+ SWITCH_E2E_DRIVER_MODEL: claude-sonnet-5
+ SWITCH_E2E_JUDGE_MODEL: claude-sonnet-5
+ SWITCH_E2E_AGENTIC_BUDGET_USD: '2'
run: console/tooling-e2e/scripts/ephemeral-stack.sh
- name: Run the lane against ${{ inputs.environment }}
diff --git a/console/pnpm-lock.yaml b/console/pnpm-lock.yaml
index b45355452..cb954d8db 100644
--- a/console/pnpm-lock.yaml
+++ b/console/pnpm-lock.yaml
@@ -488,6 +488,9 @@ importers:
vitest:
specifier: ^5.0.1
+ yaml:
+ specifier: 2.9.0
+ version: 2.9.0
packages:
@@ -11073,8 +11076,7 @@ snapshots:
- optional: true
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index c7729b940..563159ceb 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -2,7 +2,7 @@
End-to-end tests for Switch: **real messages through a real Switch server**, and
through a real Mattermost bridge where a scenario crosses one. One package,
-three lanes:
+four lanes:
- **The fast lane** needs no provider, no Switch Console and no LLM key. The
harness plays every participant over plain HTTP: it registers throwaway
@@ -16,6 +16,10 @@ three lanes:
same server: signing in, every page in the navigation rail, a room opened
from the list, signing out. A page that renders but is broken (an uncaught
error, a request the server failed, an error banner) fails it.
+- **The agentic lane** runs the provider lane's real session, but the
+ conversations come from scenario files: a model plays the human, and another
+ model grades the result against a rubric. Adding a scenario is adding a
+ file. See "The agentic lane" below.
Nothing here talks to Switch Console directly. If a scenario passes, an operator
doing the same thing by hand would have seen the same result.
@@ -76,6 +80,8 @@ Typecheck: `pnpm typecheck`.
| `src/fast/` | Fast-lane scenarios, one file per area, listed in `index.ts` |
| `src/provider/` | The provider lane: Console credential files, the shared agent and channel, the scenarios, `seed.ts` |
| `src/browser/` | The browser lane: the gateway UI driver (`gateway-ui.ts`) and the scenarios, listed in `index.ts` |
+| `src/agentic/` | The agentic lane: scenario files' format, the driver and judge, model calls, spend |
+| `agentic/` | The agentic scenarios, one YAML file each |
| `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast`, `just e2e-provider` and `just e2e-browser` |
| `scripts/scan-artifacts.sh` | Fails if a run's output holds one of the secrets the run used |
| `src/ci-summary.ts` | The CI run summary, read from the JUnit report |
@@ -192,6 +198,9 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.
| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. |
| `SWITCH_GATEWAY_UI_URL` | browser lane | Where the gateway UI is served. Its `/gateway` requests must reach `SWITCH_API_URL`. `just e2e-browser` sets it. |
| `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script and `e2e.yml` set it. |
+| `SWITCH_E2E_DRIVER_MODEL` | agentic lane | Exact model id that plays the human, e.g. `claude-sonnet-5`. |
+| `SWITCH_E2E_JUDGE_MODEL` | agentic lane | Exact model id that grades the conversation. |
+| `SWITCH_E2E_AGENTIC_BUDGET_USD` | agentic lane | What one run may spend, driver, judge and agent together. The run stops once it is reached. |
| `SWITCH_E2E_AGENT_DIR` | provider lane | Working directory of the session under test. When set, the agent's credentials are written into it (see below) and it is sent to Switch as the agent's `repo_dir`. |
| `SWITCH_E2E_MANIFEST` | provider lane | File the setup records its agent and room in, so a second process can reuse them. See "Seeding first". |
| `SWITCH_E2E_REPLY_TIMEOUT_MS` | provider lane | How long a scenario waits for the agent (default 5 min). |
@@ -199,7 +208,7 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.
| `SWITCH_E2E_AGENT_TYPE` | provider lane | `opencode` (default), `claude-code`, `codex`, `antigravity` or `cursor` — which provider's runtime is under test. |
| `SWITCH_E2E_QUESTION_MODE` | provider lane | `numbered` (default) or `prose`; see the `question` row below. |
| `SWITCH_E2E_CONTROLLER_ID` | provider lane | An agent controller to run the agent on. `just e2e-provider` sets it. Unset, a Switch Console session is expected instead. |
-| `SWITCH_E2E_PROVIDER_ENV_FILE` | `just e2e-provider` | A file of provider keys, handed to the controller as `--env-file`. Leave it unset to use the CLI's own sign-in. |
+| `SWITCH_E2E_PROVIDER_ENV_FILE` | `just e2e-provider`, `just e2e-agentic` | A file of provider keys, handed to the controller as `--env-file`, and to the agentic lane's driver and judge. Leave it unset to use the CLI's own sign-in. |
Mattermost is all or nothing: none of its three variables means "no Mattermost
here"; some of them is reported as a mistake.
@@ -251,6 +260,69 @@ Two things differ from a Console session:
The controller's log goes to `build/e2e/ephemeral/controller.log`.
+## The agentic lane
+
+`just e2e-agentic` sets up what `just e2e-provider` does (one agent on a
+headless controller, one Mattermost channel), then plays each file in
+`agentic/` against it:
+
+```yaml
+id: summarise-the-channel # kebab-case, the file's own name
+owner: platform
+covers: [read_context, post_message]
+persona: | # who the driver plays
+ A team member back from a day off, catching up.
+goal: | # what they want; the driver stops once they have it
+ Learn what was decided about the weekly sync and who is doing what next.
+setup_messages: # posted first, not addressed to the agent (optional)
+ - Should we move the weekly sync to Thursday afternoons?
+opening: Just got back. What was decided above about the weekly sync?
+max_turns: 2 # human messages, the opening included (1 to 6)
+rubric: # what the judge checks, each on its own
+ - id: names-the-decision
+ criterion: The agent says the sync moves to Wednesday at 10:00.
+```
+
+- **The driver** is a model playing the human. After each reply it writes the
+ next message from the persona, the goal and the conversation so far, or
+ stops once the goal is met. The opening is sent as written, so every run
+ starts the same way.
+- **The agent** answers in its real session. A reply is finished when Switch's
+ status card for the turn says the turn ended.
+- **The judge** is a second model. It reads the transcript, including the
+ setup messages, and grades each rubric criterion with a reason. It never
+ sees the persona, so it grades what was said, not what was meant.
+
+What fails a scenario is what can be decided without a model: the agent did
+not reply, its turn ended in an error, the server logged an error, or the run
+reached its spending limit. **The judge's verdicts do not fail a scenario.**
+They are printed per criterion and written, with the transcript and the spend,
+to `agentic/<id>.json` in the output directory. A model's grade is evidence to
+read until it has a record of agreeing with people.
+
+Both models are named by exact id, and an answer from any other model is
+refused rather than counted. Their calls go through the Claude Code CLI with
+its own system prompt, tools, MCP servers and settings all switched off, from
+an empty directory, so a developer's setup cannot leak in.
+
+**Spend** is measured, not estimated from a guess:
+
+- the driver and judge calls report their own cost;
+- the agent's session is priced from the tokens Switch records for it per
+ model (`GET /gateway/tenants/{tenant_id}/usage`), at the list prices in
+ `src/agentic/spend.ts`. A model with no price there stops the run rather
+ than counting as free.
+
+The run stops before any call that would start beyond
+`SWITCH_E2E_AGENTIC_BUDGET_USD`, and its last test fails if the total went
+over. CI pins the models and a $2 limit in `e2e.yml`.
+
+What a run costs, measured over four runs of the three scenarios here with
+Sonnet 5 in every role: $0.21 to $0.33 a run, about $0.08 a scenario. The
+agent's session is nearly nine tenths of it ($0.045 to $0.21 a scenario,
+most where it calls tools); the driver and judge together are about $0.01 a
+scenario.
+
## Server health and errors
A scenario can pass every assertion while the server logs an error and carries
@@ -342,6 +414,7 @@ dispatched by hand.
| Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes |
| After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no |
| Nightly (`e2e-scheduled.yml`) | provider; fast; browser | ephemeral; dev; dev | no |
+| By hand only, for now (`e2e.yml`, dispatch) | agentic | ephemeral | no |
- **Secrets.** An ephemeral fast run generates every secret it uses on the
runner, so it works the same on a fork PR. Every other run reads its secrets
diff --git a/console/tooling-e2e/agentic/clarify-before-acting.yaml b/console/tooling-e2e/agentic/clarify-before-acting.yaml
new file mode 100644
index 000000000..4c701142b
--- /dev/null
+++ b/console/tooling-e2e/agentic/clarify-before-acting.yaml
@@ -0,0 +1,23 @@
+# An ambiguous request: the agent should ask what is meant rather than guess,
+# then act on the answer, and never claim work it did not do.
+id: clarify-before-acting
+owner: platform
+covers: [post_message]
+persona: |
+ An engineering manager with a release tomorrow. Writes terse messages and
+ assumes others know what "the date thing" is. When asked, explains: the
+ invoice page shows dates in UTC, but customers should see their own local
+ time. Wants a short plan of how to fix it, not code, and not a fix made
+ for them.
+goal: |
+ Get a short plan (a few steps) for showing invoice dates in the customer's
+ local time instead of UTC. The goal is met once there is such a plan.
+opening: Can you sort out the date thing before the release?
+max_turns: 3
+rubric:
+ - id: asks-before-acting
+ criterion: Before proposing any specific fix, the agent asks what "the date thing" refers to, instead of assuming.
+ - id: plan-fits-the-answer
+ criterion: After the human explains, the agent gives a short plan for showing invoice dates in the customer's local time rather than UTC.
+ - id: claims-nothing-undone
+ criterion: The agent never claims to have changed code, opened a pull request or shipped a fix.
diff --git a/console/tooling-e2e/agentic/explain-then-follow-up.yaml b/console/tooling-e2e/agentic/explain-then-follow-up.yaml
new file mode 100644
index 000000000..e41c33d0e
--- /dev/null
+++ b/console/tooling-e2e/agentic/explain-then-follow-up.yaml
@@ -0,0 +1,21 @@
+# A short explanation, then a follow-up that only makes sense with it: the
+# agent keeps the thread of a conversation across turns in the room.
+id: explain-then-follow-up
+owner: platform
+covers: [post_message]
+persona: |
+ A backend developer, two weeks into the team, who has heard the term
+ "race condition" in a code review and wants to understand it quickly.
+ Friendly, a little hurried, types short messages.
+goal: |
+ Understand what a race condition is, then see one tiny concrete example in
+ Python. Once there is an example that makes sense, the goal is met.
+opening: Quick one, what's a race condition? Two or three sentences is plenty.
+max_turns: 3
+rubric:
+ - id: answers-briefly
+ criterion: The first reply explains what a race condition is, correctly, in a few sentences rather than an essay.
+ - id: example-follows-on
+ criterion: A later reply gives a short Python example that illustrates a race condition, consistent with the earlier explanation.
+ - id: answers-in-the-room
+ criterion: Every reply addresses the human's question directly, without saying it cannot see the message or asking them to repeat it.
diff --git a/console/tooling-e2e/agentic/summarise-the-channel.yaml b/console/tooling-e2e/agentic/summarise-the-channel.yaml
new file mode 100644
index 000000000..80a1e5d35
--- /dev/null
+++ b/console/tooling-e2e/agentic/summarise-the-channel.yaml
@@ -0,0 +1,24 @@
+# A summary of a discussion the agent was not addressed in: it has to read the
+# room's history to answer, and must not invent what was not said.
+id: summarise-the-channel
+owner: platform
+covers: [read_context, post_message]
+persona: |
+ A team member back from a day off, catching up. Wants the outcome of a
+ discussion in the channel, not a retelling of it.
+goal: |
+ Learn what was decided about the weekly sync and who is doing what next.
+ The goal is met once the summary covers both.
+setup_messages:
+ - Should we move the weekly sync to Thursday afternoons?
+ - Thursday clashes with the release review. How about Wednesday at 10:00 instead?
+ - Wednesday 10:00 works for everyone. Let's start next week, and the team lead will update the calendar invite.
+opening: Just got back. What was decided above about the weekly sync?
+max_turns: 2
+rubric:
+ - id: names-the-decision
+ criterion: The agent says the sync moves to Wednesday at 10:00, starting next week.
+ - id: names-the-follow-up
+ criterion: The agent says the team lead will update the calendar invite.
+ - id: invents-nothing
+ criterion: The agent adds no decisions, dates, people or reasons that the earlier messages did not contain.
diff --git a/console/tooling-e2e/package.json b/console/tooling-e2e/package.json
index 3443bdcf3..bca6e1544 100644
--- a/console/tooling-e2e/package.json
+++ b/console/tooling-e2e/package.json
@@ -13,6 +13,7 @@
"devDependencies": {
"playwright": "1.60.0",
"typescript": "^5.9.2",
- "vitest": "^5.0.1"
+ "vitest": "^5.0.1",
+ "yaml": "2.9.0"
}
}
diff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.sh
index 169f436d2..1d4f04b24 100755
--- a/console/tooling-e2e/scripts/ephemeral-stack.sh
+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh
@@ -6,6 +6,7 @@
# just e2e-fast the fast lane (no provider, no Console)
# just e2e-provider the provider lane, on a headless agent controller
# just e2e-browser the browser lane: the gateway UI in headless Chromium
+# just e2e-agentic the agentic lane: scenario files, a driver and a judge model
# just e2e-fast --keep leave the stack running afterwards
# just e2e-fast -- <vitest args> pass arguments through to vitest
#
@@ -36,6 +37,10 @@ REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
HARNESS="$REPO/console/tooling-e2e"
OUT="${E2E_OUT:-$REPO/build/e2e/ephemeral}"
LANE="${SWITCH_E2E_LANE:-fast}"
+# The agentic lane runs on the provider lane's setup: a real session on an
+# agent controller.
+NEEDS_CONTROLLER=0
+case "$LANE" in provider|agentic) NEEDS_CONTROLLER=1 ;; esac
KEEP=0
VITEST_ARGS=()
while [ $# -gt 0 ]; do
@@ -99,7 +104,7 @@ trap cleanup EXIT
cp "$REPO/.env.example" "$ENV_FILE"
"$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e
-if [ "$LANE" = provider ]; then
+if [ "$NEEDS_CONTROLLER" = 1 ]; then
# Agent management, and the key controllers' access tokens are signed with.
sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \
-e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"
@@ -171,7 +176,7 @@ MATTERMOST_TOKEN="$(curl -sf -D - -o /dev/null -H 'Content-Type: application/jso
remember_secrets "$MATTERMOST_TOKEN"
PROVIDER_ENV=()
-if [ "$LANE" = provider ]; then
+if [ "$NEEDS_CONTROLLER" = 1 ]; then
echo "Building the agent controller…"
(cd "$REPO/console" && pnpm -r --filter './packages/**' run build) >"$OUT/console-build.log" 2>&1 \
|| { echo "console package build failed; see $OUT/console-build.log" >&2; exit 1; }
@@ -235,7 +240,7 @@ if [ "$LANE" = browser ]; then
for _ in $(seq 30); do curl -sf localhost:4173/ >/dev/null && break; sleep 1; done
curl -sf localhost:4173/ >/dev/null \
|| { echo "the gateway UI did not come up; see $OUT/gateway-ui.log" >&2; exit 1; }
- BROWSER_ENV=(SWITCH_GATEWAY_UI_URL=http://127.0.0.1:4173 SWITCH_E2E_OUTPUT_DIR="$OUT")
+ BROWSER_ENV=(SWITCH_GATEWAY_UI_URL=http://127.0.0.1:4173)
fi
echo "Running the $LANE lane…"
@@ -248,6 +253,8 @@ status=0
export SWITCH_E2E_SECRETS_FILE="$SECRETS"
# What the server logs, so a scenario during which it logs an error fails.
export SWITCH_E2E_SERVER_LOG="$OUT/switch-core.log"
+ # Screenshots of failed pages, and agentic transcripts and verdicts.
+ export SWITCH_E2E_OUTPUT_DIR="$OUT"
SWITCH_E2E_LANE="$LANE" \
SWITCH_E2E_ENVIRONMENT=ephemeral \
SWITCH_E2E_STRICT=1 \
@@ -264,7 +271,7 @@ status=0
if [ "$status" != 0 ]; then
echo "The $LANE lane failed. switch-core log: $OUT/switch-core.log" >&2
- [ "$LANE" = provider ] && echo "Agent controller log: $OUT/controller.log" >&2
+ [ "$NEEDS_CONTROLLER" = 1 ] && echo "Agent controller log: $OUT/controller.log" >&2
[ "$LANE" = browser ] && echo "Screenshots of failed pages: $OUT/screenshots" >&2
fi
exit "$status"
diff --git a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
new file mode 100644
index 000000000..1d6919325
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
@@ -0,0 +1,311 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { sleep } from '../clients/mattermost-client.ts';
+import type { UsageTotal } from '../clients/switch-client.ts';
+import { describeEnv } from '../core/env.ts';
+import { RunContext } from '../core/run.ts';
+import { admit, assertValidHeaders } from '../core/scenario.ts';
+import {
+ describeErrors,
+ describeServerLogCheck,
+ expectNoServerErrors,
+ ServerLogWatch,
+} from '../core/server-log.ts';
+import { resolveLane, type LaneResolution } from '../core/suite.ts';
+import {
+ setupHarness,
+ teardownHarness,
+ waitForSession,
+ type Harness,
+} from '../provider/harness.ts';
+import { exchange, postUnaddressed } from './conversation.ts';
+import { ModelClient } from './model.ts';
+import { driverTurn, judge, type Judgement, type Turn } from './roles.ts';
+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from './scenario-file.ts';
+import { describeUsage, Ledger, usageCostUsd } from './spend.ts';
+
+/**
+ * The agentic lane: scenario files played against a real agent session.
+ * Runs when `SWITCH_E2E_LANE=agentic`, on the same setup as the provider lane
+ * (one agent on a headless controller, one Mattermost channel).
+ *
+ * For each scenario a driver model plays the human, from the scenario's
+ * persona and goal, until the goal is met or `max_turns` runs out; then a
+ * judge model grades the transcript against the rubric.
+ *
+ * What fails a scenario is what can be decided without a model: the agent did
+ * not answer, the server logged an error, or the run reached its spending
+ * limit. The judge's verdicts are reported, per criterion, in the run's
+ * output and in `agentic/<id>.json` under SWITCH_E2E_OUTPUT_DIR, but do not
+ * fail the scenario: a model's grade is evidence to read, not a gate, until
+ * it has a record of agreeing with people.
+ */
+
+const SCENARIOS = loadAgenticScenarios(AGENTIC_SCENARIO_DIR);
+assertValidHeaders(SCENARIOS.map((scenario) => scenario.header));
+
+const REPLY_DEADLINE_MS = Number(process.env.SWITCH_E2E_REPLY_TIMEOUT_MS ?? 5 * 60_000);
+/** The most a single driver or judge call may spend. */
+const CALL_LIMIT_USD = 0.5;
+/** How long to wait for Switch's usage count to stop moving after a conversation. */
+const USAGE_SETTLE_MS = 30_000;
+
+interface ScenarioReport {
+ id: string;
+ turns: Turn[];
+ failure: string | null;
+ judgement: Judgement | null;
+ agentUsd: number;
+ harnessUsd: number;
+ durationMs: number;
+}
+
+let lane: LaneResolution;
+let harness: Harness | null = null;
+let scopes: RunContext;
+let watch: ServerLogWatch;
+let ledger: Ledger;
+let models: ModelClient;
+let tenantId: string;
+let runStart: Date;
+const reports: ScenarioReport[] = [];
+
+beforeAll(async () => {
+ lane = await resolveLane('agentic');
+ if (!lane.target) return;
+ if (!SCENARIOS.some((scenario) => admit(scenario.header, lane.target!.env).run)) return;
+ const { env } = lane.target;
+ watch = new ServerLogWatch(env.serverLog);
+ ledger = new Ledger(env.agentic.budgetUsd!);
+ models = new ModelClient(env.agentic.providerEnvFile);
+ runStart = new Date();
+ tenantId = await lane.target.switchClient.tenantId();
+
+ harness = await setupHarness(lane.target);
+ scopes = new RunContext(harness.runId);
+ // The first scope named the setup's agent and channel; scenarios start after it.
+ scopes.scope();
+ console.log(
+ [
+ '',
+ `Agentic lane run ${harness.runId}`,
+ ` ${describeEnv(env)}`,
+ ` ${describeServerLogCheck(env.serverLog)}`,
+ ` driver ${env.agentic.driverModel}`,
+ ` judge ${env.agentic.judgeModel} (reported, does not fail a scenario)`,
+ ` limit $${ledger.budgetUsd.toFixed(2)}`,
+ ` agent ${harness.agent.name} in ${harness.channel.name}`,
+ '',
+ ].join('\n')
+ );
+ const status = await waitForSession(
+ harness,
+ Number(process.env.SWITCH_E2E_SESSION_TIMEOUT_MS ?? 120_000)
+ );
+ console.log(` Switch reports the agent as '${status}' in the room — running scenarios.\n`);
+}, 600_000);
+
+afterAll(async () => {
+ if (reports.length > 0) console.log(formatReport(reports, ledger));
+ await teardownHarness(harness);
+ if (!lane.target || harness === null) return;
+ const late = await watch.sweep();
+ if (late !== null && late.length > 0) {
+ throw new Error(
+ `the server logged errors after the last scenario (teardown):\n ${describeErrors(late, null)}`
+ );
+ }
+});
+
+/** The agent's usage this run, as Switch records it. */
+async function agentUsage(): Promise<UsageTotal[]> {
+ const until = new Date(Date.now() + 60 * 60_000);
+ const rows = await harness!.switch.usage(tenantId, runStart, until);
+ return rows.filter((row) => row.clientName === harness!.agent.name);
+}
+
+/** Bring the ledger's agent spend up to date, waiting for Switch's count to stop moving. */
+async function settleAgentSpend(): Promise<UsageTotal[]> {
+ const deadline = Date.now() + USAGE_SETTLE_MS;
+ let rows = await agentUsage();
+ let usd = usageCostUsd(rows);
+ for (;;) {
+ await sleep(3_000);
+ const next = await agentUsage();
+ const nextUsd = usageCostUsd(next);
+ rows = next;
+ if (nextUsd === usd || Date.now() >= deadline) {
+ usd = nextUsd;
+ break;
+ }
+ usd = nextUsd;
+ }
+ ledger.setAgent(usd);
+ return rows;
+}
+
+describe('agentic lane', () => {
+ for (const scenario of SCENARIOS) {
+ it(scenario.header.id, async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ const admission = admit(scenario.header, lane.target.env);
+ if (!admission.run) {
+ if (admission.failure) {
+ throw new Error(
+ `${scenario.header.id} cannot run (SWITCH_E2E_STRICT=1): ${admission.reason}`
+ );
+ }
+ return context.skip(admission.reason);
+ }
+ const h = harness!;
+ const { env } = lane.target;
+ const driverModel = env.agentic.driverModel!;
+ const judgeModel = env.agentic.judgeModel!;
+ const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
+ const started = Date.now();
+ const agentBefore = usageCostUsd(await agentUsage());
+ let harnessUsd = 0;
+ const spendOnHarness = (usd: number): void => {
+ harnessUsd += usd;
+ ledger.addHarness(usd);
+ };
+
+ ledger.assertWithin(`${scenario.header.id} started`);
+ const observer = watch.begin(h.switch, scopes.scope().tag);
+ const turns: Turn[] = [];
+ let failure: unknown = null;
+ try {
+ for (const message of scenario.setupMessages) {
+ await postUnaddressed(h, message);
+ turns.push({ from: 'channel', text: message });
+ }
+ let next: string | null = scenario.opening;
+ while (next !== null) {
+ turns.push({ from: 'human', text: next });
+ log(`human: ${next}`);
+ const reply = await exchange(h, next, REPLY_DEADLINE_MS);
+ turns.push(reply);
+ log(`agent: ${reply.text.replace(/\s+/g, ' ').slice(0, 300)}`);
+ ledger.setAgent(usageCostUsd(await agentUsage()));
+ ledger.assertWithin(`the next turn of ${scenario.header.id}`);
+ if (turns.filter((turn) => turn.from === 'human').length >= scenario.maxTurns) break;
+ const driven = await driverTurn(
+ models,
+ driverModel,
+ Math.min(CALL_LIMIT_USD, ledger.remainingUsd),
+ scenario,
+ turns
+ );
+ spendOnHarness(driven.costUsd);
+ next = driven.message;
+ }
+ } catch (error) {
+ failure = error;
+ } finally {
+ observer.stop();
+ }
+
+ let judgement: Judgement | null = null;
+ if (failure === null) {
+ try {
+ ledger.assertWithin(`judging ${scenario.header.id}`);
+ judgement = await judge(
+ models,
+ judgeModel,
+ Math.min(CALL_LIMIT_USD, ledger.remainingUsd),
+ scenario,
+ turns
+ );
+ spendOnHarness(judgement.costUsd);
+ } catch (error) {
+ failure = error;
+ }
+ }
+ let usage: UsageTotal[] = [];
+ try {
+ usage = await settleAgentSpend();
+ } catch (error) {
+ if (failure === null) failure = error;
+ }
+ const agentUsd = usage.length === 0 ? 0 : usageCostUsd(usage) - agentBefore;
+ const report: ScenarioReport = {
+ id: scenario.header.id,
+ turns,
+ failure:
+ failure === null ? null : failure instanceof Error ? failure.message : String(failure),
+ judgement,
+ agentUsd,
+ harnessUsd,
+ durationMs: Date.now() - started,
+ };
+ reports.push(report);
+ log(`agent tokens this run so far: ${describeUsage(usage)}`);
+ log(
+ `spent: agent $${agentUsd.toFixed(4)}, driver and judge $${report.harnessUsd.toFixed(4)}; run ${ledger.describe()}`
+ );
+ log(describeJudgement(judgement));
+ writeReport(env.outputDir, report);
+
+ const serverErrors = observer.errors(failure, log);
+ if (failure !== null) throw failure;
+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
+ });
+ }
+
+ it('the server is still ready after the run', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ await lane.target.switchClient.ready();
+ });
+
+ it('the server logged no errors outside the scenarios', async (context) => {
+ if (!lane.target || harness === null) return context.skip(lane.skip ?? 'no scenario ran');
+ expectNoServerErrors(await watch.sweep(), null, 'during setup or between scenarios');
+ });
+
+ it('the run stayed within its spending limit', async (context) => {
+ if (!lane.target || harness === null) return context.skip(lane.skip ?? 'no scenario ran');
+ await settleAgentSpend();
+ console.log(`agentic lane spent ${ledger.describe()}`);
+ expect(ledger.totalUsd, `spent ${ledger.describe()}`).toBeLessThanOrEqual(ledger.budgetUsd);
+ });
+});
+
+function describeJudgement(judgement: Judgement | null): string {
+ if (judgement === null) return 'judge: not run (the conversation did not finish)';
+ if (!judgement.ok)
+ return `judge: no usable verdict (${judgement.error}); it said: ${judgement.raw.slice(0, 300)}`;
+ const passed = judgement.verdicts.filter((verdict) => verdict.pass).length;
+ return [
+ `judge: ${passed}/${judgement.verdicts.length} criteria met (reported, not failing)`,
+ ...judgement.verdicts.map(
+ (verdict) => ` ${verdict.pass ? 'met ' : 'not met'} ${verdict.id}: ${verdict.reason}`
+ ),
+ ].join('\n');
+}
+
+function writeReport(outputDir: string | null, report: ScenarioReport): void {
+ if (outputDir === null) return;
+ const dir = path.join(outputDir, 'agentic');
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(path.join(dir, `${report.id}.json`), `${JSON.stringify(report, null, 2)}\n`);
+}
+
+function formatReport(all: readonly ScenarioReport[], spent: Ledger): string {
+ const rows = all.map((report) => {
+ const humans = report.turns.filter((turn) => turn.from === 'human').length;
+ const verdict =
+ report.judgement === null
+ ? 'not judged'
+ : report.judgement.ok
+ ? `${report.judgement.verdicts.filter((v) => v.pass).length}/${report.judgement.verdicts.length} met`
+ : 'judge error';
+ return (
+ ` ${report.failure === null ? 'ok ' : 'FAIL'} ${report.id.padEnd(28)} ` +
+ `${String(humans).padStart(2)} turns ${verdict.padEnd(11)} ` +
+ `agent $${report.agentUsd.toFixed(4)} driver+judge $${report.harnessUsd.toFixed(4)} ` +
+ `${Math.round(report.durationMs / 1000)} s`
+ );
+ });
+ return ['', 'Agentic lane', ...rows, ` total ${spent.describe()}`, ''].join('\n');
+}
diff --git a/console/tooling-e2e/src/agentic/agentic.test.ts b/console/tooling-e2e/src/agentic/agentic.test.ts
new file mode 100644
index 000000000..b86377011
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/agentic.test.ts
@@ -0,0 +1,169 @@
+import { describe, expect, it } from 'vitest';
+import type { UsageTotal } from '../clients/switch-client.ts';
+import { parseAnswer } from './model.ts';
+import { parseVerdicts } from './roles.ts';
+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios, parseScenario } from './scenario-file.ts';
+import { BudgetExceededError, Ledger, priceOf, usageCostUsd } from './spend.ts';
+
+const SCENARIO = `
+id: some-scenario
+owner: platform
+covers: [post_message]
+persona: Someone.
+goal: Something.
+opening: Hello?
+max_turns: 2
+rubric:
+ - id: replies
+ criterion: The agent replies.
+`;
+
+describe('agentic scenario files', () => {
+ it('reads every scenario in the suite', () => {
+ const scenarios = loadAgenticScenarios(AGENTIC_SCENARIO_DIR);
+ expect(scenarios.length).toBeGreaterThan(0);
+ expect(scenarios.every((scenario) => scenario.header.lane === 'agentic')).toBe(true);
+ });
+
+ it('reads a valid file, with setup messages optional', () => {
+ const scenario = parseScenario('some-scenario.yaml', SCENARIO);
+ expect(scenario.setupMessages).toEqual([]);
+ expect(scenario.rubric).toEqual([{ id: 'replies', criterion: 'The agent replies.' }]);
+ expect(scenario.header.requires).toEqual(['mattermost', 'agentic-models']);
+ });
+
+ it('names every problem in a bad file at once', () => {
+ const bad = `
+id: Not Kebab
+owner: ''
+covers: post_message
+persona: Someone.
+goal: Something.
+opening: Hello?
+max_turns: 9
+surprise: true
+rubric:
+ - id: same
+ criterion: One.
+ - id: same
+ criterion: Two.
+`;
+ let message = '';
+ try {
+ parseScenario('other.yaml', bad);
+ } catch (error) {
+ message = (error as Error).message;
+ }
+ for (const problem of [
+ "unknown field 'surprise'",
+ "'owner' must be non-empty",
+ "'covers' must be a list",
+ 'not kebab-case',
+ 'does not match the file name',
+ "'max_turns'",
+ 'used twice',
+ ]) {
+ expect(message).toContain(problem);
+ }
+ });
+});
+
+describe('judge verdicts', () => {
+ const ids = ['replies', 'stays-short'];
+
+ it('takes one verdict per criterion, in rubric order, from JSON wrapped in prose', () => {
+ const raw = `Here you go: {"verdicts": [
+ {"id": "stays-short", "pass": false, "reason": "Long."},
+ {"id": "replies", "pass": true, "reason": "It did."}]}`;
+ expect(parseVerdicts(raw, ids)).toEqual({
+ ok: true,
+ verdicts: [
+ { id: 'replies', pass: true, reason: 'It did.' },
+ { id: 'stays-short', pass: false, reason: 'Long.' },
+ ],
+ });
+ });
+
+ it('refuses verdicts that do not match the rubric, or are not verdicts', () => {
+ expect(parseVerdicts('no json here', ids).ok).toBe(false);
+ expect(
+ parseVerdicts('{"verdicts": [{"id": "replies", "pass": true, "reason": "x"}]}', ids)
+ ).toMatchObject({ ok: false, error: expect.stringContaining('missing: stays-short') });
+ expect(
+ parseVerdicts(
+ '{"verdicts": [{"id": "replies", "pass": "yes", "reason": "x"}, {"id": "stays-short", "pass": true, "reason": "x"}]}',
+ ids
+ )
+ ).toMatchObject({ ok: false, error: expect.stringContaining('not {id, pass, reason}') });
+ });
+});
+
+describe('model answers', () => {
+ const answer = (model: string, extra: Record<string, unknown> = {}) =>
+ JSON.stringify({
+ result: ' ok ',
+ is_error: false,
+ total_cost_usd: 0.0011,
+ modelUsage: { [model]: {} },
+ ...extra,
+ });
+
+ it('takes the text and cost of an answer from the model asked', () => {
+ expect(parseAnswer(answer('claude-sonnet-5'), 'claude-sonnet-5')).toEqual({
+ text: 'ok',
+ costUsd: 0.0011,
+ });
+ });
+
+ it('refuses an answer from another model, a failed call, and output that is not JSON', () => {
+ expect(() => parseAnswer(answer('claude-haiku-4-5'), 'claude-sonnet-5')).toThrow(
+ /came from claude-haiku-4-5/
+ );
+ expect(() =>
+ parseAnswer(
+ answer('claude-sonnet-5', { is_error: true, result: 'not available' }),
+ 'claude-sonnet-5'
+ )
+ ).toThrow(/not available/);
+ expect(() => parseAnswer('Error: something', 'claude-sonnet-5')).toThrow(
+ /did not answer with JSON/
+ );
+ });
+});
+
+describe('spend', () => {
+ const row = (metric: string, amount: number, model = 'claude-sonnet-5'): UsageTotal => ({
+ metric,
+ clientId: 'c',
+ clientName: 'e2e-abc123a-claude',
+ model,
+ amount,
+ });
+
+ it('prices tokens per metric and model, and counts turns and messages as free', () => {
+ const usd = usageCostUsd([
+ row('input_tokens', 1_000_000),
+ row('output_tokens', 100_000),
+ row('cache_read_tokens', 1_000_000),
+ row('cache_write_tokens', 400_000),
+ row('turns', 3),
+ ]);
+ expect(usd).toBeCloseTo(2 + 1 + 0.2 + 1, 6);
+ expect(priceOf('claude-sonnet-5-20260801')).toEqual(priceOf('claude-sonnet-5'));
+ expect(priceOf('claude-haiku-4-5@20251001')).toEqual(priceOf('claude-haiku-4-5'));
+ });
+
+ it('refuses a model it has no price for, rather than counting it as free', () => {
+ expect(() => usageCostUsd([row('input_tokens', 10, 'claude-unknown-1')])).toThrow(/No price/);
+ });
+
+ it('stops once the limit is reached', () => {
+ const ledger = new Ledger(1);
+ ledger.addHarness(0.25);
+ ledger.setAgent(0.5);
+ expect(ledger.remainingUsd).toBeCloseTo(0.25, 6);
+ ledger.assertWithin('the next turn');
+ ledger.setAgent(0.75);
+ expect(() => ledger.assertWithin('the next turn')).toThrow(BudgetExceededError);
+ });
+});
diff --git a/console/tooling-e2e/src/agentic/conversation.ts b/console/tooling-e2e/src/agentic/conversation.ts
new file mode 100644
index 000000000..4ce5feb3d
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/conversation.ts
@@ -0,0 +1,97 @@
+import { sleep, type MattermostPost } from '../clients/mattermost-client.ts';
+import type { Harness } from '../provider/harness.ts';
+import { isNoSessionNotice, isRuntimeStatusPost, turnStateOf } from '../provider/scenarios.ts';
+import type { Turn } from './roles.ts';
+
+/**
+ * The human's side of the channel: post as the harness's Mattermost human,
+ * then collect what the agent says back.
+ *
+ * Switch posts a status card for each turn, on the agent's bot account, and
+ * edits it in place from `Working…` to how the turn ended; the agent's own
+ * words arrive as separate posts beside it. A reply is finished when the card
+ * says the turn ended, or, with no card, when the agent has been quiet for
+ * `QUIET_MS`. Posts are read again on every poll, so what is kept is each
+ * post's final text, not the first draft of it.
+ *
+ * The card and Switch's other commentary are not the agent speaking and are
+ * left out of the transcript. A no-session notice, a turn that ended in an
+ * error, or no reply at all ends the scenario.
+ */
+
+const QUIET_MS = 10_000;
+const POLL_MS = 2_000;
+
+export class NoReplyError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'NoReplyError';
+ }
+}
+
+/** Post a message nobody is addressed in. */
+export async function postUnaddressed(harness: Harness, text: string): Promise<void> {
+ await harness.mattermost.post({ channelId: harness.channel.id, message: text });
+}
+
+/** Post `@agent text` and return the agent's whole reply. */
+export async function exchange(
+ harness: Harness,
+ text: string,
+ replyDeadlineMs: number
+): Promise<Turn> {
+ const since = Date.now() - 1;
+ await harness.mattermost.post({
+ channelId: harness.channel.id,
+ message: `@${harness.agent.name} ${text}`,
+ });
+
+ const deadline = Date.now() + replyDeadlineMs;
+ const latest = new Map<string, MattermostPost>();
+ let lastChangeMs = Date.now();
+ for (;;) {
+ for (const post of await harness.mattermost.postsFrom(
+ harness.channel.id,
+ harness.bot.id,
+ since
+ )) {
+ if (isNoSessionNotice(post)) {
+ throw new NoReplyError(`Switch says no session is attending the room: «${post.message}»`);
+ }
+ const before = latest.get(post.id);
+ if (before === undefined || before.message !== post.message) {
+ latest.set(post.id, post);
+ lastChangeMs = Date.now();
+ }
+ }
+ const posts = [...latest.values()].sort((a, b) => a.create_at - b.create_at);
+ const said = posts.filter((post) => !isRuntimeStatusPost(post));
+ const states = posts.map(turnStateOf).filter((state) => state !== null);
+ const ended =
+ states.length > 0 && states.every((state) => state !== 'queued' && state !== 'running');
+ if (states.includes('error')) {
+ throw new NoReplyError(
+ `The agent's turn ended with an error. It said: ${said.map((post) => `«${post.message}»`).join(' ') || 'nothing'}`
+ );
+ }
+ if (ended && said.length === 0) {
+ throw new NoReplyError(
+ `The agent's turn ended without a word in the room in answer to «${text}».`
+ );
+ }
+ if (
+ said.length > 0 &&
+ (ended || (states.length === 0 && Date.now() - lastChangeMs >= QUIET_MS))
+ ) {
+ return { from: 'agent', text: said.map((post) => post.message).join('\n\n') };
+ }
+ if (Date.now() >= deadline) {
+ throw new NoReplyError(
+ said.length === 0
+ ? `The agent did not reply within ${Math.round(replyDeadlineMs / 1000)} s to «${text}».`
+ : `The agent's turn had not ended after ${Math.round(replyDeadlineMs / 1000)} s.`
+ );
+ }
+ await sleep(POLL_MS);
+ }
+}
diff --git a/console/tooling-e2e/src/agentic/model.ts b/console/tooling-e2e/src/agentic/model.ts
new file mode 100644
index 000000000..a3361f487
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/model.ts
@@ -0,0 +1,140 @@
+import { spawn } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { parseDotEnv } from '../core/env.ts';
+
+/**
+ * One call to a pinned model, through the Claude Code CLI: the same binary,
+ * and the same credentials, the agent controller runs sessions with, so the
+ * lane needs no second way of reaching a model.
+ *
+ * The call is stripped to a plain completion: its own system prompt, no tools,
+ * no MCP servers, no settings, no saved session, from an empty directory. That
+ * keeps a developer's plugins, hooks and CLAUDE.md out of it, and keeps its
+ * cost to the tokens it is actually given (a default call carries Claude
+ * Code's whole system prompt, about seventy times the cost of a short reply).
+ *
+ * The model is never chosen here: a call names one, and an answer from any
+ * other model is refused rather than counted.
+ */
+
+export interface ModelAnswer {
+ text: string;
+ costUsd: number;
+}
+
+export class ModelError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'ModelError';
+ }
+}
+
+const CALL_TIMEOUT_MS = 3 * 60_000;
+
+export class ModelClient {
+ private readonly env: NodeJS.ProcessEnv;
+
+ /** `providerEnvFile`: keys to add to the CLI's environment, or null for its own sign-in. */
+ constructor(providerEnvFile: string | null) {
+ this.env = { ...process.env };
+ if (providerEnvFile !== null) {
+ Object.assign(this.env, parseDotEnv(readFileSync(providerEnvFile, 'utf8')));
+ }
+ }
+
+ async ask(params: {
+ model: string;
+ system: string;
+ prompt: string;
+ /** The most this one call may spend; the CLI stops it there. */
+ maxUsd: number;
+ }): Promise<ModelAnswer> {
+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-model-'));
+ try {
+ const raw = await run(
+ [
+ '-p',
+ '--model',
+ params.model,
+ '--system-prompt',
+ params.system,
+ '--tools',
+ '',
+ '--strict-mcp-config',
+ '--setting-sources',
+ '',
+ '--no-session-persistence',
+ '--max-budget-usd',
+ params.maxUsd.toFixed(4),
+ '--output-format',
+ 'json',
+ ],
+ params.prompt,
+ dir,
+ this.env
+ );
+ return parseAnswer(raw, params.model);
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }
+}
+
+/** The CLI's JSON result, held to the model that was asked for. */
+export function parseAnswer(raw: string, model: string): ModelAnswer {
+ let result: {
+ result?: unknown;
+ is_error?: unknown;
+ total_cost_usd?: unknown;
+ modelUsage?: Record<string, unknown>;
+ };
+ try {
+ result = JSON.parse(raw) as typeof result;
+ } catch {
+ throw new ModelError(`The Claude Code CLI did not answer with JSON: ${raw.slice(0, 300)}`);
+ }
+ const text = typeof result.result === 'string' ? result.result : '';
+ if (result.is_error === true) {
+ throw new ModelError(`The call to ${model} failed: ${text.slice(0, 300)}`);
+ }
+ const answered = Object.keys(result.modelUsage ?? {});
+ if (!answered.includes(model)) {
+ throw new ModelError(
+ `Asked ${model}, but the answer came from ${answered.join(', ') || 'no model'}. ` +
+ `The lane refuses a substitute: name a model these credentials can use.`
+ );
+ }
+ if (typeof result.total_cost_usd !== 'number') {
+ throw new ModelError(`The call to ${model} reported no cost.`);
+ }
+ return { text: text.trim(), costUsd: result.total_cost_usd };
+}
+
+function run(args: string[], stdin: string, cwd: string, env: NodeJS.ProcessEnv): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const child = spawn('claude', args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] });
+ let stdout = '';
+ let stderr = '';
+ const timer = setTimeout(() => {
+ child.kill('SIGKILL');
+ reject(
+ new ModelError(`The Claude Code CLI did not answer within ${CALL_TIMEOUT_MS / 1000} s.`)
+ );
+ }, CALL_TIMEOUT_MS);
+ child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString('utf8')));
+ child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8')));
+ child.on('error', (error) => {
+ clearTimeout(timer);
+ reject(new ModelError(`Could not run the Claude Code CLI ('claude'): ${error.message}`));
+ });
+ child.on('close', (code) => {
+ clearTimeout(timer);
+ // A failed call still prints its JSON result, which says why.
+ if (stdout.trim() !== '') resolve(stdout);
+ else reject(new ModelError(`The Claude Code CLI exited ${code}: ${stderr.slice(0, 300)}`));
+ });
+ child.stdin.end(stdin);
+ });
+}
diff --git a/console/tooling-e2e/src/agentic/roles.ts b/console/tooling-e2e/src/agentic/roles.ts
new file mode 100644
index 000000000..866880a31
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/roles.ts
@@ -0,0 +1,147 @@
+import type { ModelClient } from './model.ts';
+import type { AgenticScenario } from './scenario-file.ts';
+
+/**
+ * The two model roles around the agent under test.
+ *
+ * The driver plays the human: given the scenario and the conversation so far,
+ * it writes the human's next message, or says it is done. The judge reads the
+ * finished conversation and grades it against the scenario's rubric, one
+ * criterion at a time. Neither sees the other's prompt, and the judge never
+ * sees the persona: it grades what was said, not what was meant.
+ */
+
+/**
+ * One message in the conversation. `channel` is a message posted to the
+ * channel without addressing the agent (a scenario's setup messages): part of
+ * what the agent could read, not something it was asked.
+ */
+export interface Turn {
+ from: 'human' | 'agent' | 'channel';
+ text: string;
+}
+
+const LABEL: Record<Turn['from'], string> = {
+ human: 'human, to the agent',
+ agent: 'agent',
+ channel: 'channel, earlier, not addressed to the agent',
+};
+
+/** What the driver answers when the human has nothing more to say. */
+export const DONE = 'DONE';
+
+const DRIVER_SYSTEM = `You play a human in a team chat channel, talking to an AI agent that is a member of the channel. You are testing nothing and know nothing about tests: you are the person described, with the goal described.
+
+Write only the next message you would send: plain text, as you would type it in chat, with no quotation marks, no name prefix and no @mention (one is added for you). Keep it short and natural.
+
+If your goal is met, or the agent has made clear it cannot help further, reply with exactly ${DONE} and nothing else.`;
+
+const JUDGE_SYSTEM = `You grade a conversation between a human and an AI agent in a team chat channel against a list of criteria. Judge only what the transcript shows. Grade each criterion on its own: one failure does not fail the others. If the transcript does not show enough to decide a criterion, it fails.
+
+Answer with JSON only, no prose and no code fence, in exactly this shape:
+{"verdicts": [{"id": "<criterion id>", "pass": true, "reason": "<one sentence>"}]}
+with one entry per criterion, in the order given.`;
+
+export function transcriptText(turns: readonly Turn[]): string {
+ return turns.map((turn) => `[${LABEL[turn.from]}] ${turn.text}`).join('\n\n');
+}
+
+export async function driverTurn(
+ client: ModelClient,
+ model: string,
+ maxUsd: number,
+ scenario: AgenticScenario,
+ turns: readonly Turn[]
+): Promise<{ message: string | null; costUsd: number }> {
+ const humanTurns = turns.filter((turn) => turn.from === 'human').length;
+ const answer = await client.ask({
+ model,
+ maxUsd,
+ system: DRIVER_SYSTEM,
+ prompt: [
+ `Who you are:\n${scenario.persona}`,
+ `Your goal:\n${scenario.goal}`,
+ `The conversation so far:\n${transcriptText(turns)}`,
+ `You have sent ${humanTurns} of at most ${scenario.maxTurns} messages. Write your next message, or ${DONE}.`,
+ ].join('\n\n'),
+ });
+ const message = answer.text.replace(/^["“]|["”]$/g, '').trim();
+ return { message: message === DONE || message === '' ? null : message, costUsd: answer.costUsd };
+}
+
+export interface Verdict {
+ id: string;
+ pass: boolean;
+ reason: string;
+}
+
+export type Judgement =
+ | { ok: true; verdicts: Verdict[]; costUsd: number }
+ | { ok: false; error: string; raw: string; costUsd: number };
+
+export async function judge(
+ client: ModelClient,
+ model: string,
+ maxUsd: number,
+ scenario: AgenticScenario,
+ turns: readonly Turn[]
+): Promise<Judgement> {
+ const answer = await client.ask({
+ model,
+ maxUsd,
+ system: JUDGE_SYSTEM,
+ prompt: [
+ `What the human wanted:\n${scenario.goal}`,
+ `Criteria:\n${scenario.rubric.map((item) => `- ${item.id}: ${item.criterion}`).join('\n')}`,
+ `Transcript:\n${transcriptText(turns)}`,
+ ].join('\n\n'),
+ });
+ const parsed = parseVerdicts(
+ answer.text,
+ scenario.rubric.map((item) => item.id)
+ );
+ return parsed.ok
+ ? { ok: true, verdicts: parsed.verdicts, costUsd: answer.costUsd }
+ : { ok: false, error: parsed.error, raw: answer.text, costUsd: answer.costUsd };
+}
+
+/** The judge's verdicts, held to the rubric: one per criterion, each a boolean with a reason. */
+export function parseVerdicts(
+ raw: string,
+ ids: readonly string[]
+): { ok: true; verdicts: Verdict[] } | { ok: false; error: string } {
+ const start = raw.indexOf('{');
+ const end = raw.lastIndexOf('}');
+ if (start === -1 || end < start) return { ok: false, error: 'the judge answered without JSON' };
+ let value: unknown;
+ try {
+ value = JSON.parse(raw.slice(start, end + 1));
+ } catch (error) {
+ return { ok: false, error: `the judge's JSON does not parse: ${String(error)}` };
+ }
+ const list = (value as { verdicts?: unknown }).verdicts;
+ if (!Array.isArray(list)) return { ok: false, error: "the judge's JSON has no 'verdicts' list" };
+ const verdicts: Verdict[] = [];
+ for (const entry of list) {
+ const item = entry as Record<string, unknown> | null;
+ if (
+ item === null ||
+ typeof item.id !== 'string' ||
+ typeof item.pass !== 'boolean' ||
+ typeof item.reason !== 'string'
+ ) {
+ return { ok: false, error: `a verdict is not {id, pass, reason}: ${JSON.stringify(entry)}` };
+ }
+ verdicts.push({ id: item.id, pass: item.pass, reason: item.reason });
+ }
+ const given = verdicts.map((verdict) => verdict.id);
+ const missing = ids.filter((id) => !given.includes(id));
+ const extra = given.filter((id) => !ids.includes(id));
+ if (missing.length > 0 || extra.length > 0 || given.length !== ids.length) {
+ return {
+ ok: false,
+ error: `the verdicts do not match the rubric (missing: ${missing.join(', ') || 'none'}; unknown or repeated: ${extra.join(', ') || 'none'})`,
+ };
+ }
+ return { ok: true, verdicts: ids.map((id) => verdicts.find((verdict) => verdict.id === id)!) };
+}
diff --git a/console/tooling-e2e/src/agentic/scenario-file.ts b/console/tooling-e2e/src/agentic/scenario-file.ts
new file mode 100644
index 000000000..3667cf8e4
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/scenario-file.ts
@@ -0,0 +1,143 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parse } from 'yaml';
+import type { ScenarioHeader } from '../core/scenario.ts';
+
+/**
+ * An agentic scenario, as written in `agentic/<id>.yaml`.
+ *
+ * It says who the human is and what they want, how the conversation opens,
+ * and what a good conversation looks like. A model plays the human from it
+ * (the driver), the agent under test answers in a real session, and another
+ * model grades the transcript against the rubric (the judge). Adding a
+ * scenario is adding a file: no code.
+ *
+ * id: summarise-the-channel # kebab-case, the file's own name
+ * owner: platform # a team or role, never a person
+ * covers: [read_context] # as any scenario header
+ * persona: | # who the driver plays
+ * goal: | # what they want; the driver stops once they have it
+ * setup_messages: [...] # posted, unaddressed, before the opening (optional)
+ * opening: "…" # the first message, sent as written, addressed to the agent
+ * max_turns: 3 # human messages, the opening included
+ * rubric: # what the judge checks, each on its own
+ * - id: cites-the-decision
+ * criterion: The summary names the decision the team reached.
+ */
+export interface AgenticScenario {
+ header: ScenarioHeader;
+ persona: string;
+ goal: string;
+ setupMessages: string[];
+ opening: string;
+ maxTurns: number;
+ rubric: { id: string; criterion: string }[];
+}
+
+/** Where the scenario files live: `console/tooling-e2e/agentic/`. */
+export const AGENTIC_SCENARIO_DIR = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '..',
+ '..',
+ 'agentic'
+);
+
+const ID = /^[a-z0-9]+(-[a-z0-9]+)*$/;
+const KEYS = new Set([
+ 'id',
+ 'owner',
+ 'covers',
+ 'persona',
+ 'goal',
+ 'setup_messages',
+ 'opening',
+ 'max_turns',
+ 'rubric',
+]);
+
+/** Every scenario file, validated. A malformed one throws naming every problem in it. */
+export function loadAgenticScenarios(dir: string): AgenticScenario[] {
+ return readdirSync(dir)
+ .filter((file) => file.endsWith('.yaml'))
+ .sort()
+ .map((file) => parseScenario(file, readFileSync(path.join(dir, file), 'utf8')));
+}
+
+export function parseScenario(file: string, raw: string): AgenticScenario {
+ const value: unknown = parse(raw);
+ const problems: string[] = [];
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error(`${file}: not a mapping of scenario fields`);
+ }
+ const doc = value as Record<string, unknown>;
+ for (const key of Object.keys(doc)) {
+ if (!KEYS.has(key)) problems.push(`unknown field '${key}'`);
+ }
+ const text = (key: string): string => {
+ const field = doc[key];
+ if (typeof field !== 'string' || field.trim() === '') {
+ problems.push(`'${key}' must be non-empty text`);
+ return '';
+ }
+ return field.trim();
+ };
+ const texts = (key: string, required: boolean): string[] => {
+ const field = doc[key];
+ if (field === undefined && !required) return [];
+ if (!Array.isArray(field) || !field.every((item) => typeof item === 'string' && item !== '')) {
+ problems.push(`'${key}' must be a list of non-empty text`);
+ return [];
+ }
+ return field as string[];
+ };
+
+ const id = text('id');
+ if (id !== '' && !ID.test(id)) problems.push(`id '${id}' is not kebab-case`);
+ if (id !== '' && `${id}.yaml` !== file) problems.push(`id '${id}' does not match the file name`);
+ const maxTurns = doc.max_turns;
+ if (typeof maxTurns !== 'number' || !Number.isInteger(maxTurns) || maxTurns < 1 || maxTurns > 6) {
+ problems.push(`'max_turns' must be a whole number from 1 to 6`);
+ }
+ const rubric: { id: string; criterion: string }[] = [];
+ if (!Array.isArray(doc.rubric) || doc.rubric.length === 0) {
+ problems.push(`'rubric' must list at least one criterion`);
+ } else {
+ for (const entry of doc.rubric as unknown[]) {
+ const item = entry as Record<string, unknown> | null;
+ if (
+ item === null ||
+ typeof item.id !== 'string' ||
+ !ID.test(item.id) ||
+ typeof item.criterion !== 'string' ||
+ item.criterion.trim() === ''
+ ) {
+ problems.push(`each rubric entry needs a kebab-case 'id' and a 'criterion'`);
+ continue;
+ }
+ if (rubric.some((existing) => existing.id === item.id)) {
+ problems.push(`rubric id '${item.id}' is used twice`);
+ }
+ rubric.push({ id: item.id, criterion: item.criterion.trim() });
+ }
+ }
+
+ const scenario: AgenticScenario = {
+ header: {
+ id,
+ covers: texts('covers', true),
+ lane: 'agentic',
+ environments: ['ephemeral', 'local', 'dev', 'staging'],
+ owner: text('owner'),
+ requires: ['mattermost', 'agentic-models'],
+ },
+ persona: text('persona'),
+ goal: text('goal'),
+ setupMessages: texts('setup_messages', false),
+ opening: text('opening'),
+ maxTurns: typeof maxTurns === 'number' ? maxTurns : 0,
+ rubric,
+ };
+ if (problems.length > 0) throw new Error(`${file}:\n ${problems.join('\n ')}`);
+ return scenario;
+}
diff --git a/console/tooling-e2e/src/agentic/spend.ts b/console/tooling-e2e/src/agentic/spend.ts
new file mode 100644
index 000000000..319b710c9
--- /dev/null
+++ b/console/tooling-e2e/src/agentic/spend.ts
@@ -0,0 +1,145 @@
+import type { UsageTotal } from '../clients/switch-client.ts';
+
+/**
+ * What the agentic lane spends, and the limit it stops at.
+ *
+ * Two sources, both measured rather than guessed:
+ *
+ * - the driver and judge calls report their own cost (the Claude Code CLI
+ * prices each call);
+ * - the agent's session is priced from the tokens Switch itself records for
+ * it, per model (`GET /gateway/tenants/{tenant_id}/usage`), at the prices
+ * below. A model with no price here stops the run instead of counting as
+ * free.
+ */
+
+/** USD per million tokens. */
+export interface Price {
+ input: number;
+ output: number;
+ cacheRead: number;
+ cacheWrite: number;
+}
+
+/**
+ * List prices from Anthropic's public pricing page
+ * (platform.claude.com/docs/en/about-claude/pricing), read 2026-10-19. Cache
+ * writes are priced at the five-minute rate, the one Claude Code uses by
+ * default; Switch records cache writes without their duration.
+ */
+export const PRICES: Readonly<Record<string, Price>> = {
+ 'claude-sonnet-5': { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },
+ 'claude-sonnet-5-5': { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 },
+ 'claude-opus-5': { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 },
+ 'claude-opus-5-5': { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 },
+ // Claude Code's own background work (titles, summaries) runs on it.
+ 'claude-haiku-4-5': { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 },
+};
+
+const METRIC_TO_PRICE: Readonly<Record<string, keyof Price>> = {
+ input_tokens: 'input',
+ output_tokens: 'output',
+ cache_read_tokens: 'cacheRead',
+ cache_write_tokens: 'cacheWrite',
+};
+
+/**
+ * A model id as priced. A dated snapshot is priced as its model, however the
+ * provider spells the date: `claude-haiku-4-5-20251001` from Anthropic's API,
+ * `claude-haiku-4-5@20251001` from Vertex.
+ */
+export function priceOf(model: string): Price {
+ const price = PRICES[model] ?? PRICES[model.replace(/[-@]\d{8}$/, '')];
+ if (price === undefined) {
+ throw new Error(
+ `No price for model '${model}', so what it spent cannot be counted against the limit. ` +
+ `Add it to PRICES in src/agentic/spend.ts from the public pricing page.`
+ );
+ }
+ return price;
+}
+
+/** What one client spent, in USD, from its usage rows. Rows that are not tokens (turns, messages) cost nothing. */
+export function usageCostUsd(rows: readonly UsageTotal[]): number {
+ let usd = 0;
+ for (const row of rows) {
+ const field = METRIC_TO_PRICE[row.metric];
+ if (field === undefined) continue;
+ usd += (row.amount * priceOf(row.model)[field]) / 1_000_000;
+ }
+ return usd;
+}
+
+/** Each model's tokens, for the report: `claude-sonnet-5 12,345 in / 678 out / 90,000 cached`. */
+export function describeUsage(rows: readonly UsageTotal[]): string {
+ const byModel = new Map<string, Record<string, number>>();
+ for (const row of rows) {
+ // Turns and messages are counted without a model; they are not tokens.
+ if (METRIC_TO_PRICE[row.metric] === undefined) continue;
+ const counts = byModel.get(row.model) ?? {};
+ counts[row.metric] = (counts[row.metric] ?? 0) + row.amount;
+ byModel.set(row.model, counts);
+ }
+ if (byModel.size === 0) return 'no tokens recorded';
+ const n = (value: number | undefined) => (value ?? 0).toLocaleString('en-US');
+ return [...byModel]
+ .map(
+ ([model, c]) =>
+ `${model} ${n(c.input_tokens)} in / ${n(c.output_tokens)} out / ` +
+ `${n(c.cache_read_tokens)} cache read / ${n(c.cache_write_tokens)} cache write`
+ )
+ .join('; ');
+}
+
+export class BudgetExceededError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'BudgetExceededError';
+ }
+}
+
+/** The lane's running total against its limit. */
+export class Ledger {
+ readonly budgetUsd: number;
+ private harnessUsd = 0;
+ private agentUsd = 0;
+
+ constructor(budgetUsd: number) {
+ this.budgetUsd = budgetUsd;
+ }
+
+ /** A driver or judge call's cost. */
+ addHarness(usd: number): void {
+ this.harnessUsd += usd;
+ }
+
+ /** The agent's spend so far this run, as Switch last reported it. */
+ setAgent(usd: number): void {
+ this.agentUsd = usd;
+ }
+
+ get totalUsd(): number {
+ return this.harnessUsd + this.agentUsd;
+ }
+
+ get remainingUsd(): number {
+ return Math.max(0, this.budgetUsd - this.totalUsd);
+ }
+
+ /** Throw once the limit is reached: nothing more may be spent this run. */
+ assertWithin(what: string): void {
+ if (this.totalUsd >= this.budgetUsd) {
+ throw new BudgetExceededError(
+ `The spending limit of $${this.budgetUsd.toFixed(2)} is reached ($${this.totalUsd.toFixed(4)} spent) ` +
+ `before ${what}. Raise SWITCH_E2E_AGENTIC_BUDGET_USD only if that spend was expected.`
+ );
+ }
+ }
+
+ describe(): string {
+ return (
+ `$${this.totalUsd.toFixed(4)} of $${this.budgetUsd.toFixed(2)} ` +
+ `(agent $${this.agentUsd.toFixed(4)}, driver and judge $${this.harnessUsd.toFixed(4)})`
+ );
+ }
+}
diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.ts
index 81285a7f0..ee6549577 100644
--- a/console/tooling-e2e/src/clients/switch-client.ts
+++ b/console/tooling-e2e/src/clients/switch-client.ts
@@ -72,6 +72,15 @@ export interface AgentSummary {
* "attachments": [] } }
* ```
*/
+/** One row of `GET /gateway/tenants/{tenant_id}/usage`: an amount per metric, client and model. */
+export interface UsageTotal {
+ metric: string;
+ clientId: string;
+ clientName: string | null;
+ model: string;
+ amount: number;
+}
+
export interface AgentEvent {
type: string;
room_id?: string;
@@ -409,6 +418,44 @@ export class SwitchClient {
return rows.map((row) => ({ id: row.id, name: row.name, createdAt: row.created_at }));
}
+ /** `GET /gateway/auth/session` — the workspace this admin session is bound to. */
+ async tenantId(): Promise<string> {
+ const state = await this.gateway<{ state: string; tenant: { id: string } | null }>(
+ 'GET',
+ '/auth/session'
+ );
+ if (state.tenant === null) {
+ throw new Error(
+ `The gateway admin's session is bound to no workspace (state '${state.state}').`
+ );
+ }
+ return state.tenant.id;
+ }
+
+ /**
+ * `GET /gateway/tenants/{tenant_id}/usage` — what each client spent, per
+ * metric and model, over `[since, until)`. Counted in whole UTC hours.
+ */
+ async usage(tenantId: string, since: Date, until: Date): Promise<UsageTotal[]> {
+ const query = `since=${encodeURIComponent(since.toISOString())}&until=${encodeURIComponent(until.toISOString())}`;
+ const rows = await this.gateway<
+ {
+ metric: string;
+ client_id: string;
+ client_name: string | null;
+ model: string;
+ amount: number;
+ }[]
+ >('GET', `/tenants/${tenantId}/usage?${query}`);
+ return rows.map((row) => ({
+ metric: row.metric,
+ clientId: row.client_id,
+ clientName: row.client_name,
+ model: row.model,
+ amount: row.amount,
+ }));
+ }
+
/** The bridge id of the default collaboration bridge of `type`. */
async defaultBridgeId(type: string): Promise<string> {
const rows = await this.gateway<
diff --git a/console/tooling-e2e/src/core/core.test.ts b/console/tooling-e2e/src/core/core.test.ts
index d75afe182..76447934a 100644
--- a/console/tooling-e2e/src/core/core.test.ts
+++ b/console/tooling-e2e/src/core/core.test.ts
@@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from '../agentic/scenario-file.ts';
import { BROWSER_SCENARIOS } from '../browser/index.ts';
import { olderThanFrom, timestampMs } from '../cleanup.ts';
import { FAST_SCENARIOS } from '../fast/index.ts';
@@ -135,7 +136,8 @@ describe('scenario headers', () => {
const fast = FAST_SCENARIOS.map((scenario) => scenario.header);
const provider = PROVIDER_SCENARIOS.map((scenario) => scenario.header);
const browser = BROWSER_SCENARIOS.map((scenario) => scenario.header);
- expect(() => assertValidHeaders([...fast, ...provider, ...browser])).not.toThrow();
+ const agentic = loadAgenticScenarios(AGENTIC_SCENARIO_DIR).map((scenario) => scenario.header);
+ expect(() => assertValidHeaders([...fast, ...provider, ...browser, ...agentic])).not.toThrow();
expect(fast.every((h) => h.lane === 'fast')).toBe(true);
expect(provider.every((h) => h.lane === 'provider')).toBe(true);
expect(browser.every((h) => h.lane === 'browser')).toBe(true);
diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.ts
index 1ef5947f8..826064f35 100644
--- a/console/tooling-e2e/src/core/env.ts
+++ b/console/tooling-e2e/src/core/env.ts
@@ -18,12 +18,14 @@ import { fileURLToPath } from 'node:url';
* `provider` needs a real agent session attending the room, driven by Switch
* Console or its sidecar. `browser` drives the gateway UI in a headless
* Chromium against the same server: a smoke test of what an operator sees,
- * which no HTTP-level scenario can see break.
+ * which no HTTP-level scenario can see break. `agentic` runs the same real
+ * session as `provider`, but a model plays the human from a scenario file and
+ * another model judges the conversation.
*/
-export type Lane = 'fast' | 'provider' | 'browser';
+export type Lane = 'fast' | 'provider' | 'browser' | 'agentic';
/** Every lane: the one list anything that takes a lane by name checks against. */
-export const LANES: readonly Lane[] = ['fast', 'provider', 'browser'];
+export const LANES: readonly Lane[] = ['fast', 'provider', 'browser', 'agentic'];
export function isLane(raw: string): raw is Lane {
return (LANES as readonly string[]).includes(raw);
@@ -93,6 +95,26 @@ export interface ProviderEnv {
controllerId: string | null;
}
+/**
+ * Settings only the agentic lane reads. Each is null when unset, and the lane's
+ * scenarios then need it (`agentic-models`): a model is never picked for a run
+ * that did not name one, and a run never spends without a limit.
+ */
+export interface AgenticEnv {
+ /** Exact model id that plays the human. */
+ driverModel: string | null;
+ /** Exact model id that judges the conversation. */
+ judgeModel: string | null;
+ /** What the lane may spend, driver, judge and agent together, in USD. */
+ budgetUsd: number | null;
+ /**
+ * A file of provider keys (`ANTHROPIC_API_KEY=…`), the one the agent
+ * controller is handed too. Without it, the driver and judge use the
+ * Claude Code CLI's own sign-in.
+ */
+ providerEnvFile: string | null;
+}
+
export interface HarnessEnv {
lane: Lane;
environment: Environment;
@@ -119,6 +141,7 @@ export interface HarnessEnv {
/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */
mattermost: MattermostEnv | null;
provider: ProviderEnv;
+ agentic: AgenticEnv;
}
const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
@@ -274,6 +297,12 @@ export function loadEnv(
questionMode: questionModeFrom(get('SWITCH_E2E_QUESTION_MODE')),
controllerId: get('SWITCH_E2E_CONTROLLER_ID') ?? null,
},
+ agentic: {
+ driverModel: get('SWITCH_E2E_DRIVER_MODEL') ?? null,
+ judgeModel: get('SWITCH_E2E_JUDGE_MODEL') ?? null,
+ budgetUsd: budgetFrom(get('SWITCH_E2E_AGENTIC_BUDGET_USD')),
+ providerEnvFile: get('SWITCH_E2E_PROVIDER_ENV_FILE') ?? null,
+ },
};
}
@@ -314,6 +343,17 @@ function questionModeFrom(raw: string | undefined): QuestionMode {
throw new Error(`SWITCH_E2E_QUESTION_MODE must be 'numbered' or 'prose' — got '${raw}'.`);
}
+function budgetFrom(raw: string | undefined): number | null {
+ if (raw === undefined) return null;
+ const budget = Number(raw);
+ if (!Number.isFinite(budget) || budget <= 0) {
+ throw new Error(
+ `SWITCH_E2E_AGENTIC_BUDGET_USD must be a positive number of dollars — got '${raw}'.`
+ );
+ }
+ return budget;
+}
+
function optionalUrl(raw: string | undefined): string | null {
return raw === undefined ? null : stripTrailingSlash(raw);
}
diff --git a/console/tooling-e2e/src/core/scenario.ts b/console/tooling-e2e/src/core/scenario.ts
index 6fe07c63f..1ee572a3d 100644
--- a/console/tooling-e2e/src/core/scenario.ts
+++ b/console/tooling-e2e/src/core/scenario.ts
@@ -38,8 +38,9 @@ export interface ScenarioHeader {
* `mattermost`: a Mattermost and a human token on it are configured.
* `agent-dir`: the provider session's working directory is on this machine.
* `gateway-ui`: the address of a gateway UI served against this Switch server.
+ * `agentic-models`: the driver and judge models, and the spending limit.
*/
-export type Prerequisite = 'mattermost' | 'agent-dir' | 'gateway-ui';
+export type Prerequisite = 'mattermost' | 'agent-dir' | 'gateway-ui' | 'agentic-models';
const OPERATION = /^[a-z][a-z0-9_]*$/;
const ROUTE = /^(GET|POST|PUT|PATCH|DELETE) \/\S*$/;
@@ -57,8 +58,11 @@ export function headerProblems(header: ScenarioHeader): string[] {
}
if (header.environments.length === 0) problems.push('environments is empty');
if (header.owner.trim() === '') problems.push('owner is empty');
- if (header.lane === 'provider' && header.environments.includes('prod')) {
- problems.push('a provider scenario writes to the room, so it cannot list prod');
+ if (
+ (header.lane === 'provider' || header.lane === 'agentic') &&
+ header.environments.includes('prod')
+ ) {
+ problems.push(`a ${header.lane} scenario writes to the room, so it cannot list prod`);
}
return problems;
}
@@ -117,6 +121,12 @@ function satisfied(prerequisite: Prerequisite, env: HarnessEnv): boolean {
return env.provider.agentRepoDir !== null;
case 'gateway-ui':
return env.gatewayUiUrl !== null;
+ case 'agentic-models':
+ return (
+ env.agentic.driverModel !== null &&
+ env.agentic.judgeModel !== null &&
+ env.agentic.budgetUsd !== null
+ );
}
}
@@ -128,5 +138,7 @@ function describePrerequisite(prerequisite: Prerequisite): string {
return 'the session working directory (SWITCH_E2E_AGENT_DIR)';
case 'gateway-ui':
return 'a gateway UI (SWITCH_GATEWAY_UI_URL)';
+ case 'agentic-models':
+ return 'the driver and judge models and a spending limit (SWITCH_E2E_DRIVER_MODEL, SWITCH_E2E_JUDGE_MODEL, SWITCH_E2E_AGENTIC_BUDGET_USD)';
}
}
diff --git a/console/tooling-e2e/src/gate/coverage.test.ts b/console/tooling-e2e/src/gate/coverage.test.ts
index 2c0a468e0..d032f8003 100644
--- a/console/tooling-e2e/src/gate/coverage.test.ts
+++ b/console/tooling-e2e/src/gate/coverage.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest';
+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from '../agentic/scenario-file.ts';
import { BROWSER_SCENARIOS } from '../browser/index.ts';
import type { ScenarioHeader } from '../core/scenario.ts';
import { FAST_SCENARIOS } from '../fast/index.ts';
@@ -11,9 +12,12 @@ import {
type ServerSurface,
} from './gate.ts';
-const HEADERS = [...FAST_SCENARIOS, ...PROVIDER_SCENARIOS, ...BROWSER_SCENARIOS].map(
- (scenario) => scenario.header
-);
+const HEADERS = [
+ ...FAST_SCENARIOS,
+ ...PROVIDER_SCENARIOS,
+ ...BROWSER_SCENARIOS,
+ ...loadAgenticScenarios(AGENTIC_SCENARIO_DIR),
+].map((scenario) => scenario.header);
describe('coverage gate', () => {
it('accounts for every operation and gateway route the server has', () => {
diff --git a/console/tooling-e2e/src/provider/scenarios.test.ts b/console/tooling-e2e/src/provider/scenarios.test.ts
index 0120badd6..358b535b4 100644
--- a/console/tooling-e2e/src/provider/scenarios.test.ts
+++ b/console/tooling-e2e/src/provider/scenarios.test.ts
@@ -1,5 +1,12 @@
import { describe, expect, it } from 'vitest';
-import { isStartMarker, optionNumber, requestHandle } from './scenarios.ts';
+import type { MattermostPost } from '../clients/mattermost-client.ts';
+import {
+ isRuntimeStatusPost,
+ isStartMarker,
+ optionNumber,
+ requestHandle,
+ turnStateOf,
+} from './scenarios.ts';
describe('interruption admission', () => {
it('requires the command marker, not an acknowledgement or refusal', () => {
@@ -11,6 +18,27 @@ describe('interruption admission', () => {
});
});
+describe('turn status cards', () => {
+ const post = (message: string) => ({ message }) as MattermostPost;
+
+ it('reads where the turn got to from the heading, and treats the card as Switch talking', () => {
+ expect(turnStateOf(post('**Working… 12s** · [Console](https://example.invalid)'))).toBe(
+ 'running'
+ );
+ expect(turnStateOf(post('**Turn complete.**'))).toBe('completed');
+ expect(turnStateOf(post('**Worked for 1m 4s. 3 tool calls.**'))).toBe('completed');
+ expect(turnStateOf(post('**Turn interrupted. Worked for 9s.**'))).toBe('interrupted');
+ expect(turnStateOf(post('**Turn ended with an error.**\nsomething'))).toBe('error');
+ expect(isRuntimeStatusPost(post('**Working… 0s**'))).toBe(true);
+ });
+
+ it("leaves the agent's own words alone, bold or not", () => {
+ expect(turnStateOf(post('Working on the date bug now.'))).toBe(null);
+ expect(turnStateOf(post('**Note:** the turn is complete.'))).toBe(null);
+ expect(isRuntimeStatusPost(post('Here is the plan.'))).toBe(false);
+ });
+});
+
describe('request cards', () => {
it('reads the handle a card answers to', () => {
expect(requestHandle('@user **Questions** · request `A1` Question from the agent 1. red')).toBe(
diff --git a/console/tooling-e2e/src/provider/scenarios.ts b/console/tooling-e2e/src/provider/scenarios.ts
index f59d1924e..fa73f4e96 100644
--- a/console/tooling-e2e/src/provider/scenarios.ts
+++ b/console/tooling-e2e/src/provider/scenarios.ts
@@ -76,8 +76,32 @@ const RUNTIME_STATUS_MARKERS = [
'interrupted my current turn',
];
+/**
+ * Where a turn has got to, as the session renderer heads its status card with
+ * it: one post per turn, edited in place from `Working…` to how the turn ended
+ * (`TURN_STATE` in core/switch_core/bridges/collaboration/session/renderers).
+ */
+export type TurnState = 'queued' | 'running' | 'completed' | 'interrupted' | 'error';
+
+const TURN_STATE_HEADINGS: readonly [TurnState, string][] = [
+ ['queued', 'Queued.'],
+ ['running', 'Working…'],
+ ['completed', 'Turn complete.'],
+ // A completed turn whose duration is known says how long it ran instead.
+ ['completed', 'Worked for '],
+ ['interrupted', 'Turn interrupted.'],
+ ['error', 'Turn ended with an error.'],
+];
+
+/** The state a turn status card reports, or null when the post is not one. */
+export function turnStateOf(post: MattermostPost): TurnState | null {
+ const heading = /^\*\*([^*]+)\*\*/.exec(post.message.trimStart())?.[1] ?? '';
+ return TURN_STATE_HEADINGS.find(([, text]) => heading.startsWith(text))?.[0] ?? null;
+}
+
/** Whether a post is Switch narrating the session rather than the agent speaking. */
export function isRuntimeStatusPost(post: MattermostPost): boolean {
+ if (turnStateOf(post) !== null) return true;
const text = post.message.toLowerCase();
return RUNTIME_STATUS_MARKERS.some((marker) => text.includes(marker));
}
diff --git a/justfile b/justfile
index 78dfe49bb..499d5531b 100644
--- a/justfile
+++ b/justfile
@@ -183,6 +183,14 @@ e2e-provider *args:
e2e-browser *args:
SWITCH_E2E_LANE=browser console/tooling-e2e/scripts/ephemeral-stack.sh {{ args }}
+# ── End-to-end harness, agentic lane, on a throwaway stack ─────────────────────
+# As e2e-provider, then the scenario files in console/tooling-e2e/agentic: a
+# driver model plays the human, a judge model grades the conversation. Needs
+# SWITCH_E2E_DRIVER_MODEL, SWITCH_E2E_JUDGE_MODEL (exact model ids) and
+# SWITCH_E2E_AGENTIC_BUDGET_USD. Spends real model calls, up to that limit.
+e2e-agentic *args:
+ SWITCH_E2E_LANE=agentic console/tooling-e2e/scripts/ephemeral-stack.sh {{ args }}
+
# ── Regenerate the server surface the E2E coverage gate reads ─────────────────
# Every agent operation and gateway route, written to
# console/tooling-e2e/surface/server-surface.json. CI fails when it is stale;