Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Agentic lane: a model plays the human, another grades the agent's replies
Petr Bauchb7dc69f84dd4
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex b781cf34c..2f3cbd9ee 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -7,6 +7,8 @@ name: E2E## lane fast provider-free scenarios over plain HTTP# provider real agent sessions on a headless agent controller+# agentic the provider lane's setup, driven by scenario files: a+# model plays the human, another judges the result# browser the gateway UI in headless Chromium; on ephemeral the# UI is built from this checkout# environment ephemeral a fresh stack on the runner, built from this checkout@@ -53,7 +55,7 @@ on:inputs:lane:type: choice- options: [fast, provider, browser]+ options: [fast, provider, browser, agentic]default: fastenvironment:type: choice@@ -73,7 +75,7 @@ jobs:name: ${{ inputs.lane }} lane on ${{ inputs.environment }}runs-on: ubuntu-latesttimeout-minutes: 45- environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || (inputs.lane == 'provider' && 'e2e-provider' || '') }}+ environment: ${{ inputs.environment != 'ephemeral' && inputs.environment || ((inputs.lane == 'provider' || inputs.lane == 'agentic') && 'e2e-provider' || '') }}# One run at a time against a shared deployment, queued rather than# cancelled: a cancelled run leaves its agents and rooms behind. A throwaway# stack is the runner's own, so ephemeral runs never wait on each other.@@ -101,10 +103,11 @@ jobs:run: |case "$LANE" in fast|provider|browser) ;; *) echo "::error::Unknown lane '$LANE'."; exit 1 ;; esaccase "$TARGET" in ephemeral|dev|staging) ;; *) echo "::error::Unknown environment '$TARGET'."; exit 1 ;; esac- if [ "$LANE" = provider ] && [ "$TARGET" != ephemeral ]; then- echo "::error::The provider lane runs on an ephemeral stack only."; exit 1+ case "$LANE" in provider|agentic) REAL_SESSION=1 ;; *) REAL_SESSION=0 ;; esac+ if [ "$REAL_SESSION" = 1 ] && [ "$TARGET" != ephemeral ]; then+ echo "::error::The $LANE lane runs on an ephemeral stack only."; exit 1fi- if [ "$LANE" = provider ] && [ "$AGENT_TYPE" != claude-code ]; then+ if [ "$REAL_SESSION" = 1 ] && [ "$AGENT_TYPE" != claude-code ]; thenecho "::error::CI has credentials for claude-code only, not '$AGENT_TYPE'."; exit 1fi@@ -137,12 +140,12 @@ jobs:run: pnpm exec playwright install --with-deps chromium- name: Install Claude Code- if: ${{ inputs.lane == 'provider' }}+ if: ${{ inputs.lane == 'provider' || inputs.lane == 'agentic' }}run: npm install -g @anthropic-ai/[email protected]# Into the controller's environment through --env-file; never into $OUT.- name: Write the provider key file- if: ${{ inputs.lane == 'provider' }}+ if: ${{ inputs.lane == 'provider' || inputs.lane == 'agentic' }}env:ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}run: |@@ -159,6 +162,12 @@ jobs:SWITCH_E2E_LANE: ${{ inputs.lane }}SWITCH_E2E_AGENT_TYPE: ${{ inputs.agent_type }}E2E_OUT: ${{ env.OUT }}+ # The agentic lane's models, by exact id, and what one run may spend+ # (driver, judge and agent together). Pinned here, so changing them+ # is a reviewed change; a model the key cannot use fails the run.+ SWITCH_E2E_DRIVER_MODEL: claude-sonnet-5+ SWITCH_E2E_JUDGE_MODEL: claude-sonnet-5+ SWITCH_E2E_AGENTIC_BUDGET_USD: '2'run: console/tooling-e2e/scripts/ephemeral-stack.sh- name: Run the lane against ${{ inputs.environment }}diff --git a/console/pnpm-lock.yaml b/console/pnpm-lock.yamlindex b45355452..cb954d8db 100644--- a/console/pnpm-lock.yaml+++ b/console/pnpm-lock.yaml@@ -488,6 +488,9 @@ importers:vitest:specifier: ^5.0.1version: 5.0.1(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])(@vitest/[email protected])([email protected](@noble/[email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))+ yaml:+ specifier: 2.9.0+ version: 2.9.0packages:@@ -11073,8 +11076,7 @@ snapshots:- optional: true+ [email protected]: {}diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex c7729b940..563159ceb 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -2,7 +2,7 @@End-to-end tests for Switch: **real messages through a real Switch server**, andthrough a real Mattermost bridge where a scenario crosses one. One package,-three lanes:+four lanes:- **The fast lane** needs no provider, no Switch Console and no LLM key. Theharness plays every participant over plain HTTP: it registers throwaway@@ -16,6 +16,10 @@ three lanes:same server: signing in, every page in the navigation rail, a room openedfrom the list, signing out. A page that renders but is broken (an uncaughterror, a request the server failed, an error banner) fails it.+- **The agentic lane** runs the provider lane's real session, but the+ conversations come from scenario files: a model plays the human, and another+ model grades the result against a rubric. Adding a scenario is adding a+ file. See "The agentic lane" below.Nothing here talks to Switch Console directly. If a scenario passes, an operatordoing the same thing by hand would have seen the same result.@@ -76,6 +80,8 @@ Typecheck: `pnpm typecheck`.| `src/fast/` | Fast-lane scenarios, one file per area, listed in `index.ts` || `src/provider/` | The provider lane: Console credential files, the shared agent and channel, the scenarios, `seed.ts` || `src/browser/` | The browser lane: the gateway UI driver (`gateway-ui.ts`) and the scenarios, listed in `index.ts` |+| `src/agentic/` | The agentic lane: scenario files' format, the driver and judge, model calls, spend |+| `agentic/` | The agentic scenarios, one YAML file each || `scripts/ephemeral-stack.sh` | The throwaway-stack runner behind `just e2e-fast`, `just e2e-provider` and `just e2e-browser` || `scripts/scan-artifacts.sh` | Fails if a run's output holds one of the secrets the run used || `src/ci-summary.ts` | The CI run summary, read from the JUnit report |@@ -192,6 +198,9 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. || `SWITCH_GATEWAY_UI_URL` | browser lane | Where the gateway UI is served. Its `/gateway` requests must reach `SWITCH_API_URL`. `just e2e-browser` sets it. || `SWITCH_E2E_OUTPUT_DIR` | no | Where the browser lane saves screenshots of failed pages. The stack script and `e2e.yml` set it. |+| `SWITCH_E2E_DRIVER_MODEL` | agentic lane | Exact model id that plays the human, e.g. `claude-sonnet-5`. |+| `SWITCH_E2E_JUDGE_MODEL` | agentic lane | Exact model id that grades the conversation. |+| `SWITCH_E2E_AGENTIC_BUDGET_USD` | agentic lane | What one run may spend, driver, judge and agent together. The run stops once it is reached. || `SWITCH_E2E_AGENT_DIR` | provider lane | Working directory of the session under test. When set, the agent's credentials are written into it (see below) and it is sent to Switch as the agent's `repo_dir`. || `SWITCH_E2E_MANIFEST` | provider lane | File the setup records its agent and room in, so a second process can reuse them. See "Seeding first". || `SWITCH_E2E_REPLY_TIMEOUT_MS` | provider lane | How long a scenario waits for the agent (default 5 min). |@@ -199,7 +208,7 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.| `SWITCH_E2E_AGENT_TYPE` | provider lane | `opencode` (default), `claude-code`, `codex`, `antigravity` or `cursor` — which provider's runtime is under test. || `SWITCH_E2E_QUESTION_MODE` | provider lane | `numbered` (default) or `prose`; see the `question` row below. || `SWITCH_E2E_CONTROLLER_ID` | provider lane | An agent controller to run the agent on. `just e2e-provider` sets it. Unset, a Switch Console session is expected instead. |-| `SWITCH_E2E_PROVIDER_ENV_FILE` | `just e2e-provider` | A file of provider keys, handed to the controller as `--env-file`. Leave it unset to use the CLI's own sign-in. |+| `SWITCH_E2E_PROVIDER_ENV_FILE` | `just e2e-provider`, `just e2e-agentic` | A file of provider keys, handed to the controller as `--env-file`, and to the agentic lane's driver and judge. Leave it unset to use the CLI's own sign-in. |Mattermost is all or nothing: none of its three variables means "no Mattermosthere"; some of them is reported as a mistake.@@ -251,6 +260,69 @@ Two things differ from a Console session:The controller's log goes to `build/e2e/ephemeral/controller.log`.+## The agentic lane++`just e2e-agentic` sets up what `just e2e-provider` does (one agent on a+headless controller, one Mattermost channel), then plays each file in+`agentic/` against it:++```yaml+id: summarise-the-channel # kebab-case, the file's own name+owner: platform+covers: [read_context, post_message]+persona: | # who the driver plays+ A team member back from a day off, catching up.+goal: | # what they want; the driver stops once they have it+ Learn what was decided about the weekly sync and who is doing what next.+setup_messages: # posted first, not addressed to the agent (optional)+ - Should we move the weekly sync to Thursday afternoons?+opening: Just got back. What was decided above about the weekly sync?+max_turns: 2 # human messages, the opening included (1 to 6)+rubric: # what the judge checks, each on its own+ - id: names-the-decision+ criterion: The agent says the sync moves to Wednesday at 10:00.+```++- **The driver** is a model playing the human. After each reply it writes the+ next message from the persona, the goal and the conversation so far, or+ stops once the goal is met. The opening is sent as written, so every run+ starts the same way.+- **The agent** answers in its real session. A reply is finished when Switch's+ status card for the turn says the turn ended.+- **The judge** is a second model. It reads the transcript, including the+ setup messages, and grades each rubric criterion with a reason. It never+ sees the persona, so it grades what was said, not what was meant.++What fails a scenario is what can be decided without a model: the agent did+not reply, its turn ended in an error, the server logged an error, or the run+reached its spending limit. **The judge's verdicts do not fail a scenario.**+They are printed per criterion and written, with the transcript and the spend,+to `agentic/<id>.json` in the output directory. A model's grade is evidence to+read until it has a record of agreeing with people.++Both models are named by exact id, and an answer from any other model is+refused rather than counted. Their calls go through the Claude Code CLI with+its own system prompt, tools, MCP servers and settings all switched off, from+an empty directory, so a developer's setup cannot leak in.++**Spend** is measured, not estimated from a guess:++- the driver and judge calls report their own cost;+- the agent's session is priced from the tokens Switch records for it per+ model (`GET /gateway/tenants/{tenant_id}/usage`), at the list prices in+ `src/agentic/spend.ts`. A model with no price there stops the run rather+ than counting as free.++The run stops before any call that would start beyond+`SWITCH_E2E_AGENTIC_BUDGET_USD`, and its last test fails if the total went+over. CI pins the models and a $2 limit in `e2e.yml`.++What a run costs, measured over four runs of the three scenarios here with+Sonnet 5 in every role: $0.21 to $0.33 a run, about $0.08 a scenario. The+agent's session is nearly nine tenths of it ($0.045 to $0.21 a scenario,+most where it calls tools); the driver and judge together are about $0.01 a+scenario.+## Server health and errorsA scenario can pass every assertion while the server logs an error and carries@@ -342,6 +414,7 @@ dispatched by hand.| Every PR, and every push to main (`pr-ci.yml`) | fast | ephemeral | yes || After a merge to main (`e2e-scheduled.yml`) | provider | ephemeral | no || Nightly (`e2e-scheduled.yml`) | provider; fast; browser | ephemeral; dev; dev | no |+| By hand only, for now (`e2e.yml`, dispatch) | agentic | ephemeral | no |- **Secrets.** An ephemeral fast run generates every secret it uses on therunner, so it works the same on a fork PR. Every other run reads its secretsdiff --git a/console/tooling-e2e/agentic/clarify-before-acting.yaml b/console/tooling-e2e/agentic/clarify-before-acting.yamlnew file mode 100644index 000000000..4c701142b--- /dev/null+++ b/console/tooling-e2e/agentic/clarify-before-acting.yaml@@ -0,0 +1,23 @@+# An ambiguous request: the agent should ask what is meant rather than guess,+# then act on the answer, and never claim work it did not do.+id: clarify-before-acting+owner: platform+covers: [post_message]+persona: |+ An engineering manager with a release tomorrow. Writes terse messages and+ assumes others know what "the date thing" is. When asked, explains: the+ invoice page shows dates in UTC, but customers should see their own local+ time. Wants a short plan of how to fix it, not code, and not a fix made+ for them.+goal: |+ Get a short plan (a few steps) for showing invoice dates in the customer's+ local time instead of UTC. The goal is met once there is such a plan.+opening: Can you sort out the date thing before the release?+max_turns: 3+rubric:+ - id: asks-before-acting+ criterion: Before proposing any specific fix, the agent asks what "the date thing" refers to, instead of assuming.+ - id: plan-fits-the-answer+ criterion: After the human explains, the agent gives a short plan for showing invoice dates in the customer's local time rather than UTC.+ - id: claims-nothing-undone+ criterion: The agent never claims to have changed code, opened a pull request or shipped a fix.diff --git a/console/tooling-e2e/agentic/explain-then-follow-up.yaml b/console/tooling-e2e/agentic/explain-then-follow-up.yamlnew file mode 100644index 000000000..e41c33d0e--- /dev/null+++ b/console/tooling-e2e/agentic/explain-then-follow-up.yaml@@ -0,0 +1,21 @@+# A short explanation, then a follow-up that only makes sense with it: the+# agent keeps the thread of a conversation across turns in the room.+id: explain-then-follow-up+owner: platform+covers: [post_message]+persona: |+ A backend developer, two weeks into the team, who has heard the term+ "race condition" in a code review and wants to understand it quickly.+ Friendly, a little hurried, types short messages.+goal: |+ Understand what a race condition is, then see one tiny concrete example in+ Python. Once there is an example that makes sense, the goal is met.+opening: Quick one, what's a race condition? Two or three sentences is plenty.+max_turns: 3+rubric:+ - id: answers-briefly+ criterion: The first reply explains what a race condition is, correctly, in a few sentences rather than an essay.+ - id: example-follows-on+ criterion: A later reply gives a short Python example that illustrates a race condition, consistent with the earlier explanation.+ - id: answers-in-the-room+ criterion: Every reply addresses the human's question directly, without saying it cannot see the message or asking them to repeat it.diff --git a/console/tooling-e2e/agentic/summarise-the-channel.yaml b/console/tooling-e2e/agentic/summarise-the-channel.yamlnew file mode 100644index 000000000..80a1e5d35--- /dev/null+++ b/console/tooling-e2e/agentic/summarise-the-channel.yaml@@ -0,0 +1,24 @@+# A summary of a discussion the agent was not addressed in: it has to read the+# room's history to answer, and must not invent what was not said.+id: summarise-the-channel+owner: platform+covers: [read_context, post_message]+persona: |+ A team member back from a day off, catching up. Wants the outcome of a+ discussion in the channel, not a retelling of it.+goal: |+ Learn what was decided about the weekly sync and who is doing what next.+ The goal is met once the summary covers both.+setup_messages:+ - Should we move the weekly sync to Thursday afternoons?+ - Thursday clashes with the release review. How about Wednesday at 10:00 instead?+ - Wednesday 10:00 works for everyone. Let's start next week, and the team lead will update the calendar invite.+opening: Just got back. What was decided above about the weekly sync?+max_turns: 2+rubric:+ - id: names-the-decision+ criterion: The agent says the sync moves to Wednesday at 10:00, starting next week.+ - id: names-the-follow-up+ criterion: The agent says the team lead will update the calendar invite.+ - id: invents-nothing+ criterion: The agent adds no decisions, dates, people or reasons that the earlier messages did not contain.diff --git a/console/tooling-e2e/package.json b/console/tooling-e2e/package.jsonindex 3443bdcf3..bca6e1544 100644--- a/console/tooling-e2e/package.json+++ b/console/tooling-e2e/package.json@@ -13,6 +13,7 @@"devDependencies": {"playwright": "1.60.0","typescript": "^5.9.2",- "vitest": "^5.0.1"+ "vitest": "^5.0.1",+ "yaml": "2.9.0"}}diff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.shindex 169f436d2..1d4f04b24 100755--- a/console/tooling-e2e/scripts/ephemeral-stack.sh+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh@@ -6,6 +6,7 @@# just e2e-fast the fast lane (no provider, no Console)# just e2e-provider the provider lane, on a headless agent controller# just e2e-browser the browser lane: the gateway UI in headless Chromium+# just e2e-agentic the agentic lane: scenario files, a driver and a judge model# just e2e-fast --keep leave the stack running afterwards# just e2e-fast -- <vitest args> pass arguments through to vitest#@@ -36,6 +37,10 @@ REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"HARNESS="$REPO/console/tooling-e2e"OUT="${E2E_OUT:-$REPO/build/e2e/ephemeral}"LANE="${SWITCH_E2E_LANE:-fast}"+# The agentic lane runs on the provider lane's setup: a real session on an+# agent controller.+NEEDS_CONTROLLER=0+case "$LANE" in provider|agentic) NEEDS_CONTROLLER=1 ;; esacKEEP=0VITEST_ARGS=()while [ $# -gt 0 ]; do@@ -99,7 +104,7 @@ trap cleanup EXITcp "$REPO/.env.example" "$ENV_FILE""$REPO/deploy/local/generate-env-secrets.sh" "$ENV_FILE" e2e-if [ "$LANE" = provider ]; then+if [ "$NEEDS_CONTROLLER" = 1 ]; then# Agent management, and the key controllers' access tokens are signed with.sed -i.bak -e "s|^FEATURE_FLAGS_ENABLED=.*|FEATURE_FLAGS_ENABLED=agent_management|" \-e "s|^CONTROLLER_TOKEN_SECRET=.*|CONTROLLER_TOKEN_SECRET=$(openssl rand -hex 32)|" "$ENV_FILE"@@ -171,7 +176,7 @@ MATTERMOST_TOKEN="$(curl -sf -D - -o /dev/null -H 'Content-Type: application/jsoremember_secrets "$MATTERMOST_TOKEN"PROVIDER_ENV=()-if [ "$LANE" = provider ]; then+if [ "$NEEDS_CONTROLLER" = 1 ]; thenecho "Building the agent controller…"(cd "$REPO/console" && pnpm -r --filter './packages/**' run build) >"$OUT/console-build.log" 2>&1 \|| { echo "console package build failed; see $OUT/console-build.log" >&2; exit 1; }@@ -235,7 +240,7 @@ if [ "$LANE" = browser ]; thenfor _ in $(seq 30); do curl -sf localhost:4173/ >/dev/null && break; sleep 1; donecurl -sf localhost:4173/ >/dev/null \|| { echo "the gateway UI did not come up; see $OUT/gateway-ui.log" >&2; exit 1; }- BROWSER_ENV=(SWITCH_GATEWAY_UI_URL=http://127.0.0.1:4173 SWITCH_E2E_OUTPUT_DIR="$OUT")+ BROWSER_ENV=(SWITCH_GATEWAY_UI_URL=http://127.0.0.1:4173)fiecho "Running the $LANE lane…"@@ -248,6 +253,8 @@ status=0export SWITCH_E2E_SECRETS_FILE="$SECRETS"# What the server logs, so a scenario during which it logs an error fails.export SWITCH_E2E_SERVER_LOG="$OUT/switch-core.log"+ # Screenshots of failed pages, and agentic transcripts and verdicts.+ export SWITCH_E2E_OUTPUT_DIR="$OUT"SWITCH_E2E_LANE="$LANE" \SWITCH_E2E_ENVIRONMENT=ephemeral \SWITCH_E2E_STRICT=1 \@@ -264,7 +271,7 @@ status=0if [ "$status" != 0 ]; thenecho "The $LANE lane failed. switch-core log: $OUT/switch-core.log" >&2- [ "$LANE" = provider ] && echo "Agent controller log: $OUT/controller.log" >&2+ [ "$NEEDS_CONTROLLER" = 1 ] && echo "Agent controller log: $OUT/controller.log" >&2[ "$LANE" = browser ] && echo "Screenshots of failed pages: $OUT/screenshots" >&2fiexit "$status"diff --git a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts b/console/tooling-e2e/src/agentic/agentic.e2e.test.tsnew file mode 100644index 000000000..1d6919325--- /dev/null+++ b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts@@ -0,0 +1,311 @@+import { mkdirSync, writeFileSync } from 'node:fs';+import path from 'node:path';+import { afterAll, beforeAll, describe, expect, it } from 'vitest';+import { sleep } from '../clients/mattermost-client.ts';+import type { UsageTotal } from '../clients/switch-client.ts';+import { describeEnv } from '../core/env.ts';+import { RunContext } from '../core/run.ts';+import { admit, assertValidHeaders } from '../core/scenario.ts';+import {+ describeErrors,+ describeServerLogCheck,+ expectNoServerErrors,+ ServerLogWatch,+} from '../core/server-log.ts';+import { resolveLane, type LaneResolution } from '../core/suite.ts';+import {+ setupHarness,+ teardownHarness,+ waitForSession,+ type Harness,+} from '../provider/harness.ts';+import { exchange, postUnaddressed } from './conversation.ts';+import { ModelClient } from './model.ts';+import { driverTurn, judge, type Judgement, type Turn } from './roles.ts';+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from './scenario-file.ts';+import { describeUsage, Ledger, usageCostUsd } from './spend.ts';++/**+ * The agentic lane: scenario files played against a real agent session.+ * Runs when `SWITCH_E2E_LANE=agentic`, on the same setup as the provider lane+ * (one agent on a headless controller, one Mattermost channel).+ *+ * For each scenario a driver model plays the human, from the scenario's+ * persona and goal, until the goal is met or `max_turns` runs out; then a+ * judge model grades the transcript against the rubric.+ *+ * What fails a scenario is what can be decided without a model: the agent did+ * not answer, the server logged an error, or the run reached its spending+ * limit. The judge's verdicts are reported, per criterion, in the run's+ * output and in `agentic/<id>.json` under SWITCH_E2E_OUTPUT_DIR, but do not+ * fail the scenario: a model's grade is evidence to read, not a gate, until+ * it has a record of agreeing with people.+ */++const SCENARIOS = loadAgenticScenarios(AGENTIC_SCENARIO_DIR);+assertValidHeaders(SCENARIOS.map((scenario) => scenario.header));++const REPLY_DEADLINE_MS = Number(process.env.SWITCH_E2E_REPLY_TIMEOUT_MS ?? 5 * 60_000);+/** The most a single driver or judge call may spend. */+const CALL_LIMIT_USD = 0.5;+/** How long to wait for Switch's usage count to stop moving after a conversation. */+const USAGE_SETTLE_MS = 30_000;++interface ScenarioReport {+ id: string;+ turns: Turn[];+ failure: string | null;+ judgement: Judgement | null;+ agentUsd: number;+ harnessUsd: number;+ durationMs: number;+}++let lane: LaneResolution;+let harness: Harness | null = null;+let scopes: RunContext;+let watch: ServerLogWatch;+let ledger: Ledger;+let models: ModelClient;+let tenantId: string;+let runStart: Date;+const reports: ScenarioReport[] = [];++beforeAll(async () => {+ lane = await resolveLane('agentic');+ if (!lane.target) return;+ if (!SCENARIOS.some((scenario) => admit(scenario.header, lane.target!.env).run)) return;+ const { env } = lane.target;+ watch = new ServerLogWatch(env.serverLog);+ ledger = new Ledger(env.agentic.budgetUsd!);+ models = new ModelClient(env.agentic.providerEnvFile);+ runStart = new Date();+ tenantId = await lane.target.switchClient.tenantId();++ harness = await setupHarness(lane.target);+ scopes = new RunContext(harness.runId);+ // The first scope named the setup's agent and channel; scenarios start after it.+ scopes.scope();+ console.log(+ [+ '',+ `Agentic lane run ${harness.runId}`,+ ` ${describeEnv(env)}`,+ ` ${describeServerLogCheck(env.serverLog)}`,+ ` driver ${env.agentic.driverModel}`,+ ` judge ${env.agentic.judgeModel} (reported, does not fail a scenario)`,+ ` limit $${ledger.budgetUsd.toFixed(2)}`,+ ` agent ${harness.agent.name} in ${harness.channel.name}`,+ '',+ ].join('\n')+ );+ const status = await waitForSession(+ harness,+ Number(process.env.SWITCH_E2E_SESSION_TIMEOUT_MS ?? 120_000)+ );+ console.log(` Switch reports the agent as '${status}' in the room — running scenarios.\n`);+}, 600_000);++afterAll(async () => {+ if (reports.length > 0) console.log(formatReport(reports, ledger));+ await teardownHarness(harness);+ if (!lane.target || harness === null) return;+ const late = await watch.sweep();+ if (late !== null && late.length > 0) {+ throw new Error(+ `the server logged errors after the last scenario (teardown):\n ${describeErrors(late, null)}`+ );+ }+});++/** The agent's usage this run, as Switch records it. */+async function agentUsage(): Promise<UsageTotal[]> {+ const until = new Date(Date.now() + 60 * 60_000);+ const rows = await harness!.switch.usage(tenantId, runStart, until);+ return rows.filter((row) => row.clientName === harness!.agent.name);+}++/** Bring the ledger's agent spend up to date, waiting for Switch's count to stop moving. */+async function settleAgentSpend(): Promise<UsageTotal[]> {+ const deadline = Date.now() + USAGE_SETTLE_MS;+ let rows = await agentUsage();+ let usd = usageCostUsd(rows);+ for (;;) {+ await sleep(3_000);+ const next = await agentUsage();+ const nextUsd = usageCostUsd(next);+ rows = next;+ if (nextUsd === usd || Date.now() >= deadline) {+ usd = nextUsd;+ break;+ }+ usd = nextUsd;+ }+ ledger.setAgent(usd);+ return rows;+}++describe('agentic lane', () => {+ for (const scenario of SCENARIOS) {+ it(scenario.header.id, async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ const admission = admit(scenario.header, lane.target.env);+ if (!admission.run) {+ if (admission.failure) {+ throw new Error(+ `${scenario.header.id} cannot run (SWITCH_E2E_STRICT=1): ${admission.reason}`+ );+ }+ return context.skip(admission.reason);+ }+ const h = harness!;+ const { env } = lane.target;+ const driverModel = env.agentic.driverModel!;+ const judgeModel = env.agentic.judgeModel!;+ const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);+ const started = Date.now();+ const agentBefore = usageCostUsd(await agentUsage());+ let harnessUsd = 0;+ const spendOnHarness = (usd: number): void => {+ harnessUsd += usd;+ ledger.addHarness(usd);+ };++ ledger.assertWithin(`${scenario.header.id} started`);+ const observer = watch.begin(h.switch, scopes.scope().tag);+ const turns: Turn[] = [];+ let failure: unknown = null;+ try {+ for (const message of scenario.setupMessages) {+ await postUnaddressed(h, message);+ turns.push({ from: 'channel', text: message });+ }+ let next: string | null = scenario.opening;+ while (next !== null) {+ turns.push({ from: 'human', text: next });+ log(`human: ${next}`);+ const reply = await exchange(h, next, REPLY_DEADLINE_MS);+ turns.push(reply);+ log(`agent: ${reply.text.replace(/\s+/g, ' ').slice(0, 300)}`);+ ledger.setAgent(usageCostUsd(await agentUsage()));+ ledger.assertWithin(`the next turn of ${scenario.header.id}`);+ if (turns.filter((turn) => turn.from === 'human').length >= scenario.maxTurns) break;+ const driven = await driverTurn(+ models,+ driverModel,+ Math.min(CALL_LIMIT_USD, ledger.remainingUsd),+ scenario,+ turns+ );+ spendOnHarness(driven.costUsd);+ next = driven.message;+ }+ } catch (error) {+ failure = error;+ } finally {+ observer.stop();+ }++ let judgement: Judgement | null = null;+ if (failure === null) {+ try {+ ledger.assertWithin(`judging ${scenario.header.id}`);+ judgement = await judge(+ models,+ judgeModel,+ Math.min(CALL_LIMIT_USD, ledger.remainingUsd),+ scenario,+ turns+ );+ spendOnHarness(judgement.costUsd);+ } catch (error) {+ failure = error;+ }+ }+ let usage: UsageTotal[] = [];+ try {+ usage = await settleAgentSpend();+ } catch (error) {+ if (failure === null) failure = error;+ }+ const agentUsd = usage.length === 0 ? 0 : usageCostUsd(usage) - agentBefore;+ const report: ScenarioReport = {+ id: scenario.header.id,+ turns,+ failure:+ failure === null ? null : failure instanceof Error ? failure.message : String(failure),+ judgement,+ agentUsd,+ harnessUsd,+ durationMs: Date.now() - started,+ };+ reports.push(report);+ log(`agent tokens this run so far: ${describeUsage(usage)}`);+ log(+ `spent: agent $${agentUsd.toFixed(4)}, driver and judge $${report.harnessUsd.toFixed(4)}; run ${ledger.describe()}`+ );+ log(describeJudgement(judgement));+ writeReport(env.outputDir, report);++ const serverErrors = observer.errors(failure, log);+ if (failure !== null) throw failure;+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');+ });+ }++ it('the server is still ready after the run', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ await lane.target.switchClient.ready();+ });++ it('the server logged no errors outside the scenarios', async (context) => {+ if (!lane.target || harness === null) return context.skip(lane.skip ?? 'no scenario ran');+ expectNoServerErrors(await watch.sweep(), null, 'during setup or between scenarios');+ });++ it('the run stayed within its spending limit', async (context) => {+ if (!lane.target || harness === null) return context.skip(lane.skip ?? 'no scenario ran');+ await settleAgentSpend();+ console.log(`agentic lane spent ${ledger.describe()}`);+ expect(ledger.totalUsd, `spent ${ledger.describe()}`).toBeLessThanOrEqual(ledger.budgetUsd);+ });+});++function describeJudgement(judgement: Judgement | null): string {+ if (judgement === null) return 'judge: not run (the conversation did not finish)';+ if (!judgement.ok)+ return `judge: no usable verdict (${judgement.error}); it said: ${judgement.raw.slice(0, 300)}`;+ const passed = judgement.verdicts.filter((verdict) => verdict.pass).length;+ return [+ `judge: ${passed}/${judgement.verdicts.length} criteria met (reported, not failing)`,+ ...judgement.verdicts.map(+ (verdict) => ` ${verdict.pass ? 'met ' : 'not met'} ${verdict.id}: ${verdict.reason}`+ ),+ ].join('\n');+}++function writeReport(outputDir: string | null, report: ScenarioReport): void {+ if (outputDir === null) return;+ const dir = path.join(outputDir, 'agentic');+ mkdirSync(dir, { recursive: true });+ writeFileSync(path.join(dir, `${report.id}.json`), `${JSON.stringify(report, null, 2)}\n`);+}++function formatReport(all: readonly ScenarioReport[], spent: Ledger): string {+ const rows = all.map((report) => {+ const humans = report.turns.filter((turn) => turn.from === 'human').length;+ const verdict =+ report.judgement === null+ ? 'not judged'+ : report.judgement.ok+ ? `${report.judgement.verdicts.filter((v) => v.pass).length}/${report.judgement.verdicts.length} met`+ : 'judge error';+ return (+ ` ${report.failure === null ? 'ok ' : 'FAIL'} ${report.id.padEnd(28)} ` ++ `${String(humans).padStart(2)} turns ${verdict.padEnd(11)} ` ++ `agent $${report.agentUsd.toFixed(4)} driver+judge $${report.harnessUsd.toFixed(4)} ` ++ `${Math.round(report.durationMs / 1000)} s`+ );+ });+ return ['', 'Agentic lane', ...rows, ` total ${spent.describe()}`, ''].join('\n');+}diff --git a/console/tooling-e2e/src/agentic/agentic.test.ts b/console/tooling-e2e/src/agentic/agentic.test.tsnew file mode 100644index 000000000..b86377011--- /dev/null+++ b/console/tooling-e2e/src/agentic/agentic.test.ts@@ -0,0 +1,169 @@+import { describe, expect, it } from 'vitest';+import type { UsageTotal } from '../clients/switch-client.ts';+import { parseAnswer } from './model.ts';+import { parseVerdicts } from './roles.ts';+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios, parseScenario } from './scenario-file.ts';+import { BudgetExceededError, Ledger, priceOf, usageCostUsd } from './spend.ts';++const SCENARIO = `+id: some-scenario+owner: platform+covers: [post_message]+persona: Someone.+goal: Something.+opening: Hello?+max_turns: 2+rubric:+ - id: replies+ criterion: The agent replies.+`;++describe('agentic scenario files', () => {+ it('reads every scenario in the suite', () => {+ const scenarios = loadAgenticScenarios(AGENTIC_SCENARIO_DIR);+ expect(scenarios.length).toBeGreaterThan(0);+ expect(scenarios.every((scenario) => scenario.header.lane === 'agentic')).toBe(true);+ });++ it('reads a valid file, with setup messages optional', () => {+ const scenario = parseScenario('some-scenario.yaml', SCENARIO);+ expect(scenario.setupMessages).toEqual([]);+ expect(scenario.rubric).toEqual([{ id: 'replies', criterion: 'The agent replies.' }]);+ expect(scenario.header.requires).toEqual(['mattermost', 'agentic-models']);+ });++ it('names every problem in a bad file at once', () => {+ const bad = `+id: Not Kebab+owner: ''+covers: post_message+persona: Someone.+goal: Something.+opening: Hello?+max_turns: 9+surprise: true+rubric:+ - id: same+ criterion: One.+ - id: same+ criterion: Two.+`;+ let message = '';+ try {+ parseScenario('other.yaml', bad);+ } catch (error) {+ message = (error as Error).message;+ }+ for (const problem of [+ "unknown field 'surprise'",+ "'owner' must be non-empty",+ "'covers' must be a list",+ 'not kebab-case',+ 'does not match the file name',+ "'max_turns'",+ 'used twice',+ ]) {+ expect(message).toContain(problem);+ }+ });+});++describe('judge verdicts', () => {+ const ids = ['replies', 'stays-short'];++ it('takes one verdict per criterion, in rubric order, from JSON wrapped in prose', () => {+ const raw = `Here you go: {"verdicts": [+ {"id": "stays-short", "pass": false, "reason": "Long."},+ {"id": "replies", "pass": true, "reason": "It did."}]}`;+ expect(parseVerdicts(raw, ids)).toEqual({+ ok: true,+ verdicts: [+ { id: 'replies', pass: true, reason: 'It did.' },+ { id: 'stays-short', pass: false, reason: 'Long.' },+ ],+ });+ });++ it('refuses verdicts that do not match the rubric, or are not verdicts', () => {+ expect(parseVerdicts('no json here', ids).ok).toBe(false);+ expect(+ parseVerdicts('{"verdicts": [{"id": "replies", "pass": true, "reason": "x"}]}', ids)+ ).toMatchObject({ ok: false, error: expect.stringContaining('missing: stays-short') });+ expect(+ parseVerdicts(+ '{"verdicts": [{"id": "replies", "pass": "yes", "reason": "x"}, {"id": "stays-short", "pass": true, "reason": "x"}]}',+ ids+ )+ ).toMatchObject({ ok: false, error: expect.stringContaining('not {id, pass, reason}') });+ });+});++describe('model answers', () => {+ const answer = (model: string, extra: Record<string, unknown> = {}) =>+ JSON.stringify({+ result: ' ok ',+ is_error: false,+ total_cost_usd: 0.0011,+ modelUsage: { [model]: {} },+ ...extra,+ });++ it('takes the text and cost of an answer from the model asked', () => {+ expect(parseAnswer(answer('claude-sonnet-5'), 'claude-sonnet-5')).toEqual({+ text: 'ok',+ costUsd: 0.0011,+ });+ });++ it('refuses an answer from another model, a failed call, and output that is not JSON', () => {+ expect(() => parseAnswer(answer('claude-haiku-4-5'), 'claude-sonnet-5')).toThrow(+ /came from claude-haiku-4-5/+ );+ expect(() =>+ parseAnswer(+ answer('claude-sonnet-5', { is_error: true, result: 'not available' }),+ 'claude-sonnet-5'+ )+ ).toThrow(/not available/);+ expect(() => parseAnswer('Error: something', 'claude-sonnet-5')).toThrow(+ /did not answer with JSON/+ );+ });+});++describe('spend', () => {+ const row = (metric: string, amount: number, model = 'claude-sonnet-5'): UsageTotal => ({+ metric,+ clientId: 'c',+ clientName: 'e2e-abc123a-claude',+ model,+ amount,+ });++ it('prices tokens per metric and model, and counts turns and messages as free', () => {+ const usd = usageCostUsd([+ row('input_tokens', 1_000_000),+ row('output_tokens', 100_000),+ row('cache_read_tokens', 1_000_000),+ row('cache_write_tokens', 400_000),+ row('turns', 3),+ ]);+ expect(usd).toBeCloseTo(2 + 1 + 0.2 + 1, 6);+ expect(priceOf('claude-sonnet-5-20260801')).toEqual(priceOf('claude-sonnet-5'));+ expect(priceOf('claude-haiku-4-5@20251001')).toEqual(priceOf('claude-haiku-4-5'));+ });++ it('refuses a model it has no price for, rather than counting it as free', () => {+ expect(() => usageCostUsd([row('input_tokens', 10, 'claude-unknown-1')])).toThrow(/No price/);+ });++ it('stops once the limit is reached', () => {+ const ledger = new Ledger(1);+ ledger.addHarness(0.25);+ ledger.setAgent(0.5);+ expect(ledger.remainingUsd).toBeCloseTo(0.25, 6);+ ledger.assertWithin('the next turn');+ ledger.setAgent(0.75);+ expect(() => ledger.assertWithin('the next turn')).toThrow(BudgetExceededError);+ });+});diff --git a/console/tooling-e2e/src/agentic/conversation.ts b/console/tooling-e2e/src/agentic/conversation.tsnew file mode 100644index 000000000..4ce5feb3d--- /dev/null+++ b/console/tooling-e2e/src/agentic/conversation.ts@@ -0,0 +1,97 @@+import { sleep, type MattermostPost } from '../clients/mattermost-client.ts';+import type { Harness } from '../provider/harness.ts';+import { isNoSessionNotice, isRuntimeStatusPost, turnStateOf } from '../provider/scenarios.ts';+import type { Turn } from './roles.ts';++/**+ * The human's side of the channel: post as the harness's Mattermost human,+ * then collect what the agent says back.+ *+ * Switch posts a status card for each turn, on the agent's bot account, and+ * edits it in place from `Working…` to how the turn ended; the agent's own+ * words arrive as separate posts beside it. A reply is finished when the card+ * says the turn ended, or, with no card, when the agent has been quiet for+ * `QUIET_MS`. Posts are read again on every poll, so what is kept is each+ * post's final text, not the first draft of it.+ *+ * The card and Switch's other commentary are not the agent speaking and are+ * left out of the transcript. A no-session notice, a turn that ended in an+ * error, or no reply at all ends the scenario.+ */++const QUIET_MS = 10_000;+const POLL_MS = 2_000;++export class NoReplyError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'NoReplyError';+ }+}++/** Post a message nobody is addressed in. */+export async function postUnaddressed(harness: Harness, text: string): Promise<void> {+ await harness.mattermost.post({ channelId: harness.channel.id, message: text });+}++/** Post `@agent text` and return the agent's whole reply. */+export async function exchange(+ harness: Harness,+ text: string,+ replyDeadlineMs: number+): Promise<Turn> {+ const since = Date.now() - 1;+ await harness.mattermost.post({+ channelId: harness.channel.id,+ message: `@${harness.agent.name} ${text}`,+ });++ const deadline = Date.now() + replyDeadlineMs;+ const latest = new Map<string, MattermostPost>();+ let lastChangeMs = Date.now();+ for (;;) {+ for (const post of await harness.mattermost.postsFrom(+ harness.channel.id,+ harness.bot.id,+ since+ )) {+ if (isNoSessionNotice(post)) {+ throw new NoReplyError(`Switch says no session is attending the room: «${post.message}»`);+ }+ const before = latest.get(post.id);+ if (before === undefined || before.message !== post.message) {+ latest.set(post.id, post);+ lastChangeMs = Date.now();+ }+ }+ const posts = [...latest.values()].sort((a, b) => a.create_at - b.create_at);+ const said = posts.filter((post) => !isRuntimeStatusPost(post));+ const states = posts.map(turnStateOf).filter((state) => state !== null);+ const ended =+ states.length > 0 && states.every((state) => state !== 'queued' && state !== 'running');+ if (states.includes('error')) {+ throw new NoReplyError(+ `The agent's turn ended with an error. It said: ${said.map((post) => `«${post.message}»`).join(' ') || 'nothing'}`+ );+ }+ if (ended && said.length === 0) {+ throw new NoReplyError(+ `The agent's turn ended without a word in the room in answer to «${text}».`+ );+ }+ if (+ said.length > 0 &&+ (ended || (states.length === 0 && Date.now() - lastChangeMs >= QUIET_MS))+ ) {+ return { from: 'agent', text: said.map((post) => post.message).join('\n\n') };+ }+ if (Date.now() >= deadline) {+ throw new NoReplyError(+ said.length === 0+ ? `The agent did not reply within ${Math.round(replyDeadlineMs / 1000)} s to «${text}».`+ : `The agent's turn had not ended after ${Math.round(replyDeadlineMs / 1000)} s.`+ );+ }+ await sleep(POLL_MS);+ }+}diff --git a/console/tooling-e2e/src/agentic/model.ts b/console/tooling-e2e/src/agentic/model.tsnew file mode 100644index 000000000..a3361f487--- /dev/null+++ b/console/tooling-e2e/src/agentic/model.ts@@ -0,0 +1,140 @@+import { spawn } from 'node:child_process';+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import path from 'node:path';+import { parseDotEnv } from '../core/env.ts';++/**+ * One call to a pinned model, through the Claude Code CLI: the same binary,+ * and the same credentials, the agent controller runs sessions with, so the+ * lane needs no second way of reaching a model.+ *+ * The call is stripped to a plain completion: its own system prompt, no tools,+ * no MCP servers, no settings, no saved session, from an empty directory. That+ * keeps a developer's plugins, hooks and CLAUDE.md out of it, and keeps its+ * cost to the tokens it is actually given (a default call carries Claude+ * Code's whole system prompt, about seventy times the cost of a short reply).+ *+ * The model is never chosen here: a call names one, and an answer from any+ * other model is refused rather than counted.+ */++export interface ModelAnswer {+ text: string;+ costUsd: number;+}++export class ModelError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'ModelError';+ }+}++const CALL_TIMEOUT_MS = 3 * 60_000;++export class ModelClient {+ private readonly env: NodeJS.ProcessEnv;++ /** `providerEnvFile`: keys to add to the CLI's environment, or null for its own sign-in. */+ constructor(providerEnvFile: string | null) {+ this.env = { ...process.env };+ if (providerEnvFile !== null) {+ Object.assign(this.env, parseDotEnv(readFileSync(providerEnvFile, 'utf8')));+ }+ }++ async ask(params: {+ model: string;+ system: string;+ prompt: string;+ /** The most this one call may spend; the CLI stops it there. */+ maxUsd: number;+ }): Promise<ModelAnswer> {+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-model-'));+ try {+ const raw = await run(+ [+ '-p',+ '--model',+ params.model,+ '--system-prompt',+ params.system,+ '--tools',+ '',+ '--strict-mcp-config',+ '--setting-sources',+ '',+ '--no-session-persistence',+ '--max-budget-usd',+ params.maxUsd.toFixed(4),+ '--output-format',+ 'json',+ ],+ params.prompt,+ dir,+ this.env+ );+ return parseAnswer(raw, params.model);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ }+}++/** The CLI's JSON result, held to the model that was asked for. */+export function parseAnswer(raw: string, model: string): ModelAnswer {+ let result: {+ result?: unknown;+ is_error?: unknown;+ total_cost_usd?: unknown;+ modelUsage?: Record<string, unknown>;+ };+ try {+ result = JSON.parse(raw) as typeof result;+ } catch {+ throw new ModelError(`The Claude Code CLI did not answer with JSON: ${raw.slice(0, 300)}`);+ }+ const text = typeof result.result === 'string' ? result.result : '';+ if (result.is_error === true) {+ throw new ModelError(`The call to ${model} failed: ${text.slice(0, 300)}`);+ }+ const answered = Object.keys(result.modelUsage ?? {});+ if (!answered.includes(model)) {+ throw new ModelError(+ `Asked ${model}, but the answer came from ${answered.join(', ') || 'no model'}. ` ++ `The lane refuses a substitute: name a model these credentials can use.`+ );+ }+ if (typeof result.total_cost_usd !== 'number') {+ throw new ModelError(`The call to ${model} reported no cost.`);+ }+ return { text: text.trim(), costUsd: result.total_cost_usd };+}++function run(args: string[], stdin: string, cwd: string, env: NodeJS.ProcessEnv): Promise<string> {+ return new Promise((resolve, reject) => {+ const child = spawn('claude', args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] });+ let stdout = '';+ let stderr = '';+ const timer = setTimeout(() => {+ child.kill('SIGKILL');+ reject(+ new ModelError(`The Claude Code CLI did not answer within ${CALL_TIMEOUT_MS / 1000} s.`)+ );+ }, CALL_TIMEOUT_MS);+ child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString('utf8')));+ child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString('utf8')));+ child.on('error', (error) => {+ clearTimeout(timer);+ reject(new ModelError(`Could not run the Claude Code CLI ('claude'): ${error.message}`));+ });+ child.on('close', (code) => {+ clearTimeout(timer);+ // A failed call still prints its JSON result, which says why.+ if (stdout.trim() !== '') resolve(stdout);+ else reject(new ModelError(`The Claude Code CLI exited ${code}: ${stderr.slice(0, 300)}`));+ });+ child.stdin.end(stdin);+ });+}diff --git a/console/tooling-e2e/src/agentic/roles.ts b/console/tooling-e2e/src/agentic/roles.tsnew file mode 100644index 000000000..866880a31--- /dev/null+++ b/console/tooling-e2e/src/agentic/roles.ts@@ -0,0 +1,147 @@+import type { ModelClient } from './model.ts';+import type { AgenticScenario } from './scenario-file.ts';++/**+ * The two model roles around the agent under test.+ *+ * The driver plays the human: given the scenario and the conversation so far,+ * it writes the human's next message, or says it is done. The judge reads the+ * finished conversation and grades it against the scenario's rubric, one+ * criterion at a time. Neither sees the other's prompt, and the judge never+ * sees the persona: it grades what was said, not what was meant.+ */++/**+ * One message in the conversation. `channel` is a message posted to the+ * channel without addressing the agent (a scenario's setup messages): part of+ * what the agent could read, not something it was asked.+ */+export interface Turn {+ from: 'human' | 'agent' | 'channel';+ text: string;+}++const LABEL: Record<Turn['from'], string> = {+ human: 'human, to the agent',+ agent: 'agent',+ channel: 'channel, earlier, not addressed to the agent',+};++/** What the driver answers when the human has nothing more to say. */+export const DONE = 'DONE';++const DRIVER_SYSTEM = `You play a human in a team chat channel, talking to an AI agent that is a member of the channel. You are testing nothing and know nothing about tests: you are the person described, with the goal described.++Write only the next message you would send: plain text, as you would type it in chat, with no quotation marks, no name prefix and no @mention (one is added for you). Keep it short and natural.++If your goal is met, or the agent has made clear it cannot help further, reply with exactly ${DONE} and nothing else.`;++const JUDGE_SYSTEM = `You grade a conversation between a human and an AI agent in a team chat channel against a list of criteria. Judge only what the transcript shows. Grade each criterion on its own: one failure does not fail the others. If the transcript does not show enough to decide a criterion, it fails.++Answer with JSON only, no prose and no code fence, in exactly this shape:+{"verdicts": [{"id": "<criterion id>", "pass": true, "reason": "<one sentence>"}]}+with one entry per criterion, in the order given.`;++export function transcriptText(turns: readonly Turn[]): string {+ return turns.map((turn) => `[${LABEL[turn.from]}] ${turn.text}`).join('\n\n');+}++export async function driverTurn(+ client: ModelClient,+ model: string,+ maxUsd: number,+ scenario: AgenticScenario,+ turns: readonly Turn[]+): Promise<{ message: string | null; costUsd: number }> {+ const humanTurns = turns.filter((turn) => turn.from === 'human').length;+ const answer = await client.ask({+ model,+ maxUsd,+ system: DRIVER_SYSTEM,+ prompt: [+ `Who you are:\n${scenario.persona}`,+ `Your goal:\n${scenario.goal}`,+ `The conversation so far:\n${transcriptText(turns)}`,+ `You have sent ${humanTurns} of at most ${scenario.maxTurns} messages. Write your next message, or ${DONE}.`,+ ].join('\n\n'),+ });+ const message = answer.text.replace(/^["“]|["”]$/g, '').trim();+ return { message: message === DONE || message === '' ? null : message, costUsd: answer.costUsd };+}++export interface Verdict {+ id: string;+ pass: boolean;+ reason: string;+}++export type Judgement =+ | { ok: true; verdicts: Verdict[]; costUsd: number }+ | { ok: false; error: string; raw: string; costUsd: number };++export async function judge(+ client: ModelClient,+ model: string,+ maxUsd: number,+ scenario: AgenticScenario,+ turns: readonly Turn[]+): Promise<Judgement> {+ const answer = await client.ask({+ model,+ maxUsd,+ system: JUDGE_SYSTEM,+ prompt: [+ `What the human wanted:\n${scenario.goal}`,+ `Criteria:\n${scenario.rubric.map((item) => `- ${item.id}: ${item.criterion}`).join('\n')}`,+ `Transcript:\n${transcriptText(turns)}`,+ ].join('\n\n'),+ });+ const parsed = parseVerdicts(+ answer.text,+ scenario.rubric.map((item) => item.id)+ );+ return parsed.ok+ ? { ok: true, verdicts: parsed.verdicts, costUsd: answer.costUsd }+ : { ok: false, error: parsed.error, raw: answer.text, costUsd: answer.costUsd };+}++/** The judge's verdicts, held to the rubric: one per criterion, each a boolean with a reason. */+export function parseVerdicts(+ raw: string,+ ids: readonly string[]+): { ok: true; verdicts: Verdict[] } | { ok: false; error: string } {+ const start = raw.indexOf('{');+ const end = raw.lastIndexOf('}');+ if (start === -1 || end < start) return { ok: false, error: 'the judge answered without JSON' };+ let value: unknown;+ try {+ value = JSON.parse(raw.slice(start, end + 1));+ } catch (error) {+ return { ok: false, error: `the judge's JSON does not parse: ${String(error)}` };+ }+ const list = (value as { verdicts?: unknown }).verdicts;+ if (!Array.isArray(list)) return { ok: false, error: "the judge's JSON has no 'verdicts' list" };+ const verdicts: Verdict[] = [];+ for (const entry of list) {+ const item = entry as Record<string, unknown> | null;+ if (+ item === null ||+ typeof item.id !== 'string' ||+ typeof item.pass !== 'boolean' ||+ typeof item.reason !== 'string'+ ) {+ return { ok: false, error: `a verdict is not {id, pass, reason}: ${JSON.stringify(entry)}` };+ }+ verdicts.push({ id: item.id, pass: item.pass, reason: item.reason });+ }+ const given = verdicts.map((verdict) => verdict.id);+ const missing = ids.filter((id) => !given.includes(id));+ const extra = given.filter((id) => !ids.includes(id));+ if (missing.length > 0 || extra.length > 0 || given.length !== ids.length) {+ return {+ ok: false,+ error: `the verdicts do not match the rubric (missing: ${missing.join(', ') || 'none'}; unknown or repeated: ${extra.join(', ') || 'none'})`,+ };+ }+ return { ok: true, verdicts: ids.map((id) => verdicts.find((verdict) => verdict.id === id)!) };+}diff --git a/console/tooling-e2e/src/agentic/scenario-file.ts b/console/tooling-e2e/src/agentic/scenario-file.tsnew file mode 100644index 000000000..3667cf8e4--- /dev/null+++ b/console/tooling-e2e/src/agentic/scenario-file.ts@@ -0,0 +1,143 @@+import { readdirSync, readFileSync } from 'node:fs';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { parse } from 'yaml';+import type { ScenarioHeader } from '../core/scenario.ts';++/**+ * An agentic scenario, as written in `agentic/<id>.yaml`.+ *+ * It says who the human is and what they want, how the conversation opens,+ * and what a good conversation looks like. A model plays the human from it+ * (the driver), the agent under test answers in a real session, and another+ * model grades the transcript against the rubric (the judge). Adding a+ * scenario is adding a file: no code.+ *+ * id: summarise-the-channel # kebab-case, the file's own name+ * owner: platform # a team or role, never a person+ * covers: [read_context] # as any scenario header+ * persona: | # who the driver plays+ * goal: | # what they want; the driver stops once they have it+ * setup_messages: [...] # posted, unaddressed, before the opening (optional)+ * opening: "…" # the first message, sent as written, addressed to the agent+ * max_turns: 3 # human messages, the opening included+ * rubric: # what the judge checks, each on its own+ * - id: cites-the-decision+ * criterion: The summary names the decision the team reached.+ */+export interface AgenticScenario {+ header: ScenarioHeader;+ persona: string;+ goal: string;+ setupMessages: string[];+ opening: string;+ maxTurns: number;+ rubric: { id: string; criterion: string }[];+}++/** Where the scenario files live: `console/tooling-e2e/agentic/`. */+export const AGENTIC_SCENARIO_DIR = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ '..',+ '..',+ 'agentic'+);++const ID = /^[a-z0-9]+(-[a-z0-9]+)*$/;+const KEYS = new Set([+ 'id',+ 'owner',+ 'covers',+ 'persona',+ 'goal',+ 'setup_messages',+ 'opening',+ 'max_turns',+ 'rubric',+]);++/** Every scenario file, validated. A malformed one throws naming every problem in it. */+export function loadAgenticScenarios(dir: string): AgenticScenario[] {+ return readdirSync(dir)+ .filter((file) => file.endsWith('.yaml'))+ .sort()+ .map((file) => parseScenario(file, readFileSync(path.join(dir, file), 'utf8')));+}++export function parseScenario(file: string, raw: string): AgenticScenario {+ const value: unknown = parse(raw);+ const problems: string[] = [];+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ throw new Error(`${file}: not a mapping of scenario fields`);+ }+ const doc = value as Record<string, unknown>;+ for (const key of Object.keys(doc)) {+ if (!KEYS.has(key)) problems.push(`unknown field '${key}'`);+ }+ const text = (key: string): string => {+ const field = doc[key];+ if (typeof field !== 'string' || field.trim() === '') {+ problems.push(`'${key}' must be non-empty text`);+ return '';+ }+ return field.trim();+ };+ const texts = (key: string, required: boolean): string[] => {+ const field = doc[key];+ if (field === undefined && !required) return [];+ if (!Array.isArray(field) || !field.every((item) => typeof item === 'string' && item !== '')) {+ problems.push(`'${key}' must be a list of non-empty text`);+ return [];+ }+ return field as string[];+ };++ const id = text('id');+ if (id !== '' && !ID.test(id)) problems.push(`id '${id}' is not kebab-case`);+ if (id !== '' && `${id}.yaml` !== file) problems.push(`id '${id}' does not match the file name`);+ const maxTurns = doc.max_turns;+ if (typeof maxTurns !== 'number' || !Number.isInteger(maxTurns) || maxTurns < 1 || maxTurns > 6) {+ problems.push(`'max_turns' must be a whole number from 1 to 6`);+ }+ const rubric: { id: string; criterion: string }[] = [];+ if (!Array.isArray(doc.rubric) || doc.rubric.length === 0) {+ problems.push(`'rubric' must list at least one criterion`);+ } else {+ for (const entry of doc.rubric as unknown[]) {+ const item = entry as Record<string, unknown> | null;+ if (+ item === null ||+ typeof item.id !== 'string' ||+ !ID.test(item.id) ||+ typeof item.criterion !== 'string' ||+ item.criterion.trim() === ''+ ) {+ problems.push(`each rubric entry needs a kebab-case 'id' and a 'criterion'`);+ continue;+ }+ if (rubric.some((existing) => existing.id === item.id)) {+ problems.push(`rubric id '${item.id}' is used twice`);+ }+ rubric.push({ id: item.id, criterion: item.criterion.trim() });+ }+ }++ const scenario: AgenticScenario = {+ header: {+ id,+ covers: texts('covers', true),+ lane: 'agentic',+ environments: ['ephemeral', 'local', 'dev', 'staging'],+ owner: text('owner'),+ requires: ['mattermost', 'agentic-models'],+ },+ persona: text('persona'),+ goal: text('goal'),+ setupMessages: texts('setup_messages', false),+ opening: text('opening'),+ maxTurns: typeof maxTurns === 'number' ? maxTurns : 0,+ rubric,+ };+ if (problems.length > 0) throw new Error(`${file}:\n ${problems.join('\n ')}`);+ return scenario;+}diff --git a/console/tooling-e2e/src/agentic/spend.ts b/console/tooling-e2e/src/agentic/spend.tsnew file mode 100644index 000000000..319b710c9--- /dev/null+++ b/console/tooling-e2e/src/agentic/spend.ts@@ -0,0 +1,145 @@+import type { UsageTotal } from '../clients/switch-client.ts';++/**+ * What the agentic lane spends, and the limit it stops at.+ *+ * Two sources, both measured rather than guessed:+ *+ * - the driver and judge calls report their own cost (the Claude Code CLI+ * prices each call);+ * - the agent's session is priced from the tokens Switch itself records for+ * it, per model (`GET /gateway/tenants/{tenant_id}/usage`), at the prices+ * below. A model with no price here stops the run instead of counting as+ * free.+ */++/** USD per million tokens. */+export interface Price {+ input: number;+ output: number;+ cacheRead: number;+ cacheWrite: number;+}++/**+ * List prices from Anthropic's public pricing page+ * (platform.claude.com/docs/en/about-claude/pricing), read 2026-10-19. Cache+ * writes are priced at the five-minute rate, the one Claude Code uses by+ * default; Switch records cache writes without their duration.+ */+export const PRICES: Readonly<Record<string, Price>> = {+ 'claude-sonnet-5': { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },+ 'claude-sonnet-5-5': { input: 2, output: 10, cacheRead: 0.1, cacheWrite: 2.5 },+ 'claude-opus-5': { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 },+ 'claude-opus-5-5': { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 },+ // Claude Code's own background work (titles, summaries) runs on it.+ 'claude-haiku-4-5': { input: 1, output: 5, cacheRead: 0.1, cacheWrite: 1.25 },+};++const METRIC_TO_PRICE: Readonly<Record<string, keyof Price>> = {+ input_tokens: 'input',+ output_tokens: 'output',+ cache_read_tokens: 'cacheRead',+ cache_write_tokens: 'cacheWrite',+};++/**+ * A model id as priced. A dated snapshot is priced as its model, however the+ * provider spells the date: `claude-haiku-4-5-20251001` from Anthropic's API,+ * `claude-haiku-4-5@20251001` from Vertex.+ */+export function priceOf(model: string): Price {+ const price = PRICES[model] ?? PRICES[model.replace(/[-@]\d{8}$/, '')];+ if (price === undefined) {+ throw new Error(+ `No price for model '${model}', so what it spent cannot be counted against the limit. ` ++ `Add it to PRICES in src/agentic/spend.ts from the public pricing page.`+ );+ }+ return price;+}++/** What one client spent, in USD, from its usage rows. Rows that are not tokens (turns, messages) cost nothing. */+export function usageCostUsd(rows: readonly UsageTotal[]): number {+ let usd = 0;+ for (const row of rows) {+ const field = METRIC_TO_PRICE[row.metric];+ if (field === undefined) continue;+ usd += (row.amount * priceOf(row.model)[field]) / 1_000_000;+ }+ return usd;+}++/** Each model's tokens, for the report: `claude-sonnet-5 12,345 in / 678 out / 90,000 cached`. */+export function describeUsage(rows: readonly UsageTotal[]): string {+ const byModel = new Map<string, Record<string, number>>();+ for (const row of rows) {+ // Turns and messages are counted without a model; they are not tokens.+ if (METRIC_TO_PRICE[row.metric] === undefined) continue;+ const counts = byModel.get(row.model) ?? {};+ counts[row.metric] = (counts[row.metric] ?? 0) + row.amount;+ byModel.set(row.model, counts);+ }+ if (byModel.size === 0) return 'no tokens recorded';+ const n = (value: number | undefined) => (value ?? 0).toLocaleString('en-US');+ return [...byModel]+ .map(+ ([model, c]) =>+ `${model} ${n(c.input_tokens)} in / ${n(c.output_tokens)} out / ` ++ `${n(c.cache_read_tokens)} cache read / ${n(c.cache_write_tokens)} cache write`+ )+ .join('; ');+}++export class BudgetExceededError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'BudgetExceededError';+ }+}++/** The lane's running total against its limit. */+export class Ledger {+ readonly budgetUsd: number;+ private harnessUsd = 0;+ private agentUsd = 0;++ constructor(budgetUsd: number) {+ this.budgetUsd = budgetUsd;+ }++ /** A driver or judge call's cost. */+ addHarness(usd: number): void {+ this.harnessUsd += usd;+ }++ /** The agent's spend so far this run, as Switch last reported it. */+ setAgent(usd: number): void {+ this.agentUsd = usd;+ }++ get totalUsd(): number {+ return this.harnessUsd + this.agentUsd;+ }++ get remainingUsd(): number {+ return Math.max(0, this.budgetUsd - this.totalUsd);+ }++ /** Throw once the limit is reached: nothing more may be spent this run. */+ assertWithin(what: string): void {+ if (this.totalUsd >= this.budgetUsd) {+ throw new BudgetExceededError(+ `The spending limit of $${this.budgetUsd.toFixed(2)} is reached ($${this.totalUsd.toFixed(4)} spent) ` ++ `before ${what}. Raise SWITCH_E2E_AGENTIC_BUDGET_USD only if that spend was expected.`+ );+ }+ }++ describe(): string {+ return (+ `$${this.totalUsd.toFixed(4)} of $${this.budgetUsd.toFixed(2)} ` ++ `(agent $${this.agentUsd.toFixed(4)}, driver and judge $${this.harnessUsd.toFixed(4)})`+ );+ }+}diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.tsindex 81285a7f0..ee6549577 100644--- a/console/tooling-e2e/src/clients/switch-client.ts+++ b/console/tooling-e2e/src/clients/switch-client.ts@@ -72,6 +72,15 @@ export interface AgentSummary {* "attachments": [] } }* ```*/+/** One row of `GET /gateway/tenants/{tenant_id}/usage`: an amount per metric, client and model. */+export interface UsageTotal {+ metric: string;+ clientId: string;+ clientName: string | null;+ model: string;+ amount: number;+}+export interface AgentEvent {type: string;room_id?: string;@@ -409,6 +418,44 @@ export class SwitchClient {return rows.map((row) => ({ id: row.id, name: row.name, createdAt: row.created_at }));}+ /** `GET /gateway/auth/session` — the workspace this admin session is bound to. */+ async tenantId(): Promise<string> {+ const state = await this.gateway<{ state: string; tenant: { id: string } | null }>(+ 'GET',+ '/auth/session'+ );+ if (state.tenant === null) {+ throw new Error(+ `The gateway admin's session is bound to no workspace (state '${state.state}').`+ );+ }+ return state.tenant.id;+ }++ /**+ * `GET /gateway/tenants/{tenant_id}/usage` — what each client spent, per+ * metric and model, over `[since, until)`. Counted in whole UTC hours.+ */+ async usage(tenantId: string, since: Date, until: Date): Promise<UsageTotal[]> {+ const query = `since=${encodeURIComponent(since.toISOString())}&until=${encodeURIComponent(until.toISOString())}`;+ const rows = await this.gateway<+ {+ metric: string;+ client_id: string;+ client_name: string | null;+ model: string;+ amount: number;+ }[]+ >('GET', `/tenants/${tenantId}/usage?${query}`);+ return rows.map((row) => ({+ metric: row.metric,+ clientId: row.client_id,+ clientName: row.client_name,+ model: row.model,+ amount: row.amount,+ }));+ }+/** The bridge id of the default collaboration bridge of `type`. */async defaultBridgeId(type: string): Promise<string> {const rows = await this.gateway<diff --git a/console/tooling-e2e/src/core/core.test.ts b/console/tooling-e2e/src/core/core.test.tsindex d75afe182..76447934a 100644--- a/console/tooling-e2e/src/core/core.test.ts+++ b/console/tooling-e2e/src/core/core.test.ts@@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import path from 'node:path';import { describe, expect, it } from 'vitest';+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from '../agentic/scenario-file.ts';import { BROWSER_SCENARIOS } from '../browser/index.ts';import { olderThanFrom, timestampMs } from '../cleanup.ts';import { FAST_SCENARIOS } from '../fast/index.ts';@@ -135,7 +136,8 @@ describe('scenario headers', () => {const fast = FAST_SCENARIOS.map((scenario) => scenario.header);const provider = PROVIDER_SCENARIOS.map((scenario) => scenario.header);const browser = BROWSER_SCENARIOS.map((scenario) => scenario.header);- expect(() => assertValidHeaders([...fast, ...provider, ...browser])).not.toThrow();+ const agentic = loadAgenticScenarios(AGENTIC_SCENARIO_DIR).map((scenario) => scenario.header);+ expect(() => assertValidHeaders([...fast, ...provider, ...browser, ...agentic])).not.toThrow();expect(fast.every((h) => h.lane === 'fast')).toBe(true);expect(provider.every((h) => h.lane === 'provider')).toBe(true);expect(browser.every((h) => h.lane === 'browser')).toBe(true);diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.tsindex 1ef5947f8..826064f35 100644--- a/console/tooling-e2e/src/core/env.ts+++ b/console/tooling-e2e/src/core/env.ts@@ -18,12 +18,14 @@ import { fileURLToPath } from 'node:url';* `provider` needs a real agent session attending the room, driven by Switch* Console or its sidecar. `browser` drives the gateway UI in a headless* Chromium against the same server: a smoke test of what an operator sees,- * which no HTTP-level scenario can see break.+ * which no HTTP-level scenario can see break. `agentic` runs the same real+ * session as `provider`, but a model plays the human from a scenario file and+ * another model judges the conversation.*/-export type Lane = 'fast' | 'provider' | 'browser';+export type Lane = 'fast' | 'provider' | 'browser' | 'agentic';/** Every lane: the one list anything that takes a lane by name checks against. */-export const LANES: readonly Lane[] = ['fast', 'provider', 'browser'];+export const LANES: readonly Lane[] = ['fast', 'provider', 'browser', 'agentic'];export function isLane(raw: string): raw is Lane {return (LANES as readonly string[]).includes(raw);@@ -93,6 +95,26 @@ export interface ProviderEnv {controllerId: string | null;}+/**+ * Settings only the agentic lane reads. Each is null when unset, and the lane's+ * scenarios then need it (`agentic-models`): a model is never picked for a run+ * that did not name one, and a run never spends without a limit.+ */+export interface AgenticEnv {+ /** Exact model id that plays the human. */+ driverModel: string | null;+ /** Exact model id that judges the conversation. */+ judgeModel: string | null;+ /** What the lane may spend, driver, judge and agent together, in USD. */+ budgetUsd: number | null;+ /**+ * A file of provider keys (`ANTHROPIC_API_KEY=…`), the one the agent+ * controller is handed too. Without it, the driver and judge use the+ * Claude Code CLI's own sign-in.+ */+ providerEnvFile: string | null;+}+export interface HarnessEnv {lane: Lane;environment: Environment;@@ -119,6 +141,7 @@ export interface HarnessEnv {/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */mattermost: MattermostEnv | null;provider: ProviderEnv;+ agentic: AgenticEnv;}const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');@@ -274,6 +297,12 @@ export function loadEnv(questionMode: questionModeFrom(get('SWITCH_E2E_QUESTION_MODE')),controllerId: get('SWITCH_E2E_CONTROLLER_ID') ?? null,},+ agentic: {+ driverModel: get('SWITCH_E2E_DRIVER_MODEL') ?? null,+ judgeModel: get('SWITCH_E2E_JUDGE_MODEL') ?? null,+ budgetUsd: budgetFrom(get('SWITCH_E2E_AGENTIC_BUDGET_USD')),+ providerEnvFile: get('SWITCH_E2E_PROVIDER_ENV_FILE') ?? null,+ },};}@@ -314,6 +343,17 @@ function questionModeFrom(raw: string | undefined): QuestionMode {throw new Error(`SWITCH_E2E_QUESTION_MODE must be 'numbered' or 'prose' — got '${raw}'.`);}+function budgetFrom(raw: string | undefined): number | null {+ if (raw === undefined) return null;+ const budget = Number(raw);+ if (!Number.isFinite(budget) || budget <= 0) {+ throw new Error(+ `SWITCH_E2E_AGENTIC_BUDGET_USD must be a positive number of dollars — got '${raw}'.`+ );+ }+ return budget;+}+function optionalUrl(raw: string | undefined): string | null {return raw === undefined ? null : stripTrailingSlash(raw);}diff --git a/console/tooling-e2e/src/core/scenario.ts b/console/tooling-e2e/src/core/scenario.tsindex 6fe07c63f..1ee572a3d 100644--- a/console/tooling-e2e/src/core/scenario.ts+++ b/console/tooling-e2e/src/core/scenario.ts@@ -38,8 +38,9 @@ export interface ScenarioHeader {* `mattermost`: a Mattermost and a human token on it are configured.* `agent-dir`: the provider session's working directory is on this machine.* `gateway-ui`: the address of a gateway UI served against this Switch server.+ * `agentic-models`: the driver and judge models, and the spending limit.*/-export type Prerequisite = 'mattermost' | 'agent-dir' | 'gateway-ui';+export type Prerequisite = 'mattermost' | 'agent-dir' | 'gateway-ui' | 'agentic-models';const OPERATION = /^[a-z][a-z0-9_]*$/;const ROUTE = /^(GET|POST|PUT|PATCH|DELETE) \/\S*$/;@@ -57,8 +58,11 @@ export function headerProblems(header: ScenarioHeader): string[] {}if (header.environments.length === 0) problems.push('environments is empty');if (header.owner.trim() === '') problems.push('owner is empty');- if (header.lane === 'provider' && header.environments.includes('prod')) {- problems.push('a provider scenario writes to the room, so it cannot list prod');+ if (+ (header.lane === 'provider' || header.lane === 'agentic') &&+ header.environments.includes('prod')+ ) {+ problems.push(`a ${header.lane} scenario writes to the room, so it cannot list prod`);}return problems;}@@ -117,6 +121,12 @@ function satisfied(prerequisite: Prerequisite, env: HarnessEnv): boolean {return env.provider.agentRepoDir !== null;case 'gateway-ui':return env.gatewayUiUrl !== null;+ case 'agentic-models':+ return (+ env.agentic.driverModel !== null &&+ env.agentic.judgeModel !== null &&+ env.agentic.budgetUsd !== null+ );}}@@ -128,5 +138,7 @@ function describePrerequisite(prerequisite: Prerequisite): string {return 'the session working directory (SWITCH_E2E_AGENT_DIR)';case 'gateway-ui':return 'a gateway UI (SWITCH_GATEWAY_UI_URL)';+ case 'agentic-models':+ return 'the driver and judge models and a spending limit (SWITCH_E2E_DRIVER_MODEL, SWITCH_E2E_JUDGE_MODEL, SWITCH_E2E_AGENTIC_BUDGET_USD)';}}diff --git a/console/tooling-e2e/src/gate/coverage.test.ts b/console/tooling-e2e/src/gate/coverage.test.tsindex 2c0a468e0..d032f8003 100644--- a/console/tooling-e2e/src/gate/coverage.test.ts+++ b/console/tooling-e2e/src/gate/coverage.test.ts@@ -1,4 +1,5 @@import { describe, expect, it } from 'vitest';+import { AGENTIC_SCENARIO_DIR, loadAgenticScenarios } from '../agentic/scenario-file.ts';import { BROWSER_SCENARIOS } from '../browser/index.ts';import type { ScenarioHeader } from '../core/scenario.ts';import { FAST_SCENARIOS } from '../fast/index.ts';@@ -11,9 +12,12 @@ import {type ServerSurface,} from './gate.ts';-const HEADERS = [...FAST_SCENARIOS, ...PROVIDER_SCENARIOS, ...BROWSER_SCENARIOS].map(- (scenario) => scenario.header-);+const HEADERS = [+ ...FAST_SCENARIOS,+ ...PROVIDER_SCENARIOS,+ ...BROWSER_SCENARIOS,+ ...loadAgenticScenarios(AGENTIC_SCENARIO_DIR),+].map((scenario) => scenario.header);describe('coverage gate', () => {it('accounts for every operation and gateway route the server has', () => {diff --git a/console/tooling-e2e/src/provider/scenarios.test.ts b/console/tooling-e2e/src/provider/scenarios.test.tsindex 0120badd6..358b535b4 100644--- a/console/tooling-e2e/src/provider/scenarios.test.ts+++ b/console/tooling-e2e/src/provider/scenarios.test.ts@@ -1,5 +1,12 @@import { describe, expect, it } from 'vitest';-import { isStartMarker, optionNumber, requestHandle } from './scenarios.ts';+import type { MattermostPost } from '../clients/mattermost-client.ts';+import {+ isRuntimeStatusPost,+ isStartMarker,+ optionNumber,+ requestHandle,+ turnStateOf,+} from './scenarios.ts';describe('interruption admission', () => {it('requires the command marker, not an acknowledgement or refusal', () => {@@ -11,6 +18,27 @@ describe('interruption admission', () => {});});+describe('turn status cards', () => {+ const post = (message: string) => ({ message }) as MattermostPost;++ it('reads where the turn got to from the heading, and treats the card as Switch talking', () => {+ expect(turnStateOf(post('**Working… 12s** · [Console](https://example.invalid)'))).toBe(+ 'running'+ );+ expect(turnStateOf(post('**Turn complete.**'))).toBe('completed');+ expect(turnStateOf(post('**Worked for 1m 4s. 3 tool calls.**'))).toBe('completed');+ expect(turnStateOf(post('**Turn interrupted. Worked for 9s.**'))).toBe('interrupted');+ expect(turnStateOf(post('**Turn ended with an error.**\nsomething'))).toBe('error');+ expect(isRuntimeStatusPost(post('**Working… 0s**'))).toBe(true);+ });++ it("leaves the agent's own words alone, bold or not", () => {+ expect(turnStateOf(post('Working on the date bug now.'))).toBe(null);+ expect(turnStateOf(post('**Note:** the turn is complete.'))).toBe(null);+ expect(isRuntimeStatusPost(post('Here is the plan.'))).toBe(false);+ });+});+describe('request cards', () => {it('reads the handle a card answers to', () => {expect(requestHandle('@user **Questions** · request `A1` Question from the agent 1. red')).toBe(diff --git a/console/tooling-e2e/src/provider/scenarios.ts b/console/tooling-e2e/src/provider/scenarios.tsindex f59d1924e..fa73f4e96 100644--- a/console/tooling-e2e/src/provider/scenarios.ts+++ b/console/tooling-e2e/src/provider/scenarios.ts@@ -76,8 +76,32 @@ const RUNTIME_STATUS_MARKERS = ['interrupted my current turn',];+/**+ * Where a turn has got to, as the session renderer heads its status card with+ * it: one post per turn, edited in place from `Working…` to how the turn ended+ * (`TURN_STATE` in core/switch_core/bridges/collaboration/session/renderers).+ */+export type TurnState = 'queued' | 'running' | 'completed' | 'interrupted' | 'error';++const TURN_STATE_HEADINGS: readonly [TurnState, string][] = [+ ['queued', 'Queued.'],+ ['running', 'Working…'],+ ['completed', 'Turn complete.'],+ // A completed turn whose duration is known says how long it ran instead.+ ['completed', 'Worked for '],+ ['interrupted', 'Turn interrupted.'],+ ['error', 'Turn ended with an error.'],+];++/** The state a turn status card reports, or null when the post is not one. */+export function turnStateOf(post: MattermostPost): TurnState | null {+ const heading = /^\*\*([^*]+)\*\*/.exec(post.message.trimStart())?.[1] ?? '';+ return TURN_STATE_HEADINGS.find(([, text]) => heading.startsWith(text))?.[0] ?? null;+}+/** Whether a post is Switch narrating the session rather than the agent speaking. */export function isRuntimeStatusPost(post: MattermostPost): boolean {+ if (turnStateOf(post) !== null) return true;const text = post.message.toLowerCase();return RUNTIME_STATUS_MARKERS.some((marker) => text.includes(marker));}diff --git a/justfile b/justfileindex 78dfe49bb..499d5531b 100644--- a/justfile+++ b/justfile@@ -183,6 +183,14 @@ e2e-provider *args:e2e-browser *args:SWITCH_E2E_LANE=browser console/tooling-e2e/scripts/ephemeral-stack.sh {{ args }}+# ── End-to-end harness, agentic lane, on a throwaway stack ─────────────────────+# As e2e-provider, then the scenario files in console/tooling-e2e/agentic: a+# driver model plays the human, a judge model grades the conversation. Needs+# SWITCH_E2E_DRIVER_MODEL, SWITCH_E2E_JUDGE_MODEL (exact model ids) and+# SWITCH_E2E_AGENTIC_BUDGET_USD. Spends real model calls, up to that limit.+e2e-agentic *args:+ SWITCH_E2E_LANE=agentic console/tooling-e2e/scripts/ephemeral-stack.sh {{ args }}+# ── Regenerate the server surface the E2E coverage gate reads ─────────────────# Every agent operation and gateway route, written to# console/tooling-e2e/surface/server-surface.json. CI fails when it is stale;