Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
E2E: readiness gate and no-errors-logged check per scenario
Petr Bauch33d83ed3188a
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex 7b569e810..333e3b9df 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -154,6 +154,7 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.| `MATTERMOST_TEAM` | for Mattermost scenarios | Team **slug** (e.g. `switch`), not its display name. || `SWITCH_E2E_STRICT` | no | `1` turns every "cannot run" into a failure. CI sets it. || `SWITCH_E2E_KEEP` | no | `1` leaves agents, rooms and channels behind for inspection. |+| `SWITCH_E2E_SERVER_LOG` | no | The server's log, written with `LOG_FORMAT=json`. When set, a scenario fails if the server logged an error while it ran. The stack script sets it. || `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. || `SWITCH_E2E_AGENT_DIR` | provider lane | Working directory of the session under test. When set, the agent's credentials are written into it (see below) and it is sent to Switch as the agent's `repo_dir`. || `SWITCH_E2E_MANIFEST` | provider lane | File the setup records its agent and room in, so a second process can reuse them. See "Seeding first". |@@ -214,6 +215,30 @@ Two things differ from a Console session:The controller's log goes to `build/e2e/ephemeral/controller.log`.+## Server health and errors++A scenario can pass every assertion while the server logs an error and carries+on: an event loop that catches one bad event, say. Two checks catch that.++- **Readiness.** `GET /health/ready` must answer 200 before a lane runs. Not+ ready means the lane cannot run: it skips, or fails under+ `SWITCH_E2E_STRICT=1`. Each lane's last test asks again, so a run that+ leaves the server unready fails.+- **No errors logged.** With `SWITCH_E2E_SERVER_LOG` set, the harness reads the+ server's JSON log after each scenario. The scenario fails if anything was+ logged at error level or above, including output that bypassed logging,+ between its start and 1.5 s after its end.+ - The check goes by time, because the work a scenario sets off runs on+ background tasks that carry no request id.+ - Each harness request carries `X-Request-Id: e2e-<run><scope>-<n>`, which+ the server stamps on every line it logs while serving the request. Each+ error is reported as coming from this scenario's request, another request,+ or background work.++ The stack script starts switch-core with `LOG_FORMAT=json` and sets the+ variable. Against a deployed environment the log is in its log backend, so+ the check is off, and the run's header says so.+## Coverage gateThe coverage gate catches functionality that ships without an end-to-enddiff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.shindex 8fed28270..fb51d331a 100755--- a/console/tooling-e2e/scripts/ephemeral-stack.sh+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh@@ -136,7 +136,10 @@ uv run --project core alembic -c core/alembic.ini upgrade head >"$OUT/migrate.lo|| { echo "migrations failed; see $OUT/migrate.log" >&2; exit 1; }echo "Starting switch-core from this checkout…"-GATEWAY_COOKIE_SECURE=false uv run --project core python -m switch_core.main \+# JSON, so the harness can read back what the server logged during each+# scenario (SWITCH_E2E_SERVER_LOG): one record per line, timed to the+# millisecond. `jq -r '"\(.timestamp) \(.status) \(.message)"'` reads it.+GATEWAY_COOKIE_SECURE=false LOG_FORMAT=json uv run --project core python -m switch_core.main \>"$OUT/switch-core.log" 2>&1 &SERVER_PID=$!for _ in $(seq 120); do curl -sf localhost:8000/health >/dev/null && break; sleep 1; done@@ -219,6 +222,8 @@ status=0if [ ${#PROVIDER_ENV[@]} -gt 0 ]; then export "${PROVIDER_ENV[@]}"; fi# The credentials the harness is issued, for the scan on exit.export SWITCH_E2E_SECRETS_FILE="$SECRETS"+ # What the server logs, so a scenario during which it logs an error fails.+ export SWITCH_E2E_SERVER_LOG="$OUT/switch-core.log"SWITCH_E2E_LANE="$LANE" \SWITCH_E2E_ENVIRONMENT=ephemeral \SWITCH_E2E_STRICT=1 \diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.tsindex f7ffe075f..81285a7f0 100644--- a/console/tooling-e2e/src/clients/switch-client.ts+++ b/console/tooling-e2e/src/clients/switch-client.ts@@ -147,6 +147,8 @@ export class SwitchClient {private readonly environment: Environment;private readonly issued: IssuedSecrets;private gatewayCookie: string | null = null;+ private requestTag: string | null = null;+ private requestCount = 0;constructor(env: SwitchEnv, environment: Environment, issued: IssuedSecrets) {this.apiUrl = env.apiUrl;@@ -159,6 +161,21 @@ export class SwitchClient {// ── plumbing ──────────────────────────────────────────────────────────────+ /**+ * Tag every request from now on with `X-Request-Id: <tag>-<n>`, which the+ * server stamps on each log line it writes while serving it. `null` stops.+ */+ tagRequests(tag: string | null): void {+ this.requestTag = tag;+ this.requestCount = 0;+ }++ private requestId(): Record<string, string> {+ if (this.requestTag === null) return {};+ this.requestCount += 1;+ return { 'X-Request-Id': `${this.requestTag}-${this.requestCount}` };+ }+private async request<T>(method: string, path: string, init: SwitchRequest = {}): Promise<T> {return (await this.rawRequest(method, path, init)).parsed as T;}@@ -169,7 +186,7 @@ export class SwitchClient {init: SwitchRequest): Promise<{ parsed: unknown; response: Response }> {assertAllowed(this.environment, method, path);- const headers: Record<string, string> = { Accept: 'application/json' };+ const headers: Record<string, string> = { Accept: 'application/json', ...this.requestId() };if (init.token) headers.Authorization = `Bearer ${init.token}`;if (init.cookie) headers.Cookie = init.cookie;if (init.body !== undefined) headers['Content-Type'] = 'application/json';@@ -199,7 +216,7 @@ export class SwitchClient {assertAllowed(this.environment, method, path);const response = await fetch(`${this.apiUrl}${path}`, {method,- headers: { Accept: 'application/json' },+ headers: { Accept: 'application/json', ...this.requestId() },redirect: 'manual',signal: AbortSignal.timeout(30_000),});@@ -216,6 +233,15 @@ export class SwitchClient {return new Set(body.flags.filter((flag) => flag.enabled).map((flag) => flag.key));}+ /**+ * `GET /health/ready`: whether the server reports itself ready to serve. It+ * answers 503 with the failing checks when it is not, which this throws as a+ * {@link SwitchHttpError} carrying them.+ */+ async ready(): Promise<void> {+ await this.request('GET', '/health/ready', { timeoutMs: 10_000 });+ }+/** `GET /health` — the reachability probe the suite skips on. */async health(): Promise<void> {const body = await this.request<{ status?: string }>('GET', '/health', {diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.tsindex 04c096591..4c25a0d9a 100644--- a/console/tooling-e2e/src/core/env.ts+++ b/console/tooling-e2e/src/core/env.ts@@ -97,6 +97,8 @@ export interface HarnessEnv {keepArtifacts: boolean;/** Where to record the credentials the server issues during the run; see `IssuedSecrets`. */secretsFile: string | null;+ /** The server's JSON log, when the harness can read it; see `ServerLog`. */+ serverLog: string | null;switch: SwitchEnv;/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */mattermost: MattermostEnv | null;@@ -233,6 +235,7 @@ export function loadEnv(strict: strictFrom(source, file),keepArtifacts: get('SWITCH_E2E_KEEP') === '1',secretsFile: get('SWITCH_E2E_SECRETS_FILE') ?? null,+ serverLog: get('SWITCH_E2E_SERVER_LOG') ?? null,switch: {apiUrl: stripTrailingSlash(required.SWITCH_API_URL as string),gatewayAdminEmail: required.SWITCH_GATEWAY_ADMIN_EMAIL as string,diff --git a/console/tooling-e2e/src/core/run.ts b/console/tooling-e2e/src/core/run.tsindex e76676412..19eca0312 100644--- a/console/tooling-e2e/src/core/run.ts+++ b/console/tooling-e2e/src/core/run.ts@@ -90,6 +90,14 @@ export class NameScope {return name;}+ /**+ * `e2e-<runId><scope>`: the prefix of every request id this scope's requests+ * carry, which is how a server log line names the scenario it came from.+ */+ get tag(): string {+ return `${ARTIFACT_PREFIX}${this.runId}${this.letter}`;+ }+/** A marker string unique to this scope, for grepping a transcript. */marker(label: string): string {return `SWITCH_E2E_${label.toUpperCase()}_${this.runId.toUpperCase()}${this.letter.toUpperCase()}`;diff --git a/console/tooling-e2e/src/core/server-log.test.ts b/console/tooling-e2e/src/core/server-log.test.tsnew file mode 100644index 000000000..dc322947b--- /dev/null+++ b/console/tooling-e2e/src/core/server-log.test.ts@@ -0,0 +1,78 @@+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import path from 'node:path';+import { afterEach, describe, expect, it } from 'vitest';+import { describeErrors, ServerLog } from './server-log.ts';++const dirs: string[] = [];++function logWith(lines: string[]): ServerLog {+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-server-log-'));+ dirs.push(dir);+ const file = path.join(dir, 'switch-core.log');+ writeFileSync(file, `${lines.join('\n')}\n`);+ return new ServerLog(file);+}++afterEach(() => {+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });+});++function record(timestamp: string, status: string, message: string, requestId?: string): string {+ return JSON.stringify({+ timestamp,+ status,+ message,+ logger: { name: 'switch_core.test' },+ ...(requestId ? { request_id: requestId } : {}),+ });+}++const at = (iso: string): number => Date.parse(iso);++describe('server log', () => {+ it('returns the errors logged inside the window, and nothing at lower levels or outside it', () => {+ const log = logWith([+ record('2026-10-19T10:00:00.000+00:00', 'error', 'before the scenario'),+ record('2026-10-19T10:00:01.000+00:00', 'info', 'ordinary'),+ record('2026-10-19T10:00:01.500+00:00', 'warning', 'degraded'),+ record('2026-10-19T10:00:02.000+00:00', 'error', 'delivery failed', 'e2e-abc123a-3'),+ record('2026-10-19T10:00:02.500+00:00', 'critical', 'loop died'),+ record('2026-10-19T10:00:09.000+00:00', 'error', 'after the scenario'),+ ]);+ const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));+ expect(errors.map((error) => error.message)).toEqual(['delivery failed', 'loop died']);+ expect(describeErrors(errors, 'e2e-abc123a')).toContain(+ "this scenario's request e2e-abc123a-3"+ );+ expect(describeErrors(errors, 'e2e-abc123a')).toContain('loop died (background work)');+ });++ it('counts output that bypassed logging as an error, at the time of the record before it', () => {+ const log = logWith([+ record('2026-10-19T10:00:02.000+00:00', 'info', 'serving'),+ 'Traceback (most recent call last):',+ record('2026-10-19T10:00:09.000+00:00', 'info', 'later'),+ ]);+ const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));+ expect(errors).toEqual([+ {+ timestamp: '2026-10-19T10:00:02.000+00:00',+ logger: '(unstructured output)',+ message: 'Traceback (most recent call last):',+ requestId: null,+ },+ ]);+ });++ it('refuses a log that is not JSON, rather than finding no errors in it', () => {+ const log = logWith(['2026-10-19T10:00:02 ERROR [switch_core] text format']);+ expect(() => log.errorsBetween(0, Date.now())).toThrow(/LOG_FORMAT=json/);+ });++ it('refuses a log it cannot read', () => {+ expect(() => new ServerLog('/nonexistent/switch-core.log').errorsBetween(0, 1)).toThrow(+ /Cannot read the server log/+ );+ });+});diff --git a/console/tooling-e2e/src/core/server-log.ts b/console/tooling-e2e/src/core/server-log.tsnew file mode 100644index 000000000..7be28af8f--- /dev/null+++ b/console/tooling-e2e/src/core/server-log.ts@@ -0,0 +1,180 @@+import { readFileSync } from 'node:fs';++/**+ * The server's own log, read back so a scenario fails when the server logged an+ * error while it ran, even if every assertion passed. A bug that logs and+ * carries on (an event loop catching one bad event) would otherwise pass every+ * scenario it breaks.+ *+ * Only a log the harness can read: the throwaway stack's `switch-core.log`,+ * written with `LOG_FORMAT=json` so each line is one record with a timestamp to+ * the millisecond. A deployed environment's log lives in its log backend, so+ * there the check is off and says so.+ *+ * Errors are found by time, not by request id: work a scenario sets off runs on+ * background tasks (a consumer delivering its message, a bridge relaying it)+ * that carry no request id. The id is attribution: each request the harness+ * sends carries `X-Request-Id: <scope tag>-<n>`, so an error logged while+ * serving one says which.+ */++export interface ServerLogError {+ timestamp: string;+ logger: string;+ message: string;+ /** The request it was logged while serving, or null for background work. */+ requestId: string | null;+}++/** How long after a scenario its background work may still log about it. */+export const SETTLE_MS = 1_500;++export class ServerLog {+ readonly path: string;++ constructor(path: string) {+ this.path = path;+ }++ /**+ * Every record at error level or above logged in `[startMs, endMs]`. A line+ * that is not a JSON record is output that bypassed logging (a traceback+ * printed to stderr, say) and counts as an error, at the time of the record+ * before it.+ */+ errorsBetween(startMs: number, endMs: number): ServerLogError[] {+ let raw: string;+ try {+ raw = readFileSync(this.path, 'utf8');+ } catch (error) {+ throw new Error(+ `Cannot read the server log at ${this.path} (SWITCH_E2E_SERVER_LOG): ` ++ `${error instanceof Error ? error.message : String(error)}`+ );+ }+ const errors: ServerLogError[] = [];+ let lastMs = Number.NEGATIVE_INFINITY;+ let lastTimestamp = '(before the first record)';+ let records = 0;+ for (const line of raw.split('\n')) {+ if (line.trim() === '') continue;+ const record = parseRecord(line);+ if (record === null) {+ if (lastMs >= startMs && lastMs <= endMs) {+ errors.push({+ timestamp: lastTimestamp,+ logger: '(unstructured output)',+ message: line.slice(0, 300),+ requestId: null,+ });+ }+ continue;+ }+ records += 1;+ lastMs = record.ms;+ lastTimestamp = record.timestamp;+ if (record.ms < startMs || record.ms > endMs) continue;+ if (record.status === 'error' || record.status === 'critical') {+ errors.push({+ timestamp: record.timestamp,+ logger: record.logger,+ message: record.message.slice(0, 500),+ requestId: record.requestId,+ });+ }+ }+ if (records === 0 && raw.trim() !== '') {+ throw new Error(+ `The server log at ${this.path} holds no JSON records: start the server ` ++ `with LOG_FORMAT=json for the harness to read it.`+ );+ }+ return errors;+ }+}++interface LogRecord {+ ms: number;+ timestamp: string;+ status: string;+ logger: string;+ message: string;+ requestId: string | null;+}++function parseRecord(line: string): LogRecord | null {+ if (!line.startsWith('{')) return null;+ let value: unknown;+ try {+ value = JSON.parse(line);+ } catch {+ return null;+ }+ if (typeof value !== 'object' || value === null) return null;+ const fields = value as { [key: string]: unknown };+ const timestamp = typeof fields.timestamp === 'string' ? fields.timestamp : null;+ const ms = timestamp === null ? Number.NaN : Date.parse(timestamp);+ if (timestamp === null || Number.isNaN(ms)) return null;+ const logger = fields.logger as { name?: unknown } | undefined;+ return {+ ms,+ timestamp,+ status: typeof fields.status === 'string' ? fields.status : '',+ logger: typeof logger?.name === 'string' ? logger.name : '?',+ message: typeof fields.message === 'string' ? fields.message : '',+ requestId: typeof fields.request_id === 'string' ? fields.request_id : null,+ };+}++/** One line per error, saying whose request it came from. */+export function describeErrors(errors: readonly ServerLogError[], tag: string): string {+ return errors+ .map((error) => {+ const source =+ error.requestId === null+ ? 'background work'+ : error.requestId.startsWith(`${tag}-`)+ ? `this scenario's request ${error.requestId}`+ : `request ${error.requestId}`;+ return `${error.timestamp} [${error.logger}] ${error.message} (${source})`;+ })+ .join('\n ');+}++/**+ * Watches one scenario: tags its requests, and on `finish` returns what the+ * server logged at error level while it ran, or null when there is no log to+ * read here.+ */+export class ScenarioObserver {+ readonly tag: string;+ private readonly log: ServerLog | null;+ private readonly client: { tagRequests(tag: string | null): void };+ private readonly startMs: number;++ constructor(+ serverLog: string | null,+ client: { tagRequests(tag: string | null): void },+ tag: string+ ) {+ this.tag = tag;+ this.log = serverLog === null ? null : new ServerLog(serverLog);+ this.client = client;+ this.startMs = Date.now();+ client.tagRequests(tag);+ }++ async finish(): Promise<ServerLogError[] | null> {+ this.client.tagRequests(null);+ if (this.log === null) return null;+ await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));+ return this.log.errorsBetween(this.startMs, Date.now());+ }+}++/** What the run's log says about the no-error check, for the run header. */+export function describeServerLogCheck(serverLog: string | null): string {+ return serverLog === null+ ? 'server-log check off: SWITCH_E2E_SERVER_LOG is not set, so errors the server logs go unseen'+ : `server-log check on: ${serverLog}`;+}diff --git a/console/tooling-e2e/src/core/suite.ts b/console/tooling-e2e/src/core/suite.tsindex 01d61f3bd..40cdcc0dd 100644--- a/console/tooling-e2e/src/core/suite.ts+++ b/console/tooling-e2e/src/core/suite.ts@@ -62,6 +62,11 @@ export async function resolveLane(lane: Lane): Promise<LaneResolution> {} catch (error) {return unable(`Switch server at ${env.switch.apiUrl} is unreachable: ${message(error)}`);}+ try {+ await switchClient.ready();+ } catch (error) {+ return unable(`Switch server at ${env.switch.apiUrl} is not ready: ${message(error)}`);+ }let mattermost: MattermostClient | null = null;if (env.mattermost) {diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.tsindex 7f28a0019..bf1e00df7 100644--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts@@ -2,6 +2,12 @@ import { beforeAll, describe, expect, it } from 'vitest';import { describeEnv } from '../core/env.ts';import { RunContext } from '../core/run.ts';import { admit, assertValidHeaders } from '../core/scenario.ts';+import {+ describeErrors,+ describeServerLogCheck,+ ScenarioObserver,+ type ServerLogError,+} from '../core/server-log.ts';import { resolveLane, type LaneResolution } from '../core/suite.ts';import { Fixtures } from '../fixtures/fixtures.ts';import { readSurface } from '../gate/gate.ts';@@ -24,7 +30,10 @@ let lane: LaneResolution;beforeAll(async () => {lane = await resolveLane('fast');- if (lane.target) console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);+ if (lane.target) {+ console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);+ console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);+ }}, 60_000);describe('fast lane', () => {@@ -46,6 +55,7 @@ describe('fast lane', () => {const names = run.scope();const fixtures = new Fixtures(env, names, switchClient, mattermost);const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);+ const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);let failure: unknown = null;try {await scenario.run({ env, names, switchClient, mattermost, fixtures, log });@@ -54,10 +64,16 @@ describe('fast lane', () => {}const leftovers = await fixtures.teardown();for (const line of leftovers) log(`teardown: ${line}`);+ const serverErrors = await observer.finish();+ if (serverErrors !== null && serverErrors.length > 0) {+ log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);+ }// The scenario's own failure is the root cause and is reported as is; a- // leak is only the verdict when nothing else went wrong.+ // leak, then a server error, is the verdict only when nothing else went+ // wrong.if (failure !== null) throw failure;if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);+ expectNoServerErrors(serverErrors, observer.tag);});}});@@ -68,12 +84,30 @@ describe('fast lane, derived checks', () => {it('every gateway route that takes a session refuses a request without one', async (context) => {if (!lane.target) return context.skip(lane.skip);if (lane.target.env.environment === 'prod') return context.skip('sends writes; not on prod');- const probe = await probeSessionRoutes(lane.target.switchClient, readSurface());+ const { env, switchClient } = lane.target;+ const observer = new ScenarioObserver(env.serverLog, switchClient, run.scope().tag);+ const probe = await probeSessionRoutes(switchClient, readSurface());+ const serverErrors = await observer.finish();console.log(`[session-required] probed ${probe.probed} session routes; ` +`${probe.notMounted} more are not mounted here (feature flags off)`);expect(probe.probed, 'no session route was probed').toBeGreaterThan(0);expect(probe.wrong, `answered without a session:\n ${probe.wrong.join('\n ')}\n`).toEqual([]);+ expectNoServerErrors(serverErrors, observer.tag);}, 120_000);++ // Last, so it sees whatever the run left the server in.+ it('the server is still ready after the run', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ await lane.target.switchClient.ready();+ });});++function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {+ if (errors === null) return;+ expect(+ errors,+ `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`+ ).toEqual([]);+}diff --git a/console/tooling-e2e/src/provider/provider.e2e.test.ts b/console/tooling-e2e/src/provider/provider.e2e.test.tsindex e621ff296..3c447e7db 100644--- a/console/tooling-e2e/src/provider/provider.e2e.test.ts+++ b/console/tooling-e2e/src/provider/provider.e2e.test.ts@@ -1,6 +1,8 @@import { afterAll, beforeAll, describe, expect, it } from 'vitest';import { describeEnv } from '../core/env.ts';+import { ARTIFACT_PREFIX } from '../core/run.ts';import { admit, assertValidHeaders } from '../core/scenario.ts';+import { describeErrors, describeServerLogCheck, ScenarioObserver } from '../core/server-log.ts';import { resolveLane, type LaneResolution } from '../core/suite.ts';import { setupHarness, teardownHarness, waitForSession, type Harness } from './harness.ts';import { formatResultsTable, SCENARIOS, type ScenarioResult } from './scenarios.ts';@@ -34,6 +36,7 @@ beforeAll(async () => {'',`Provider lane run ${harness.runId}`,` ${describeEnv(lane.target.env)}`,+ ` ${describeServerLogCheck(lane.target.env.serverLog)}`,` agent ${harness.agent.name} (${harness.agent.id})`,` bot @${harness.bot.username} (${harness.bot.id})`,` channel ${harness.channel.name} (${harness.channel.id})`,@@ -72,8 +75,14 @@ describe('provider lane', () => {return context.skip(admission.reason);}const h = harness!;+ const observer = new ScenarioObserver(+ lane.target.env.serverLog,+ h.switch,+ `${ARTIFACT_PREFIX}${h.runId}-${scenario.header.id}`+ );const result = await scenario.run(h);results.push(result);+ const serverErrors = await observer.finish();if (!result.ok) {console.log(`${result.name} transcript:\n` +@@ -83,6 +92,18 @@ describe('provider lane', () => {);}expect(result.ok, result.details).toBe(true);+ if (serverErrors !== null) {+ expect(+ serverErrors,+ `the server logged errors while this ran:\n ${describeErrors(serverErrors, observer.tag)}\n`+ ).toEqual([]);+ }});}++ // Last, so it sees whatever the run left the server in.+ it('the server is still ready after the run', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ await lane.target.switchClient.ready();+ });});