Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

E2E: readiness gate and no-errors-logged check per scenario

Petr Bauch33d83ed3188a
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index 7b569e810..333e3b9df 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -154,6 +154,7 @@ beside this file. Never commit one. `just e2e-fast` sets all of them itself.
| `MATTERMOST_TEAM` | for Mattermost scenarios | Team **slug** (e.g. `switch`), not its display name. |
| `SWITCH_E2E_STRICT` | no | `1` turns every "cannot run" into a failure. CI sets it. |
| `SWITCH_E2E_KEEP` | no | `1` leaves agents, rooms and channels behind for inspection. |
+| `SWITCH_E2E_SERVER_LOG` | no | The server's log, written with `LOG_FORMAT=json`. When set, a scenario fails if the server logged an error while it ran. The stack script sets it. |
| `SWITCH_E2E_SECRETS_FILE` | no | File to append each credential the server issues during the run to (agent keys, gateway sessions), for CI's scan of the output. The stack script and `e2e.yml` set it. |
| `SWITCH_E2E_AGENT_DIR` | provider lane | Working directory of the session under test. When set, the agent's credentials are written into it (see below) and it is sent to Switch as the agent's `repo_dir`. |
| `SWITCH_E2E_MANIFEST` | provider lane | File the setup records its agent and room in, so a second process can reuse them. See "Seeding first". |
@@ -214,6 +215,30 @@ Two things differ from a Console session:
The controller's log goes to `build/e2e/ephemeral/controller.log`.
+## Server health and errors
+
+A scenario can pass every assertion while the server logs an error and carries
+on: an event loop that catches one bad event, say. Two checks catch that.
+
+- **Readiness.** `GET /health/ready` must answer 200 before a lane runs. Not
+ ready means the lane cannot run: it skips, or fails under
+ `SWITCH_E2E_STRICT=1`. Each lane's last test asks again, so a run that
+ leaves the server unready fails.
+- **No errors logged.** With `SWITCH_E2E_SERVER_LOG` set, the harness reads the
+ server's JSON log after each scenario. The scenario fails if anything was
+ logged at error level or above, including output that bypassed logging,
+ between its start and 1.5 s after its end.
+ - The check goes by time, because the work a scenario sets off runs on
+ background tasks that carry no request id.
+ - Each harness request carries `X-Request-Id: e2e-<run><scope>-<n>`, which
+ the server stamps on every line it logs while serving the request. Each
+ error is reported as coming from this scenario's request, another request,
+ or background work.
+
+ The stack script starts switch-core with `LOG_FORMAT=json` and sets the
+ variable. Against a deployed environment the log is in its log backend, so
+ the check is off, and the run's header says so.
+
## Coverage gate
The coverage gate catches functionality that ships without an end-to-end
diff --git a/console/tooling-e2e/scripts/ephemeral-stack.sh b/console/tooling-e2e/scripts/ephemeral-stack.sh
index 8fed28270..fb51d331a 100755
--- a/console/tooling-e2e/scripts/ephemeral-stack.sh
+++ b/console/tooling-e2e/scripts/ephemeral-stack.sh
@@ -136,7 +136,10 @@ uv run --project core alembic -c core/alembic.ini upgrade head >"$OUT/migrate.lo
|| { echo "migrations failed; see $OUT/migrate.log" >&2; exit 1; }
echo "Starting switch-core from this checkout…"
-GATEWAY_COOKIE_SECURE=false uv run --project core python -m switch_core.main \
+# JSON, so the harness can read back what the server logged during each
+# scenario (SWITCH_E2E_SERVER_LOG): one record per line, timed to the
+# millisecond. `jq -r '"\(.timestamp) \(.status) \(.message)"'` reads it.
+GATEWAY_COOKIE_SECURE=false LOG_FORMAT=json uv run --project core python -m switch_core.main \
>"$OUT/switch-core.log" 2>&1 &
SERVER_PID=$!
for _ in $(seq 120); do curl -sf localhost:8000/health >/dev/null && break; sleep 1; done
@@ -219,6 +222,8 @@ status=0
if [ ${#PROVIDER_ENV[@]} -gt 0 ]; then export "${PROVIDER_ENV[@]}"; fi
# The credentials the harness is issued, for the scan on exit.
export SWITCH_E2E_SECRETS_FILE="$SECRETS"
+ # What the server logs, so a scenario during which it logs an error fails.
+ export SWITCH_E2E_SERVER_LOG="$OUT/switch-core.log"
SWITCH_E2E_LANE="$LANE" \
SWITCH_E2E_ENVIRONMENT=ephemeral \
SWITCH_E2E_STRICT=1 \
diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.ts
index f7ffe075f..81285a7f0 100644
--- a/console/tooling-e2e/src/clients/switch-client.ts
+++ b/console/tooling-e2e/src/clients/switch-client.ts
@@ -147,6 +147,8 @@ export class SwitchClient {
private readonly environment: Environment;
private readonly issued: IssuedSecrets;
private gatewayCookie: string | null = null;
+ private requestTag: string | null = null;
+ private requestCount = 0;
constructor(env: SwitchEnv, environment: Environment, issued: IssuedSecrets) {
this.apiUrl = env.apiUrl;
@@ -159,6 +161,21 @@ export class SwitchClient {
// ── plumbing ──────────────────────────────────────────────────────────────
+ /**
+ * Tag every request from now on with `X-Request-Id: <tag>-<n>`, which the
+ * server stamps on each log line it writes while serving it. `null` stops.
+ */
+ tagRequests(tag: string | null): void {
+ this.requestTag = tag;
+ this.requestCount = 0;
+ }
+
+ private requestId(): Record<string, string> {
+ if (this.requestTag === null) return {};
+ this.requestCount += 1;
+ return { 'X-Request-Id': `${this.requestTag}-${this.requestCount}` };
+ }
+
private async request<T>(method: string, path: string, init: SwitchRequest = {}): Promise<T> {
return (await this.rawRequest(method, path, init)).parsed as T;
}
@@ -169,7 +186,7 @@ export class SwitchClient {
init: SwitchRequest
): Promise<{ parsed: unknown; response: Response }> {
assertAllowed(this.environment, method, path);
- const headers: Record<string, string> = { Accept: 'application/json' };
+ const headers: Record<string, string> = { Accept: 'application/json', ...this.requestId() };
if (init.token) headers.Authorization = `Bearer ${init.token}`;
if (init.cookie) headers.Cookie = init.cookie;
if (init.body !== undefined) headers['Content-Type'] = 'application/json';
@@ -199,7 +216,7 @@ export class SwitchClient {
assertAllowed(this.environment, method, path);
const response = await fetch(`${this.apiUrl}${path}`, {
method,
- headers: { Accept: 'application/json' },
+ headers: { Accept: 'application/json', ...this.requestId() },
redirect: 'manual',
signal: AbortSignal.timeout(30_000),
});
@@ -216,6 +233,15 @@ export class SwitchClient {
return new Set(body.flags.filter((flag) => flag.enabled).map((flag) => flag.key));
}
+ /**
+ * `GET /health/ready`: whether the server reports itself ready to serve. It
+ * answers 503 with the failing checks when it is not, which this throws as a
+ * {@link SwitchHttpError} carrying them.
+ */
+ async ready(): Promise<void> {
+ await this.request('GET', '/health/ready', { timeoutMs: 10_000 });
+ }
+
/** `GET /health` — the reachability probe the suite skips on. */
async health(): Promise<void> {
const body = await this.request<{ status?: string }>('GET', '/health', {
diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.ts
index 04c096591..4c25a0d9a 100644
--- a/console/tooling-e2e/src/core/env.ts
+++ b/console/tooling-e2e/src/core/env.ts
@@ -97,6 +97,8 @@ export interface HarnessEnv {
keepArtifacts: boolean;
/** Where to record the credentials the server issues during the run; see `IssuedSecrets`. */
secretsFile: string | null;
+ /** The server's JSON log, when the harness can read it; see `ServerLog`. */
+ serverLog: string | null;
switch: SwitchEnv;
/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */
mattermost: MattermostEnv | null;
@@ -233,6 +235,7 @@ export function loadEnv(
strict: strictFrom(source, file),
keepArtifacts: get('SWITCH_E2E_KEEP') === '1',
secretsFile: get('SWITCH_E2E_SECRETS_FILE') ?? null,
+ serverLog: get('SWITCH_E2E_SERVER_LOG') ?? null,
switch: {
apiUrl: stripTrailingSlash(required.SWITCH_API_URL as string),
gatewayAdminEmail: required.SWITCH_GATEWAY_ADMIN_EMAIL as string,
diff --git a/console/tooling-e2e/src/core/run.ts b/console/tooling-e2e/src/core/run.ts
index e76676412..19eca0312 100644
--- a/console/tooling-e2e/src/core/run.ts
+++ b/console/tooling-e2e/src/core/run.ts
@@ -90,6 +90,14 @@ export class NameScope {
return name;
}
+ /**
+ * `e2e-<runId><scope>`: the prefix of every request id this scope's requests
+ * carry, which is how a server log line names the scenario it came from.
+ */
+ get tag(): string {
+ return `${ARTIFACT_PREFIX}${this.runId}${this.letter}`;
+ }
+
/** A marker string unique to this scope, for grepping a transcript. */
marker(label: string): string {
return `SWITCH_E2E_${label.toUpperCase()}_${this.runId.toUpperCase()}${this.letter.toUpperCase()}`;
diff --git a/console/tooling-e2e/src/core/server-log.test.ts b/console/tooling-e2e/src/core/server-log.test.ts
new file mode 100644
index 000000000..dc322947b
--- /dev/null
+++ b/console/tooling-e2e/src/core/server-log.test.ts
@@ -0,0 +1,78 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { describeErrors, ServerLog } from './server-log.ts';
+
+const dirs: string[] = [];
+
+function logWith(lines: string[]): ServerLog {
+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-server-log-'));
+ dirs.push(dir);
+ const file = path.join(dir, 'switch-core.log');
+ writeFileSync(file, `${lines.join('\n')}\n`);
+ return new ServerLog(file);
+}
+
+afterEach(() => {
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function record(timestamp: string, status: string, message: string, requestId?: string): string {
+ return JSON.stringify({
+ timestamp,
+ status,
+ message,
+ logger: { name: 'switch_core.test' },
+ ...(requestId ? { request_id: requestId } : {}),
+ });
+}
+
+const at = (iso: string): number => Date.parse(iso);
+
+describe('server log', () => {
+ it('returns the errors logged inside the window, and nothing at lower levels or outside it', () => {
+ const log = logWith([
+ record('2026-10-19T10:00:00.000+00:00', 'error', 'before the scenario'),
+ record('2026-10-19T10:00:01.000+00:00', 'info', 'ordinary'),
+ record('2026-10-19T10:00:01.500+00:00', 'warning', 'degraded'),
+ record('2026-10-19T10:00:02.000+00:00', 'error', 'delivery failed', 'e2e-abc123a-3'),
+ record('2026-10-19T10:00:02.500+00:00', 'critical', 'loop died'),
+ record('2026-10-19T10:00:09.000+00:00', 'error', 'after the scenario'),
+ ]);
+ const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));
+ expect(errors.map((error) => error.message)).toEqual(['delivery failed', 'loop died']);
+ expect(describeErrors(errors, 'e2e-abc123a')).toContain(
+ "this scenario's request e2e-abc123a-3"
+ );
+ expect(describeErrors(errors, 'e2e-abc123a')).toContain('loop died (background work)');
+ });
+
+ it('counts output that bypassed logging as an error, at the time of the record before it', () => {
+ const log = logWith([
+ record('2026-10-19T10:00:02.000+00:00', 'info', 'serving'),
+ 'Traceback (most recent call last):',
+ record('2026-10-19T10:00:09.000+00:00', 'info', 'later'),
+ ]);
+ const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));
+ expect(errors).toEqual([
+ {
+ timestamp: '2026-10-19T10:00:02.000+00:00',
+ logger: '(unstructured output)',
+ message: 'Traceback (most recent call last):',
+ requestId: null,
+ },
+ ]);
+ });
+
+ it('refuses a log that is not JSON, rather than finding no errors in it', () => {
+ const log = logWith(['2026-10-19T10:00:02 ERROR [switch_core] text format']);
+ expect(() => log.errorsBetween(0, Date.now())).toThrow(/LOG_FORMAT=json/);
+ });
+
+ it('refuses a log it cannot read', () => {
+ expect(() => new ServerLog('/nonexistent/switch-core.log').errorsBetween(0, 1)).toThrow(
+ /Cannot read the server log/
+ );
+ });
+});
diff --git a/console/tooling-e2e/src/core/server-log.ts b/console/tooling-e2e/src/core/server-log.ts
new file mode 100644
index 000000000..7be28af8f
--- /dev/null
+++ b/console/tooling-e2e/src/core/server-log.ts
@@ -0,0 +1,180 @@
+import { readFileSync } from 'node:fs';
+
+/**
+ * The server's own log, read back so a scenario fails when the server logged an
+ * error while it ran, even if every assertion passed. A bug that logs and
+ * carries on (an event loop catching one bad event) would otherwise pass every
+ * scenario it breaks.
+ *
+ * Only a log the harness can read: the throwaway stack's `switch-core.log`,
+ * written with `LOG_FORMAT=json` so each line is one record with a timestamp to
+ * the millisecond. A deployed environment's log lives in its log backend, so
+ * there the check is off and says so.
+ *
+ * Errors are found by time, not by request id: work a scenario sets off runs on
+ * background tasks (a consumer delivering its message, a bridge relaying it)
+ * that carry no request id. The id is attribution: each request the harness
+ * sends carries `X-Request-Id: <scope tag>-<n>`, so an error logged while
+ * serving one says which.
+ */
+
+export interface ServerLogError {
+ timestamp: string;
+ logger: string;
+ message: string;
+ /** The request it was logged while serving, or null for background work. */
+ requestId: string | null;
+}
+
+/** How long after a scenario its background work may still log about it. */
+export const SETTLE_MS = 1_500;
+
+export class ServerLog {
+ readonly path: string;
+
+ constructor(path: string) {
+ this.path = path;
+ }
+
+ /**
+ * Every record at error level or above logged in `[startMs, endMs]`. A line
+ * that is not a JSON record is output that bypassed logging (a traceback
+ * printed to stderr, say) and counts as an error, at the time of the record
+ * before it.
+ */
+ errorsBetween(startMs: number, endMs: number): ServerLogError[] {
+ let raw: string;
+ try {
+ raw = readFileSync(this.path, 'utf8');
+ } catch (error) {
+ throw new Error(
+ `Cannot read the server log at ${this.path} (SWITCH_E2E_SERVER_LOG): ` +
+ `${error instanceof Error ? error.message : String(error)}`
+ );
+ }
+ const errors: ServerLogError[] = [];
+ let lastMs = Number.NEGATIVE_INFINITY;
+ let lastTimestamp = '(before the first record)';
+ let records = 0;
+ for (const line of raw.split('\n')) {
+ if (line.trim() === '') continue;
+ const record = parseRecord(line);
+ if (record === null) {
+ if (lastMs >= startMs && lastMs <= endMs) {
+ errors.push({
+ timestamp: lastTimestamp,
+ logger: '(unstructured output)',
+ message: line.slice(0, 300),
+ requestId: null,
+ });
+ }
+ continue;
+ }
+ records += 1;
+ lastMs = record.ms;
+ lastTimestamp = record.timestamp;
+ if (record.ms < startMs || record.ms > endMs) continue;
+ if (record.status === 'error' || record.status === 'critical') {
+ errors.push({
+ timestamp: record.timestamp,
+ logger: record.logger,
+ message: record.message.slice(0, 500),
+ requestId: record.requestId,
+ });
+ }
+ }
+ if (records === 0 && raw.trim() !== '') {
+ throw new Error(
+ `The server log at ${this.path} holds no JSON records: start the server ` +
+ `with LOG_FORMAT=json for the harness to read it.`
+ );
+ }
+ return errors;
+ }
+}
+
+interface LogRecord {
+ ms: number;
+ timestamp: string;
+ status: string;
+ logger: string;
+ message: string;
+ requestId: string | null;
+}
+
+function parseRecord(line: string): LogRecord | null {
+ if (!line.startsWith('{')) return null;
+ let value: unknown;
+ try {
+ value = JSON.parse(line);
+ } catch {
+ return null;
+ }
+ if (typeof value !== 'object' || value === null) return null;
+ const fields = value as { [key: string]: unknown };
+ const timestamp = typeof fields.timestamp === 'string' ? fields.timestamp : null;
+ const ms = timestamp === null ? Number.NaN : Date.parse(timestamp);
+ if (timestamp === null || Number.isNaN(ms)) return null;
+ const logger = fields.logger as { name?: unknown } | undefined;
+ return {
+ ms,
+ timestamp,
+ status: typeof fields.status === 'string' ? fields.status : '',
+ logger: typeof logger?.name === 'string' ? logger.name : '?',
+ message: typeof fields.message === 'string' ? fields.message : '',
+ requestId: typeof fields.request_id === 'string' ? fields.request_id : null,
+ };
+}
+
+/** One line per error, saying whose request it came from. */
+export function describeErrors(errors: readonly ServerLogError[], tag: string): string {
+ return errors
+ .map((error) => {
+ const source =
+ error.requestId === null
+ ? 'background work'
+ : error.requestId.startsWith(`${tag}-`)
+ ? `this scenario's request ${error.requestId}`
+ : `request ${error.requestId}`;
+ return `${error.timestamp} [${error.logger}] ${error.message} (${source})`;
+ })
+ .join('\n ');
+}
+
+/**
+ * Watches one scenario: tags its requests, and on `finish` returns what the
+ * server logged at error level while it ran, or null when there is no log to
+ * read here.
+ */
+export class ScenarioObserver {
+ readonly tag: string;
+ private readonly log: ServerLog | null;
+ private readonly client: { tagRequests(tag: string | null): void };
+ private readonly startMs: number;
+
+ constructor(
+ serverLog: string | null,
+ client: { tagRequests(tag: string | null): void },
+ tag: string
+ ) {
+ this.tag = tag;
+ this.log = serverLog === null ? null : new ServerLog(serverLog);
+ this.client = client;
+ this.startMs = Date.now();
+ client.tagRequests(tag);
+ }
+
+ async finish(): Promise<ServerLogError[] | null> {
+ this.client.tagRequests(null);
+ if (this.log === null) return null;
+ await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));
+ return this.log.errorsBetween(this.startMs, Date.now());
+ }
+}
+
+/** What the run's log says about the no-error check, for the run header. */
+export function describeServerLogCheck(serverLog: string | null): string {
+ return serverLog === null
+ ? 'server-log check off: SWITCH_E2E_SERVER_LOG is not set, so errors the server logs go unseen'
+ : `server-log check on: ${serverLog}`;
+}
diff --git a/console/tooling-e2e/src/core/suite.ts b/console/tooling-e2e/src/core/suite.ts
index 01d61f3bd..40cdcc0dd 100644
--- a/console/tooling-e2e/src/core/suite.ts
+++ b/console/tooling-e2e/src/core/suite.ts
@@ -62,6 +62,11 @@ export async function resolveLane(lane: Lane): Promise<LaneResolution> {
} catch (error) {
return unable(`Switch server at ${env.switch.apiUrl} is unreachable: ${message(error)}`);
}
+ try {
+ await switchClient.ready();
+ } catch (error) {
+ return unable(`Switch server at ${env.switch.apiUrl} is not ready: ${message(error)}`);
+ }
let mattermost: MattermostClient | null = null;
if (env.mattermost) {
diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.ts
index 7f28a0019..bf1e00df7 100644
--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts
+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts
@@ -2,6 +2,12 @@ import { beforeAll, describe, expect, it } from 'vitest';
import { describeEnv } from '../core/env.ts';
import { RunContext } from '../core/run.ts';
import { admit, assertValidHeaders } from '../core/scenario.ts';
+import {
+ describeErrors,
+ describeServerLogCheck,
+ ScenarioObserver,
+ type ServerLogError,
+} from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { Fixtures } from '../fixtures/fixtures.ts';
import { readSurface } from '../gate/gate.ts';
@@ -24,7 +30,10 @@ let lane: LaneResolution;
beforeAll(async () => {
lane = await resolveLane('fast');
- if (lane.target) console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);
+ if (lane.target) {
+ console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);
+ console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);
+ }
}, 60_000);
describe('fast lane', () => {
@@ -46,6 +55,7 @@ describe('fast lane', () => {
const names = run.scope();
const fixtures = new Fixtures(env, names, switchClient, mattermost);
const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
+ const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);
let failure: unknown = null;
try {
await scenario.run({ env, names, switchClient, mattermost, fixtures, log });
@@ -54,10 +64,16 @@ describe('fast lane', () => {
}
const leftovers = await fixtures.teardown();
for (const line of leftovers) log(`teardown: ${line}`);
+ const serverErrors = await observer.finish();
+ if (serverErrors !== null && serverErrors.length > 0) {
+ log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);
+ }
// The scenario's own failure is the root cause and is reported as is; a
- // leak is only the verdict when nothing else went wrong.
+ // leak, then a server error, is the verdict only when nothing else went
+ // wrong.
if (failure !== null) throw failure;
if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);
+ expectNoServerErrors(serverErrors, observer.tag);
});
}
});
@@ -68,12 +84,30 @@ describe('fast lane, derived checks', () => {
it('every gateway route that takes a session refuses a request without one', async (context) => {
if (!lane.target) return context.skip(lane.skip);
if (lane.target.env.environment === 'prod') return context.skip('sends writes; not on prod');
- const probe = await probeSessionRoutes(lane.target.switchClient, readSurface());
+ const { env, switchClient } = lane.target;
+ const observer = new ScenarioObserver(env.serverLog, switchClient, run.scope().tag);
+ const probe = await probeSessionRoutes(switchClient, readSurface());
+ const serverErrors = await observer.finish();
console.log(
`[session-required] probed ${probe.probed} session routes; ` +
`${probe.notMounted} more are not mounted here (feature flags off)`
);
expect(probe.probed, 'no session route was probed').toBeGreaterThan(0);
expect(probe.wrong, `answered without a session:\n ${probe.wrong.join('\n ')}\n`).toEqual([]);
+ expectNoServerErrors(serverErrors, observer.tag);
}, 120_000);
+
+ // Last, so it sees whatever the run left the server in.
+ it('the server is still ready after the run', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ await lane.target.switchClient.ready();
+ });
});
+
+function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {
+ if (errors === null) return;
+ expect(
+ errors,
+ `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`
+ ).toEqual([]);
+}
diff --git a/console/tooling-e2e/src/provider/provider.e2e.test.ts b/console/tooling-e2e/src/provider/provider.e2e.test.ts
index e621ff296..3c447e7db 100644
--- a/console/tooling-e2e/src/provider/provider.e2e.test.ts
+++ b/console/tooling-e2e/src/provider/provider.e2e.test.ts
@@ -1,6 +1,8 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { describeEnv } from '../core/env.ts';
+import { ARTIFACT_PREFIX } from '../core/run.ts';
import { admit, assertValidHeaders } from '../core/scenario.ts';
+import { describeErrors, describeServerLogCheck, ScenarioObserver } from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { setupHarness, teardownHarness, waitForSession, type Harness } from './harness.ts';
import { formatResultsTable, SCENARIOS, type ScenarioResult } from './scenarios.ts';
@@ -34,6 +36,7 @@ beforeAll(async () => {
'',
`Provider lane run ${harness.runId}`,
` ${describeEnv(lane.target.env)}`,
+ ` ${describeServerLogCheck(lane.target.env.serverLog)}`,
` agent ${harness.agent.name} (${harness.agent.id})`,
` bot @${harness.bot.username} (${harness.bot.id})`,
` channel ${harness.channel.name} (${harness.channel.id})`,
@@ -72,8 +75,14 @@ describe('provider lane', () => {
return context.skip(admission.reason);
}
const h = harness!;
+ const observer = new ScenarioObserver(
+ lane.target.env.serverLog,
+ h.switch,
+ `${ARTIFACT_PREFIX}${h.runId}-${scenario.header.id}`
+ );
const result = await scenario.run(h);
results.push(result);
+ const serverErrors = await observer.finish();
if (!result.ok) {
console.log(
`${result.name} transcript:\n` +
@@ -83,6 +92,18 @@ describe('provider lane', () => {
);
}
expect(result.ok, result.details).toBe(true);
+ if (serverErrors !== null) {
+ expect(
+ serverErrors,
+ `the server logged errors while this ran:\n ${describeErrors(serverErrors, observer.tag)}\n`
+ ).toEqual([]);
+ }
});
}
+
+ // Last, so it sees whatever the run left the server in.
+ it('the server is still ready after the run', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ await lane.target.switchClient.ready();
+ });
});