Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Read the server log by position, and check what is logged outside scenarios
Petr Bauch7225d1f50528
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex 333e3b9df..ba4b01eb4 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -225,15 +225,24 @@ on: an event loop that catches one bad event, say. Two checks catch that.`SWITCH_E2E_STRICT=1`. Each lane's last test asks again, so a run thatleaves the server unready fails.- **No errors logged.** With `SWITCH_E2E_SERVER_LOG` set, the harness reads the- server's JSON log after each scenario. The scenario fails if anything was- logged at error level or above, including output that bypassed logging,- between its start and 1.5 s after its end.- - The check goes by time, because the work a scenario sets off runs on- background tasks that carry no request id.+ server's JSON log as the lane runs. Each read starts where the previous one+ stopped, and a line the server is still writing waits for the next read.+ - A scenario fails if anything in the lines written while it ran is at error+ level or above.+ - Whatever is logged outside the scenarios also counts: during setup,+ between scenarios (late background work, say), and in the 1.5 s after the+ last one. It fails the lane's last test, "the server logged no errors+ outside the scenarios". The provider lane reads once more after its+ teardown, and fails the run if that teardown made the server log an error.+ - Output that bypassed logging counts when it reads as a failure: a+ traceback, an exception a thread died of, a fatal interpreter error, or a+ task destroyed while pending. Anything else that is not a JSON record (a+ dependency's warning, a stray print) does not.- Each harness request carries `X-Request-Id: e2e-<run><scope>-<n>`, whichthe server stamps on every line it logs while serving the request. Eacherror is reported as coming from this scenario's request, another request,- or background work.+ or background work. That holds wherever the error lands, so a late one+ still names the scenario that caused it.The stack script starts switch-core with `LOG_FORMAT=json` and sets thevariable. Against a deployed environment the log is in its log backend, sodiff --git a/console/tooling-e2e/src/core/server-log.test.ts b/console/tooling-e2e/src/core/server-log.test.tsindex dc322947b..b8215b0b1 100644--- a/console/tooling-e2e/src/core/server-log.test.ts+++ b/console/tooling-e2e/src/core/server-log.test.ts@@ -1,26 +1,30 @@-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { appendFileSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import path from 'node:path';import { afterEach, describe, expect, it } from 'vitest';-import { describeErrors, ServerLog } from './server-log.ts';+import { describeErrors, ServerLogCursor, ServerLogWatch } from './server-log.ts';const dirs: string[] = [];-function logWith(lines: string[]): ServerLog {+function logFile(lines: string[]): string {const dir = mkdtempSync(path.join(tmpdir(), 'e2e-server-log-'));dirs.push(dir);const file = path.join(dir, 'switch-core.log');- writeFileSync(file, `${lines.join('\n')}\n`);- return new ServerLog(file);+ writeFileSync(file, lines.map((line) => `${line}\n`).join(''));+ return file;+}++function append(file: string, lines: string[]): void {+ appendFileSync(file, lines.map((line) => `${line}\n`).join(''));}afterEach(() => {for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });});-function record(timestamp: string, status: string, message: string, requestId?: string): string {+function record(status: string, message: string, requestId?: string): string {return JSON.stringify({- timestamp,+ timestamp: '2026-10-19T10:00:00.000+00:00',status,message,logger: { name: 'switch_core.test' },@@ -28,51 +32,101 @@ function record(timestamp: string, status: string, message: string, requestId?:});}-const at = (iso: string): number => Date.parse(iso);+const tagger = { tagRequests: (_tag: string | null) => {} };-describe('server log', () => {- it('returns the errors logged inside the window, and nothing at lower levels or outside it', () => {- const log = logWith([- record('2026-10-19T10:00:00.000+00:00', 'error', 'before the scenario'),- record('2026-10-19T10:00:01.000+00:00', 'info', 'ordinary'),- record('2026-10-19T10:00:01.500+00:00', 'warning', 'degraded'),- record('2026-10-19T10:00:02.000+00:00', 'error', 'delivery failed', 'e2e-abc123a-3'),- record('2026-10-19T10:00:02.500+00:00', 'critical', 'loop died'),- record('2026-10-19T10:00:09.000+00:00', 'error', 'after the scenario'),+describe('server log cursor', () => {+ it('reads only what was written after it, at error level and above', () => {+ const file = logFile([record('error', 'at boot, before the lane')]);+ const cursor = ServerLogCursor.atEnd(file);+ append(file, [+ record('info', 'ordinary'),+ record('warning', 'degraded'),+ record('error', 'delivery failed', 'e2e-abc123a-3'),+ record('critical', 'loop died'),]);- const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));+ const errors = cursor.take();expect(errors.map((error) => error.message)).toEqual(['delivery failed', 'loop died']);expect(describeErrors(errors, 'e2e-abc123a')).toContain("this scenario's request e2e-abc123a-3");expect(describeErrors(errors, 'e2e-abc123a')).toContain('loop died (background work)');+ expect(cursor.take()).toEqual([]);+ });++ it('leaves a line the server is still writing for the next read', () => {+ const file = logFile([record('info', 'booted')]);+ const cursor = ServerLogCursor.atEnd(file);+ const line = record('error', 'half written');+ appendFileSync(file, line.slice(0, 20));+ expect(cursor.take()).toEqual([]);+ appendFileSync(file, `${line.slice(20)}\n`);+ expect(cursor.take().map((error) => error.message)).toEqual(['half written']);});- it('counts output that bypassed logging as an error, at the time of the record before it', () => {- const log = logWith([- record('2026-10-19T10:00:02.000+00:00', 'info', 'serving'),+ it('counts a traceback that bypassed logging, whenever it was printed, and ignores other output', () => {+ const file = logFile([record('info', 'booted')]);+ const cursor = ServerLogCursor.atEnd(file);+ append(file, [+ 'DeprecationWarning: something a dependency still does',+ 'Exception in thread mm-ws-agent:','Traceback (most recent call last):',- record('2026-10-19T10:00:09.000+00:00', 'info', 'later'),+ ' File "adapter.py", line 1, in _run_ws',+ 'RuntimeError: boom',]);- const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));- expect(errors).toEqual([- {- timestamp: '2026-10-19T10:00:02.000+00:00',- logger: '(unstructured output)',- message: 'Traceback (most recent call last):',- requestId: null,- },+ expect(cursor.take().map((error) => error.message)).toEqual([+ 'Exception in thread mm-ws-agent:',+ 'Traceback (most recent call last):',]);});it('refuses a log that is not JSON, rather than finding no errors in it', () => {- const log = logWith(['2026-10-19T10:00:02 ERROR [switch_core] text format']);- expect(() => log.errorsBetween(0, Date.now())).toThrow(/LOG_FORMAT=json/);+ const file = logFile(['2026-10-19T10:00:02 ERROR [switch_core] text format']);+ expect(() => ServerLogCursor.atEnd(file)).toThrow(/LOG_FORMAT=json/);});- it('refuses a log it cannot read', () => {- expect(() => new ServerLog('/nonexistent/switch-core.log').errorsBetween(0, 1)).toThrow(+ it('refuses a log it cannot read, and one that shrank under it', () => {+ expect(() => ServerLogCursor.atEnd('/nonexistent/switch-core.log')).toThrow(/Cannot read the server log/);+ const file = logFile([record('info', 'booted'), record('info', 'more')]);+ const cursor = ServerLogCursor.atEnd(file);+ truncateSync(file, 10);+ expect(() => cursor.take()).toThrow(/rotated or truncated/);+ });+});++describe('server log watch', () => {+ it('sets aside what was logged between scenarios, for the end-of-lane sweep', async () => {+ const file = logFile([record('info', 'booted')]);+ const watch = new ServerLogWatch(file);+ append(file, [record('error', 'during setup')]);+ const observer = watch.begin(tagger, 'e2e-abc123b');+ append(file, [record('error', 'in the scenario', 'e2e-abc123b-1')]);+ observer.stop();+ expect(observer.errors(null, () => {})?.map((error) => error.message)).toEqual([+ 'in the scenario',+ ]);+ append(file, [record('error', 'after the last scenario')]);+ expect((await watch.sweep())?.map((error) => error.message)).toEqual([+ 'during setup',+ 'after the last scenario',+ ]);+ });++ it("never lets an unreadable log replace the scenario's own failure", () => {+ const file = logFile([record('info', 'booted'), record('info', 'more')]);+ const watch = new ServerLogWatch(file);+ const observer = watch.begin(tagger, 'e2e-abc123b');+ truncateSync(file, 10);+ const said: string[] = [];+ expect(observer.errors(new Error('the scenario failed'), (line) => said.push(line))).toBe(null);+ expect(said.join('\n')).toMatch(/could not read the server log/);+ expect(() => observer.errors(null, () => {})).toThrow(/rotated or truncated/);+ });++ it('is off, reading nothing, without a log', async () => {+ const watch = new ServerLogWatch(null);+ expect(watch.begin(tagger, 'e2e-abc123b').errors(null, () => {})).toBe(null);+ expect(await watch.sweep()).toBe(null);});});diff --git a/console/tooling-e2e/src/core/server-log.ts b/console/tooling-e2e/src/core/server-log.tsindex 7be28af8f..333317241 100644--- a/console/tooling-e2e/src/core/server-log.ts+++ b/console/tooling-e2e/src/core/server-log.ts@@ -1,4 +1,5 @@-import { readFileSync } from 'node:fs';+import { closeSync, fstatSync, openSync, readSync } from 'node:fs';+import { expect } from 'vitest';/*** The server's own log, read back so a scenario fails when the server logged an@@ -7,15 +8,19 @@ import { readFileSync } from 'node:fs';* scenario it breaks.** Only a log the harness can read: the throwaway stack's `switch-core.log`,- * written with `LOG_FORMAT=json` so each line is one record with a timestamp to- * the millisecond. A deployed environment's log lives in its log backend, so- * there the check is off and says so.+ * written with `LOG_FORMAT=json`, one record per line. A deployed+ * environment's log lives in its log backend, so there the check is off and+ * says so.*- * Errors are found by time, not by request id: work a scenario sets off runs on- * background tasks (a consumer delivering its message, a bridge relaying it)- * that carry no request id. The id is attribution: each request the harness- * sends carries `X-Request-Id: <scope tag>-<n>`, so an error logged while- * serving one says which.+ * The log is read by position, not by time: a cursor starts at the end of the+ * log when the lane starts, and each read takes the complete lines written+ * since the one before. Work a scenario sets off runs on background tasks+ * (a consumer delivering its message, a bridge relaying it) that carry no+ * request id and may log after the scenario has returned, so nothing is+ * skipped: what is logged between scenarios, during setup, or after the last+ * one is gathered as "outside the scenarios" and checked at the end of the+ * lane. Each request the harness sends carries `X-Request-Id: <scope tag>-<n>`,+ * which is how an error says which scenario it came from, wherever it lands.*/export interface ServerLogError {@@ -26,43 +31,67 @@ export interface ServerLogError {requestId: string | null;}-/** How long after a scenario its background work may still log about it. */+/**+ * How long the end-of-lane sweep waits for background work the last scenario+ * set off to log about it.+ */export const SETTLE_MS = 1_500;-export class ServerLog {+/**+ * Output that bypassed logging and means something failed: an uncaught+ * exception's traceback, one a thread reported on its way out, the+ * interpreter dying, or asyncio collecting a task still pending. Any other+ * line that is not a JSON record (a dependency's DeprecationWarning, a stray+ * print, uv's own messages) is not an error.+ */+const UNLOGGED_FAILURE =+ /^(Traceback \(most recent call last\):|Exception in thread |Fatal Python error:|Task was destroyed but it is pending!)/;++/** A position in the server log, always at the start of a line. */+export class ServerLogCursor {readonly path: string;+ private offset: number;- constructor(path: string) {+ private constructor(path: string, offset: number) {this.path = path;+ this.offset = offset;}/**- * Every record at error level or above logged in `[startMs, endMs]`. A line- * that is not a JSON record is output that bypassed logging (a traceback- * printed to stderr, say) and counts as an error, at the time of the record- * before it.+ * A cursor at the end of the log's last complete line. The server has booted+ * by now, so a log with no JSON record in it was not written with+ * `LOG_FORMAT=json` and would read as error-free whatever happened: refused.*/- errorsBetween(startMs: number, endMs: number): ServerLogError[] {- let raw: string;- try {- raw = readFileSync(this.path, 'utf8');- } catch (error) {+ static atEnd(path: string): ServerLogCursor {+ const raw = readLog(path, 0);+ const complete = raw.subarray(0, raw.lastIndexOf(0x0a) + 1).toString('utf8');+ if (!complete.split('\n').some((line) => parseRecord(line) !== null)) {throw new Error(- `Cannot read the server log at ${this.path} (SWITCH_E2E_SERVER_LOG): ` +- `${error instanceof Error ? error.message : String(error)}`+ `The server log at ${path} holds no JSON records: start the server ` ++ `with LOG_FORMAT=json for the harness to read it.`);}+ return new ServerLogCursor(path, Buffer.byteLength(complete, 'utf8'));+ }++ /**+ * The errors in every complete line written since the last read, moving past+ * them. A last line with no newline yet is one the server is still writing,+ * and is left for the next read.+ */+ take(): ServerLogError[] {+ const raw = readLog(this.path, this.offset);+ const end = raw.lastIndexOf(0x0a) + 1;+ this.offset += end;const errors: ServerLogError[] = [];- let lastMs = Number.NEGATIVE_INFINITY;- let lastTimestamp = '(before the first record)';- let records = 0;- for (const line of raw.split('\n')) {+ let lastTimestamp = '(no record yet)';+ for (const line of raw.subarray(0, end).toString('utf8').split('\n')) {if (line.trim() === '') continue;const record = parseRecord(line);if (record === null) {- if (lastMs >= startMs && lastMs <= endMs) {+ if (UNLOGGED_FAILURE.test(line)) {errors.push({- timestamp: lastTimestamp,+ timestamp: `after ${lastTimestamp}`,logger: '(unstructured output)',message: line.slice(0, 300),requestId: null,@@ -70,10 +99,7 @@ export class ServerLog {}continue;}- records += 1;- lastMs = record.ms;lastTimestamp = record.timestamp;- if (record.ms < startMs || record.ms > endMs) continue;if (record.status === 'error' || record.status === 'critical') {errors.push({timestamp: record.timestamp,@@ -83,18 +109,42 @@ export class ServerLog {});}}- if (records === 0 && raw.trim() !== '') {+ return errors;+ }+}++function readLog(path: string, offset: number): Buffer {+ let fd: number;+ try {+ fd = openSync(path, 'r');+ } catch (error) {+ throw new Error(+ `Cannot read the server log at ${path} (SWITCH_E2E_SERVER_LOG): ` ++ `${error instanceof Error ? error.message : String(error)}`+ );+ }+ try {+ const size = fstatSync(fd).size;+ if (size < offset) {throw new Error(- `The server log at ${this.path} holds no JSON records: start the server ` +- `with LOG_FORMAT=json for the harness to read it.`+ `The server log at ${path} shrank below what was already read (${size} < ${offset} bytes): ` ++ `was it rotated or truncated during the run?`);}- return errors;+ const buffer = Buffer.alloc(size - offset);+ let read = 0;+ while (read < buffer.length) {+ const n = readSync(fd, buffer, read, buffer.length - read, offset + read);+ if (n === 0) break;+ read += n;+ }+ return buffer.subarray(0, read);+ } finally {+ closeSync(fd);}}interface LogRecord {- ms: number;timestamp: string;status: string;logger: string;@@ -112,13 +162,10 @@ function parseRecord(line: string): LogRecord | null {}if (typeof value !== 'object' || value === null) return null;const fields = value as { [key: string]: unknown };- const timestamp = typeof fields.timestamp === 'string' ? fields.timestamp : null;- const ms = timestamp === null ? Number.NaN : Date.parse(timestamp);- if (timestamp === null || Number.isNaN(ms)) return null;+ if (typeof fields.timestamp !== 'string') return null;const logger = fields.logger as { name?: unknown } | undefined;return {- ms,- timestamp,+ timestamp: fields.timestamp,status: typeof fields.status === 'string' ? fields.status : '',logger: typeof logger?.name === 'string' ? logger.name : '?',message: typeof fields.message === 'string' ? fields.message : '',@@ -127,13 +174,13 @@ function parseRecord(line: string): LogRecord | null {}/** One line per error, saying whose request it came from. */-export function describeErrors(errors: readonly ServerLogError[], tag: string): string {+export function describeErrors(errors: readonly ServerLogError[], tag: string | null): string {return errors.map((error) => {const source =error.requestId === null? 'background work'- : error.requestId.startsWith(`${tag}-`)+ : tag !== null && error.requestId.startsWith(`${tag}-`)? `this scenario's request ${error.requestId}`: `request ${error.requestId}`;return `${error.timestamp} [${error.logger}] ${error.message} (${source})`;@@ -141,35 +188,97 @@ export function describeErrors(errors: readonly ServerLogError[], tag: string):.join('\n ');}+interface Taggable {+ tagRequests(tag: string | null): void;+}+/**- * Watches one scenario: tags its requests, and on `finish` returns what the- * server logged at error level while it ran, or null when there is no log to- * read here.+ * One lane's watch over the server log: per scenario, and for everything+ * logged outside them. Off (every read is null) when there is no log to read+ * here.*/+export class ServerLogWatch {+ private readonly cursor: ServerLogCursor | null;+ private readonly outside: ServerLogError[] = [];++ constructor(serverLog: string | null) {+ this.cursor = serverLog === null ? null : ServerLogCursor.atEnd(serverLog);+ }++ /**+ * Start watching a scenario: what was logged since the last read is set+ * aside as outside any scenario, and the client tags its requests with+ * `tag` until the observer stops.+ */+ begin(client: Taggable, tag: string): ScenarioObserver {+ if (this.cursor !== null) this.outside.push(...this.cursor.take());+ client.tagRequests(tag);+ return new ScenarioObserver(this.cursor, client, tag);+ }++ /**+ * Everything logged outside the scenarios: during setup, between them, and+ * after the last, once its background work has had `SETTLE_MS` to log.+ * Null when the check is off.+ */+ async sweep(): Promise<ServerLogError[] | null> {+ if (this.cursor === null) return null;+ await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));+ this.outside.push(...this.cursor.take());+ return this.outside.splice(0);+ }+}+export class ScenarioObserver {readonly tag: string;- private readonly log: ServerLog | null;- private readonly client: { tagRequests(tag: string | null): void };- private readonly startMs: number;-- constructor(- serverLog: string | null,- client: { tagRequests(tag: string | null): void },- tag: string- ) {- this.tag = tag;- this.log = serverLog === null ? null : new ServerLog(serverLog);+ private readonly cursor: ServerLogCursor | null;+ private readonly client: Taggable;++ constructor(cursor: ServerLogCursor | null, client: Taggable, tag: string) {+ this.cursor = cursor;this.client = client;- this.startMs = Date.now();- client.tagRequests(tag);+ this.tag = tag;}- async finish(): Promise<ServerLogError[] | null> {+ /** Stop tagging the client's requests. Call it in a `finally`. */+ stop(): void {this.client.tagRequests(null);- if (this.log === null) return null;- await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));- return this.log.errorsBetween(this.startMs, Date.now());}++ /**+ * What the server logged at error level since the scenario began, or null+ * when the check is off.+ *+ * The scenario's own failure is the root cause, so a log that cannot be read+ * never replaces it: with `failure` set, that read error is only logged.+ */+ errors(failure: unknown, log: (message: string) => void): ServerLogError[] | null {+ if (this.cursor === null) return null;+ let errors: ServerLogError[];+ try {+ errors = this.cursor.take();+ } catch (error) {+ if (failure === null) throw error;+ log(+ `could not read the server log: ${error instanceof Error ? error.message : String(error)}`+ );+ return null;+ }+ if (errors.length > 0) log(`the server logged errors:\n ${describeErrors(errors, this.tag)}`);+ return errors;+ }+}++/** Fail when the server logged errors; `null` (the check is off) passes. */+export function expectNoServerErrors(+ errors: ServerLogError[] | null,+ tag: string | null,+ when: string+): void {+ if (errors === null) return;+ expect(errors, `the server logged errors ${when}:\n ${describeErrors(errors, tag)}\n`).toEqual(+ []+ );}/** What the run's log says about the no-error check, for the run header. */diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.tsindex bf1e00df7..3ce722f62 100644--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts@@ -3,10 +3,9 @@ import { describeEnv } from '../core/env.ts';import { RunContext } from '../core/run.ts';import { admit, assertValidHeaders } from '../core/scenario.ts';import {- describeErrors,describeServerLogCheck,- ScenarioObserver,- type ServerLogError,+ expectNoServerErrors,+ ServerLogWatch,} from '../core/server-log.ts';import { resolveLane, type LaneResolution } from '../core/suite.ts';import { Fixtures } from '../fixtures/fixtures.ts';@@ -27,12 +26,14 @@ assertValidHeaders(FAST_SCENARIOS.map((scenario) => scenario.header));const run = new RunContext();let lane: LaneResolution;+let watch: ServerLogWatch;beforeAll(async () => {lane = await resolveLane('fast');if (lane.target) {console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);+ watch = new ServerLogWatch(lane.target.env.serverLog);}}, 60_000);@@ -55,25 +56,27 @@ describe('fast lane', () => {const names = run.scope();const fixtures = new Fixtures(env, names, switchClient, mattermost);const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);- const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);+ const observer = watch.begin(switchClient, names.tag);let failure: unknown = null;+ let leftovers: string[];try {- await scenario.run({ env, names, switchClient, mattermost, fixtures, log });- } catch (error) {- failure = error;+ try {+ await scenario.run({ env, names, switchClient, mattermost, fixtures, log });+ } catch (error) {+ failure = error;+ }+ leftovers = await fixtures.teardown();+ } finally {+ observer.stop();}- const leftovers = await fixtures.teardown();for (const line of leftovers) log(`teardown: ${line}`);- const serverErrors = await observer.finish();- if (serverErrors !== null && serverErrors.length > 0) {- log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);- }+ const serverErrors = observer.errors(failure, log);// The scenario's own failure is the root cause and is reported as is; a// leak, then a server error, is the verdict only when nothing else went// wrong.if (failure !== null) throw failure;if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);- expectNoServerErrors(serverErrors, observer.tag);+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');});}});@@ -84,30 +87,32 @@ describe('fast lane, derived checks', () => {it('every gateway route that takes a session refuses a request without one', async (context) => {if (!lane.target) return context.skip(lane.skip);if (lane.target.env.environment === 'prod') return context.skip('sends writes; not on prod');- const { env, switchClient } = lane.target;- const observer = new ScenarioObserver(env.serverLog, switchClient, run.scope().tag);- const probe = await probeSessionRoutes(switchClient, readSurface());- const serverErrors = await observer.finish();+ const { switchClient } = lane.target;+ const observer = watch.begin(switchClient, run.scope().tag);+ let probe: Awaited<ReturnType<typeof probeSessionRoutes>>;+ try {+ probe = await probeSessionRoutes(switchClient, readSurface());+ } finally {+ observer.stop();+ }+ const serverErrors = observer.errors(null, console.log);console.log(`[session-required] probed ${probe.probed} session routes; ` +`${probe.notMounted} more are not mounted here (feature flags off)`);expect(probe.probed, 'no session route was probed').toBeGreaterThan(0);expect(probe.wrong, `answered without a session:\n ${probe.wrong.join('\n ')}\n`).toEqual([]);- expectNoServerErrors(serverErrors, observer.tag);+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');}, 120_000);- // Last, so it sees whatever the run left the server in.+ // Last, so they see whatever the run left the server in.it('the server is still ready after the run', async (context) => {if (!lane.target) return context.skip(lane.skip);await lane.target.switchClient.ready();});-});-function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {- if (errors === null) return;- expect(- errors,- `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`- ).toEqual([]);-}+ it('the server logged no errors outside the scenarios', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ expectNoServerErrors(await watch.sweep(), null, 'outside any scenario');+ });+});diff --git a/console/tooling-e2e/src/provider/provider.e2e.test.ts b/console/tooling-e2e/src/provider/provider.e2e.test.tsindex 3c447e7db..cb8d4ee20 100644--- a/console/tooling-e2e/src/provider/provider.e2e.test.ts+++ b/console/tooling-e2e/src/provider/provider.e2e.test.ts@@ -1,8 +1,13 @@import { afterAll, beforeAll, describe, expect, it } from 'vitest';import { describeEnv } from '../core/env.ts';-import { ARTIFACT_PREFIX } from '../core/run.ts';+import { RunContext } from '../core/run.ts';import { admit, assertValidHeaders } from '../core/scenario.ts';-import { describeErrors, describeServerLogCheck, ScenarioObserver } from '../core/server-log.ts';+import {+ describeErrors,+ describeServerLogCheck,+ expectNoServerErrors,+ ServerLogWatch,+} from '../core/server-log.ts';import { resolveLane, type LaneResolution } from '../core/suite.ts';import { setupHarness, teardownHarness, waitForSession, type Harness } from './harness.ts';import { formatResultsTable, SCENARIOS, type ScenarioResult } from './scenarios.ts';@@ -21,16 +26,24 @@ assertValidHeaders(SCENARIOS.map((scenario) => scenario.header));let lane: LaneResolution;let harness: Harness | null = null;+/** The scopes whose tags this run's scenario requests carry. */+let scopes: RunContext;+let watch: ServerLogWatch;const results: ScenarioResult[] = [];beforeAll(async () => {lane = await resolveLane('provider');if (!lane.target) return;+ // Before setup, so whatever setup makes the server log is read too.+ watch = new ServerLogWatch(lane.target.env.serverLog);// Nothing to set up when no scenario can run here — say, a target that// lacks the prerequisites every scenario shares.if (!SCENARIOS.some((scenario) => admit(scenario.header, lane.target!.env).run)) return;harness = await setupHarness(lane.target);+ scopes = new RunContext(harness.runId);+ // The first scope named the setup's agent and channel; scenarios start after it.+ scopes.scope();console.log(['',@@ -59,6 +72,14 @@ beforeAll(async () => {afterAll(async () => {if (results.length > 0) console.log(formatResultsTable(results));await teardownHarness(harness);+ // Teardown runs after the last test, so its errors fail the hook instead.+ if (!lane.target) return;+ const late = await watch.sweep();+ if (late !== null && late.length > 0) {+ throw new Error(+ `the server logged errors after the last scenario (teardown):\n ${describeErrors(late, null)}`+ );+ }});describe('provider lane', () => {@@ -75,14 +96,16 @@ describe('provider lane', () => {return context.skip(admission.reason);}const h = harness!;- const observer = new ScenarioObserver(- lane.target.env.serverLog,- h.switch,- `${ARTIFACT_PREFIX}${h.runId}-${scenario.header.id}`- );- const result = await scenario.run(h);+ const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);+ const observer = watch.begin(h.switch, scopes.scope().tag);+ let result: ScenarioResult;+ try {+ result = await scenario.run(h);+ } finally {+ observer.stop();+ }results.push(result);- const serverErrors = await observer.finish();+ const serverErrors = observer.errors(result.ok ? null : result.details, log);if (!result.ok) {console.log(`${result.name} transcript:\n` +@@ -92,18 +115,18 @@ describe('provider lane', () => {);}expect(result.ok, result.details).toBe(true);- if (serverErrors !== null) {- expect(- serverErrors,- `the server logged errors while this ran:\n ${describeErrors(serverErrors, observer.tag)}\n`- ).toEqual([]);- }+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');});}- // Last, so it sees whatever the run left the server in.+ // Last, so they see whatever the run left the server in.it('the server is still ready after the run', async (context) => {if (!lane.target) return context.skip(lane.skip);await lane.target.switchClient.ready();});++ it('the server logged no errors outside the scenarios', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ expectNoServerErrors(await watch.sweep(), null, 'during setup or between scenarios');+ });});