Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Read the server log by position, and check what is logged outside scenarios

Petr Bauch7225d1f50528
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index 333e3b9df..ba4b01eb4 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -225,15 +225,24 @@ on: an event loop that catches one bad event, say. Two checks catch that.
`SWITCH_E2E_STRICT=1`. Each lane's last test asks again, so a run that
leaves the server unready fails.
- **No errors logged.** With `SWITCH_E2E_SERVER_LOG` set, the harness reads the
- server's JSON log after each scenario. The scenario fails if anything was
- logged at error level or above, including output that bypassed logging,
- between its start and 1.5 s after its end.
- - The check goes by time, because the work a scenario sets off runs on
- background tasks that carry no request id.
+ server's JSON log as the lane runs. Each read starts where the previous one
+ stopped, and a line the server is still writing waits for the next read.
+ - A scenario fails if anything in the lines written while it ran is at error
+ level or above.
+ - Whatever is logged outside the scenarios also counts: during setup,
+ between scenarios (late background work, say), and in the 1.5 s after the
+ last one. It fails the lane's last test, "the server logged no errors
+ outside the scenarios". The provider lane reads once more after its
+ teardown, and fails the run if that teardown made the server log an error.
+ - Output that bypassed logging counts when it reads as a failure: a
+ traceback, an exception a thread died of, a fatal interpreter error, or a
+ task destroyed while pending. Anything else that is not a JSON record (a
+ dependency's warning, a stray print) does not.
- Each harness request carries `X-Request-Id: e2e-<run><scope>-<n>`, which
the server stamps on every line it logs while serving the request. Each
error is reported as coming from this scenario's request, another request,
- or background work.
+ or background work. That holds wherever the error lands, so a late one
+ still names the scenario that caused it.
The stack script starts switch-core with `LOG_FORMAT=json` and sets the
variable. Against a deployed environment the log is in its log backend, so
diff --git a/console/tooling-e2e/src/core/server-log.test.ts b/console/tooling-e2e/src/core/server-log.test.ts
index dc322947b..b8215b0b1 100644
--- a/console/tooling-e2e/src/core/server-log.test.ts
+++ b/console/tooling-e2e/src/core/server-log.test.ts
@@ -1,26 +1,30 @@
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
-import { describeErrors, ServerLog } from './server-log.ts';
+import { describeErrors, ServerLogCursor, ServerLogWatch } from './server-log.ts';
const dirs: string[] = [];
-function logWith(lines: string[]): ServerLog {
+function logFile(lines: string[]): string {
const dir = mkdtempSync(path.join(tmpdir(), 'e2e-server-log-'));
dirs.push(dir);
const file = path.join(dir, 'switch-core.log');
- writeFileSync(file, `${lines.join('\n')}\n`);
- return new ServerLog(file);
+ writeFileSync(file, lines.map((line) => `${line}\n`).join(''));
+ return file;
+}
+
+function append(file: string, lines: string[]): void {
+ appendFileSync(file, lines.map((line) => `${line}\n`).join(''));
}
afterEach(() => {
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
-function record(timestamp: string, status: string, message: string, requestId?: string): string {
+function record(status: string, message: string, requestId?: string): string {
return JSON.stringify({
- timestamp,
+ timestamp: '2026-10-19T10:00:00.000+00:00',
status,
message,
logger: { name: 'switch_core.test' },
@@ -28,51 +32,101 @@ function record(timestamp: string, status: string, message: string, requestId?:
});
}
-const at = (iso: string): number => Date.parse(iso);
+const tagger = { tagRequests: (_tag: string | null) => {} };
-describe('server log', () => {
- it('returns the errors logged inside the window, and nothing at lower levels or outside it', () => {
- const log = logWith([
- record('2026-10-19T10:00:00.000+00:00', 'error', 'before the scenario'),
- record('2026-10-19T10:00:01.000+00:00', 'info', 'ordinary'),
- record('2026-10-19T10:00:01.500+00:00', 'warning', 'degraded'),
- record('2026-10-19T10:00:02.000+00:00', 'error', 'delivery failed', 'e2e-abc123a-3'),
- record('2026-10-19T10:00:02.500+00:00', 'critical', 'loop died'),
- record('2026-10-19T10:00:09.000+00:00', 'error', 'after the scenario'),
+describe('server log cursor', () => {
+ it('reads only what was written after it, at error level and above', () => {
+ const file = logFile([record('error', 'at boot, before the lane')]);
+ const cursor = ServerLogCursor.atEnd(file);
+ append(file, [
+ record('info', 'ordinary'),
+ record('warning', 'degraded'),
+ record('error', 'delivery failed', 'e2e-abc123a-3'),
+ record('critical', 'loop died'),
]);
- const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));
+ const errors = cursor.take();
expect(errors.map((error) => error.message)).toEqual(['delivery failed', 'loop died']);
expect(describeErrors(errors, 'e2e-abc123a')).toContain(
"this scenario's request e2e-abc123a-3"
);
expect(describeErrors(errors, 'e2e-abc123a')).toContain('loop died (background work)');
+ expect(cursor.take()).toEqual([]);
+ });
+
+ it('leaves a line the server is still writing for the next read', () => {
+ const file = logFile([record('info', 'booted')]);
+ const cursor = ServerLogCursor.atEnd(file);
+ const line = record('error', 'half written');
+ appendFileSync(file, line.slice(0, 20));
+ expect(cursor.take()).toEqual([]);
+ appendFileSync(file, `${line.slice(20)}\n`);
+ expect(cursor.take().map((error) => error.message)).toEqual(['half written']);
});
- it('counts output that bypassed logging as an error, at the time of the record before it', () => {
- const log = logWith([
- record('2026-10-19T10:00:02.000+00:00', 'info', 'serving'),
+ it('counts a traceback that bypassed logging, whenever it was printed, and ignores other output', () => {
+ const file = logFile([record('info', 'booted')]);
+ const cursor = ServerLogCursor.atEnd(file);
+ append(file, [
+ 'DeprecationWarning: something a dependency still does',
+ 'Exception in thread mm-ws-agent:',
'Traceback (most recent call last):',
- record('2026-10-19T10:00:09.000+00:00', 'info', 'later'),
+ ' File "adapter.py", line 1, in _run_ws',
+ 'RuntimeError: boom',
]);
- const errors = log.errorsBetween(at('2026-10-19T10:00:01Z'), at('2026-10-19T10:00:03Z'));
- expect(errors).toEqual([
- {
- timestamp: '2026-10-19T10:00:02.000+00:00',
- logger: '(unstructured output)',
- message: 'Traceback (most recent call last):',
- requestId: null,
- },
+ expect(cursor.take().map((error) => error.message)).toEqual([
+ 'Exception in thread mm-ws-agent:',
+ 'Traceback (most recent call last):',
]);
});
it('refuses a log that is not JSON, rather than finding no errors in it', () => {
- const log = logWith(['2026-10-19T10:00:02 ERROR [switch_core] text format']);
- expect(() => log.errorsBetween(0, Date.now())).toThrow(/LOG_FORMAT=json/);
+ const file = logFile(['2026-10-19T10:00:02 ERROR [switch_core] text format']);
+ expect(() => ServerLogCursor.atEnd(file)).toThrow(/LOG_FORMAT=json/);
});
- it('refuses a log it cannot read', () => {
- expect(() => new ServerLog('/nonexistent/switch-core.log').errorsBetween(0, 1)).toThrow(
+ it('refuses a log it cannot read, and one that shrank under it', () => {
+ expect(() => ServerLogCursor.atEnd('/nonexistent/switch-core.log')).toThrow(
/Cannot read the server log/
);
+ const file = logFile([record('info', 'booted'), record('info', 'more')]);
+ const cursor = ServerLogCursor.atEnd(file);
+ truncateSync(file, 10);
+ expect(() => cursor.take()).toThrow(/rotated or truncated/);
+ });
+});
+
+describe('server log watch', () => {
+ it('sets aside what was logged between scenarios, for the end-of-lane sweep', async () => {
+ const file = logFile([record('info', 'booted')]);
+ const watch = new ServerLogWatch(file);
+ append(file, [record('error', 'during setup')]);
+ const observer = watch.begin(tagger, 'e2e-abc123b');
+ append(file, [record('error', 'in the scenario', 'e2e-abc123b-1')]);
+ observer.stop();
+ expect(observer.errors(null, () => {})?.map((error) => error.message)).toEqual([
+ 'in the scenario',
+ ]);
+ append(file, [record('error', 'after the last scenario')]);
+ expect((await watch.sweep())?.map((error) => error.message)).toEqual([
+ 'during setup',
+ 'after the last scenario',
+ ]);
+ });
+
+ it("never lets an unreadable log replace the scenario's own failure", () => {
+ const file = logFile([record('info', 'booted'), record('info', 'more')]);
+ const watch = new ServerLogWatch(file);
+ const observer = watch.begin(tagger, 'e2e-abc123b');
+ truncateSync(file, 10);
+ const said: string[] = [];
+ expect(observer.errors(new Error('the scenario failed'), (line) => said.push(line))).toBe(null);
+ expect(said.join('\n')).toMatch(/could not read the server log/);
+ expect(() => observer.errors(null, () => {})).toThrow(/rotated or truncated/);
+ });
+
+ it('is off, reading nothing, without a log', async () => {
+ const watch = new ServerLogWatch(null);
+ expect(watch.begin(tagger, 'e2e-abc123b').errors(null, () => {})).toBe(null);
+ expect(await watch.sweep()).toBe(null);
});
});
diff --git a/console/tooling-e2e/src/core/server-log.ts b/console/tooling-e2e/src/core/server-log.ts
index 7be28af8f..333317241 100644
--- a/console/tooling-e2e/src/core/server-log.ts
+++ b/console/tooling-e2e/src/core/server-log.ts
@@ -1,4 +1,5 @@
-import { readFileSync } from 'node:fs';
+import { closeSync, fstatSync, openSync, readSync } from 'node:fs';
+import { expect } from 'vitest';
/**
* The server's own log, read back so a scenario fails when the server logged an
@@ -7,15 +8,19 @@ import { readFileSync } from 'node:fs';
* scenario it breaks.
*
* Only a log the harness can read: the throwaway stack's `switch-core.log`,
- * written with `LOG_FORMAT=json` so each line is one record with a timestamp to
- * the millisecond. A deployed environment's log lives in its log backend, so
- * there the check is off and says so.
+ * written with `LOG_FORMAT=json`, one record per line. A deployed
+ * environment's log lives in its log backend, so there the check is off and
+ * says so.
*
- * Errors are found by time, not by request id: work a scenario sets off runs on
- * background tasks (a consumer delivering its message, a bridge relaying it)
- * that carry no request id. The id is attribution: each request the harness
- * sends carries `X-Request-Id: <scope tag>-<n>`, so an error logged while
- * serving one says which.
+ * The log is read by position, not by time: a cursor starts at the end of the
+ * log when the lane starts, and each read takes the complete lines written
+ * since the one before. Work a scenario sets off runs on background tasks
+ * (a consumer delivering its message, a bridge relaying it) that carry no
+ * request id and may log after the scenario has returned, so nothing is
+ * skipped: what is logged between scenarios, during setup, or after the last
+ * one is gathered as "outside the scenarios" and checked at the end of the
+ * lane. Each request the harness sends carries `X-Request-Id: <scope tag>-<n>`,
+ * which is how an error says which scenario it came from, wherever it lands.
*/
export interface ServerLogError {
@@ -26,43 +31,67 @@ export interface ServerLogError {
requestId: string | null;
}
-/** How long after a scenario its background work may still log about it. */
+/**
+ * How long the end-of-lane sweep waits for background work the last scenario
+ * set off to log about it.
+ */
export const SETTLE_MS = 1_500;
-export class ServerLog {
+/**
+ * Output that bypassed logging and means something failed: an uncaught
+ * exception's traceback, one a thread reported on its way out, the
+ * interpreter dying, or asyncio collecting a task still pending. Any other
+ * line that is not a JSON record (a dependency's DeprecationWarning, a stray
+ * print, uv's own messages) is not an error.
+ */
+const UNLOGGED_FAILURE =
+ /^(Traceback \(most recent call last\):|Exception in thread |Fatal Python error:|Task was destroyed but it is pending!)/;
+
+/** A position in the server log, always at the start of a line. */
+export class ServerLogCursor {
readonly path: string;
+ private offset: number;
- constructor(path: string) {
+ private constructor(path: string, offset: number) {
this.path = path;
+ this.offset = offset;
}
/**
- * Every record at error level or above logged in `[startMs, endMs]`. A line
- * that is not a JSON record is output that bypassed logging (a traceback
- * printed to stderr, say) and counts as an error, at the time of the record
- * before it.
+ * A cursor at the end of the log's last complete line. The server has booted
+ * by now, so a log with no JSON record in it was not written with
+ * `LOG_FORMAT=json` and would read as error-free whatever happened: refused.
*/
- errorsBetween(startMs: number, endMs: number): ServerLogError[] {
- let raw: string;
- try {
- raw = readFileSync(this.path, 'utf8');
- } catch (error) {
+ static atEnd(path: string): ServerLogCursor {
+ const raw = readLog(path, 0);
+ const complete = raw.subarray(0, raw.lastIndexOf(0x0a) + 1).toString('utf8');
+ if (!complete.split('\n').some((line) => parseRecord(line) !== null)) {
throw new Error(
- `Cannot read the server log at ${this.path} (SWITCH_E2E_SERVER_LOG): ` +
- `${error instanceof Error ? error.message : String(error)}`
+ `The server log at ${path} holds no JSON records: start the server ` +
+ `with LOG_FORMAT=json for the harness to read it.`
);
}
+ return new ServerLogCursor(path, Buffer.byteLength(complete, 'utf8'));
+ }
+
+ /**
+ * The errors in every complete line written since the last read, moving past
+ * them. A last line with no newline yet is one the server is still writing,
+ * and is left for the next read.
+ */
+ take(): ServerLogError[] {
+ const raw = readLog(this.path, this.offset);
+ const end = raw.lastIndexOf(0x0a) + 1;
+ this.offset += end;
const errors: ServerLogError[] = [];
- let lastMs = Number.NEGATIVE_INFINITY;
- let lastTimestamp = '(before the first record)';
- let records = 0;
- for (const line of raw.split('\n')) {
+ let lastTimestamp = '(no record yet)';
+ for (const line of raw.subarray(0, end).toString('utf8').split('\n')) {
if (line.trim() === '') continue;
const record = parseRecord(line);
if (record === null) {
- if (lastMs >= startMs && lastMs <= endMs) {
+ if (UNLOGGED_FAILURE.test(line)) {
errors.push({
- timestamp: lastTimestamp,
+ timestamp: `after ${lastTimestamp}`,
logger: '(unstructured output)',
message: line.slice(0, 300),
requestId: null,
@@ -70,10 +99,7 @@ export class ServerLog {
}
continue;
}
- records += 1;
- lastMs = record.ms;
lastTimestamp = record.timestamp;
- if (record.ms < startMs || record.ms > endMs) continue;
if (record.status === 'error' || record.status === 'critical') {
errors.push({
timestamp: record.timestamp,
@@ -83,18 +109,42 @@ export class ServerLog {
});
}
}
- if (records === 0 && raw.trim() !== '') {
+ return errors;
+ }
+}
+
+function readLog(path: string, offset: number): Buffer {
+ let fd: number;
+ try {
+ fd = openSync(path, 'r');
+ } catch (error) {
+ throw new Error(
+ `Cannot read the server log at ${path} (SWITCH_E2E_SERVER_LOG): ` +
+ `${error instanceof Error ? error.message : String(error)}`
+ );
+ }
+ try {
+ const size = fstatSync(fd).size;
+ if (size < offset) {
throw new Error(
- `The server log at ${this.path} holds no JSON records: start the server ` +
- `with LOG_FORMAT=json for the harness to read it.`
+ `The server log at ${path} shrank below what was already read (${size} < ${offset} bytes): ` +
+ `was it rotated or truncated during the run?`
);
}
- return errors;
+ const buffer = Buffer.alloc(size - offset);
+ let read = 0;
+ while (read < buffer.length) {
+ const n = readSync(fd, buffer, read, buffer.length - read, offset + read);
+ if (n === 0) break;
+ read += n;
+ }
+ return buffer.subarray(0, read);
+ } finally {
+ closeSync(fd);
}
}
interface LogRecord {
- ms: number;
timestamp: string;
status: string;
logger: string;
@@ -112,13 +162,10 @@ function parseRecord(line: string): LogRecord | null {
}
if (typeof value !== 'object' || value === null) return null;
const fields = value as { [key: string]: unknown };
- const timestamp = typeof fields.timestamp === 'string' ? fields.timestamp : null;
- const ms = timestamp === null ? Number.NaN : Date.parse(timestamp);
- if (timestamp === null || Number.isNaN(ms)) return null;
+ if (typeof fields.timestamp !== 'string') return null;
const logger = fields.logger as { name?: unknown } | undefined;
return {
- ms,
- timestamp,
+ timestamp: fields.timestamp,
status: typeof fields.status === 'string' ? fields.status : '',
logger: typeof logger?.name === 'string' ? logger.name : '?',
message: typeof fields.message === 'string' ? fields.message : '',
@@ -127,13 +174,13 @@ function parseRecord(line: string): LogRecord | null {
}
/** One line per error, saying whose request it came from. */
-export function describeErrors(errors: readonly ServerLogError[], tag: string): string {
+export function describeErrors(errors: readonly ServerLogError[], tag: string | null): string {
return errors
.map((error) => {
const source =
error.requestId === null
? 'background work'
- : error.requestId.startsWith(`${tag}-`)
+ : tag !== null && error.requestId.startsWith(`${tag}-`)
? `this scenario's request ${error.requestId}`
: `request ${error.requestId}`;
return `${error.timestamp} [${error.logger}] ${error.message} (${source})`;
@@ -141,35 +188,97 @@ export function describeErrors(errors: readonly ServerLogError[], tag: string):
.join('\n ');
}
+interface Taggable {
+ tagRequests(tag: string | null): void;
+}
+
/**
- * Watches one scenario: tags its requests, and on `finish` returns what the
- * server logged at error level while it ran, or null when there is no log to
- * read here.
+ * One lane's watch over the server log: per scenario, and for everything
+ * logged outside them. Off (every read is null) when there is no log to read
+ * here.
*/
+export class ServerLogWatch {
+ private readonly cursor: ServerLogCursor | null;
+ private readonly outside: ServerLogError[] = [];
+
+ constructor(serverLog: string | null) {
+ this.cursor = serverLog === null ? null : ServerLogCursor.atEnd(serverLog);
+ }
+
+ /**
+ * Start watching a scenario: what was logged since the last read is set
+ * aside as outside any scenario, and the client tags its requests with
+ * `tag` until the observer stops.
+ */
+ begin(client: Taggable, tag: string): ScenarioObserver {
+ if (this.cursor !== null) this.outside.push(...this.cursor.take());
+ client.tagRequests(tag);
+ return new ScenarioObserver(this.cursor, client, tag);
+ }
+
+ /**
+ * Everything logged outside the scenarios: during setup, between them, and
+ * after the last, once its background work has had `SETTLE_MS` to log.
+ * Null when the check is off.
+ */
+ async sweep(): Promise<ServerLogError[] | null> {
+ if (this.cursor === null) return null;
+ await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));
+ this.outside.push(...this.cursor.take());
+ return this.outside.splice(0);
+ }
+}
+
export class ScenarioObserver {
readonly tag: string;
- private readonly log: ServerLog | null;
- private readonly client: { tagRequests(tag: string | null): void };
- private readonly startMs: number;
-
- constructor(
- serverLog: string | null,
- client: { tagRequests(tag: string | null): void },
- tag: string
- ) {
- this.tag = tag;
- this.log = serverLog === null ? null : new ServerLog(serverLog);
+ private readonly cursor: ServerLogCursor | null;
+ private readonly client: Taggable;
+
+ constructor(cursor: ServerLogCursor | null, client: Taggable, tag: string) {
+ this.cursor = cursor;
this.client = client;
- this.startMs = Date.now();
- client.tagRequests(tag);
+ this.tag = tag;
}
- async finish(): Promise<ServerLogError[] | null> {
+ /** Stop tagging the client's requests. Call it in a `finally`. */
+ stop(): void {
this.client.tagRequests(null);
- if (this.log === null) return null;
- await new Promise((resolve) => setTimeout(resolve, SETTLE_MS));
- return this.log.errorsBetween(this.startMs, Date.now());
}
+
+ /**
+ * What the server logged at error level since the scenario began, or null
+ * when the check is off.
+ *
+ * The scenario's own failure is the root cause, so a log that cannot be read
+ * never replaces it: with `failure` set, that read error is only logged.
+ */
+ errors(failure: unknown, log: (message: string) => void): ServerLogError[] | null {
+ if (this.cursor === null) return null;
+ let errors: ServerLogError[];
+ try {
+ errors = this.cursor.take();
+ } catch (error) {
+ if (failure === null) throw error;
+ log(
+ `could not read the server log: ${error instanceof Error ? error.message : String(error)}`
+ );
+ return null;
+ }
+ if (errors.length > 0) log(`the server logged errors:\n ${describeErrors(errors, this.tag)}`);
+ return errors;
+ }
+}
+
+/** Fail when the server logged errors; `null` (the check is off) passes. */
+export function expectNoServerErrors(
+ errors: ServerLogError[] | null,
+ tag: string | null,
+ when: string
+): void {
+ if (errors === null) return;
+ expect(errors, `the server logged errors ${when}:\n ${describeErrors(errors, tag)}\n`).toEqual(
+ []
+ );
}
/** What the run's log says about the no-error check, for the run header. */
diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.ts
index bf1e00df7..3ce722f62 100644
--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts
+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts
@@ -3,10 +3,9 @@ import { describeEnv } from '../core/env.ts';
import { RunContext } from '../core/run.ts';
import { admit, assertValidHeaders } from '../core/scenario.ts';
import {
- describeErrors,
describeServerLogCheck,
- ScenarioObserver,
- type ServerLogError,
+ expectNoServerErrors,
+ ServerLogWatch,
} from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { Fixtures } from '../fixtures/fixtures.ts';
@@ -27,12 +26,14 @@ assertValidHeaders(FAST_SCENARIOS.map((scenario) => scenario.header));
const run = new RunContext();
let lane: LaneResolution;
+let watch: ServerLogWatch;
beforeAll(async () => {
lane = await resolveLane('fast');
if (lane.target) {
console.log(`fast lane run ${run.runId}: ${describeEnv(lane.target.env)}`);
console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);
+ watch = new ServerLogWatch(lane.target.env.serverLog);
}
}, 60_000);
@@ -55,25 +56,27 @@ describe('fast lane', () => {
const names = run.scope();
const fixtures = new Fixtures(env, names, switchClient, mattermost);
const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
- const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);
+ const observer = watch.begin(switchClient, names.tag);
let failure: unknown = null;
+ let leftovers: string[];
try {
- await scenario.run({ env, names, switchClient, mattermost, fixtures, log });
- } catch (error) {
- failure = error;
+ try {
+ await scenario.run({ env, names, switchClient, mattermost, fixtures, log });
+ } catch (error) {
+ failure = error;
+ }
+ leftovers = await fixtures.teardown();
+ } finally {
+ observer.stop();
}
- const leftovers = await fixtures.teardown();
for (const line of leftovers) log(`teardown: ${line}`);
- const serverErrors = await observer.finish();
- if (serverErrors !== null && serverErrors.length > 0) {
- log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);
- }
+ const serverErrors = observer.errors(failure, log);
// The scenario's own failure is the root cause and is reported as is; a
// leak, then a server error, is the verdict only when nothing else went
// wrong.
if (failure !== null) throw failure;
if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);
- expectNoServerErrors(serverErrors, observer.tag);
+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
});
}
});
@@ -84,30 +87,32 @@ describe('fast lane, derived checks', () => {
it('every gateway route that takes a session refuses a request without one', async (context) => {
if (!lane.target) return context.skip(lane.skip);
if (lane.target.env.environment === 'prod') return context.skip('sends writes; not on prod');
- const { env, switchClient } = lane.target;
- const observer = new ScenarioObserver(env.serverLog, switchClient, run.scope().tag);
- const probe = await probeSessionRoutes(switchClient, readSurface());
- const serverErrors = await observer.finish();
+ const { switchClient } = lane.target;
+ const observer = watch.begin(switchClient, run.scope().tag);
+ let probe: Awaited<ReturnType<typeof probeSessionRoutes>>;
+ try {
+ probe = await probeSessionRoutes(switchClient, readSurface());
+ } finally {
+ observer.stop();
+ }
+ const serverErrors = observer.errors(null, console.log);
console.log(
`[session-required] probed ${probe.probed} session routes; ` +
`${probe.notMounted} more are not mounted here (feature flags off)`
);
expect(probe.probed, 'no session route was probed').toBeGreaterThan(0);
expect(probe.wrong, `answered without a session:\n ${probe.wrong.join('\n ')}\n`).toEqual([]);
- expectNoServerErrors(serverErrors, observer.tag);
+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
}, 120_000);
- // Last, so it sees whatever the run left the server in.
+ // Last, so they see whatever the run left the server in.
it('the server is still ready after the run', async (context) => {
if (!lane.target) return context.skip(lane.skip);
await lane.target.switchClient.ready();
});
-});
-function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {
- if (errors === null) return;
- expect(
- errors,
- `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`
- ).toEqual([]);
-}
+ it('the server logged no errors outside the scenarios', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ expectNoServerErrors(await watch.sweep(), null, 'outside any scenario');
+ });
+});
diff --git a/console/tooling-e2e/src/provider/provider.e2e.test.ts b/console/tooling-e2e/src/provider/provider.e2e.test.ts
index 3c447e7db..cb8d4ee20 100644
--- a/console/tooling-e2e/src/provider/provider.e2e.test.ts
+++ b/console/tooling-e2e/src/provider/provider.e2e.test.ts
@@ -1,8 +1,13 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { describeEnv } from '../core/env.ts';
-import { ARTIFACT_PREFIX } from '../core/run.ts';
+import { RunContext } from '../core/run.ts';
import { admit, assertValidHeaders } from '../core/scenario.ts';
-import { describeErrors, describeServerLogCheck, ScenarioObserver } from '../core/server-log.ts';
+import {
+ describeErrors,
+ describeServerLogCheck,
+ expectNoServerErrors,
+ ServerLogWatch,
+} from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { setupHarness, teardownHarness, waitForSession, type Harness } from './harness.ts';
import { formatResultsTable, SCENARIOS, type ScenarioResult } from './scenarios.ts';
@@ -21,16 +26,24 @@ assertValidHeaders(SCENARIOS.map((scenario) => scenario.header));
let lane: LaneResolution;
let harness: Harness | null = null;
+/** The scopes whose tags this run's scenario requests carry. */
+let scopes: RunContext;
+let watch: ServerLogWatch;
const results: ScenarioResult[] = [];
beforeAll(async () => {
lane = await resolveLane('provider');
if (!lane.target) return;
+ // Before setup, so whatever setup makes the server log is read too.
+ watch = new ServerLogWatch(lane.target.env.serverLog);
// Nothing to set up when no scenario can run here — say, a target that
// lacks the prerequisites every scenario shares.
if (!SCENARIOS.some((scenario) => admit(scenario.header, lane.target!.env).run)) return;
harness = await setupHarness(lane.target);
+ scopes = new RunContext(harness.runId);
+ // The first scope named the setup's agent and channel; scenarios start after it.
+ scopes.scope();
console.log(
[
'',
@@ -59,6 +72,14 @@ beforeAll(async () => {
afterAll(async () => {
if (results.length > 0) console.log(formatResultsTable(results));
await teardownHarness(harness);
+ // Teardown runs after the last test, so its errors fail the hook instead.
+ if (!lane.target) return;
+ const late = await watch.sweep();
+ if (late !== null && late.length > 0) {
+ throw new Error(
+ `the server logged errors after the last scenario (teardown):\n ${describeErrors(late, null)}`
+ );
+ }
});
describe('provider lane', () => {
@@ -75,14 +96,16 @@ describe('provider lane', () => {
return context.skip(admission.reason);
}
const h = harness!;
- const observer = new ScenarioObserver(
- lane.target.env.serverLog,
- h.switch,
- `${ARTIFACT_PREFIX}${h.runId}-${scenario.header.id}`
- );
- const result = await scenario.run(h);
+ const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
+ const observer = watch.begin(h.switch, scopes.scope().tag);
+ let result: ScenarioResult;
+ try {
+ result = await scenario.run(h);
+ } finally {
+ observer.stop();
+ }
results.push(result);
- const serverErrors = await observer.finish();
+ const serverErrors = observer.errors(result.ok ? null : result.details, log);
if (!result.ok) {
console.log(
`${result.name} transcript:\n` +
@@ -92,18 +115,18 @@ describe('provider lane', () => {
);
}
expect(result.ok, result.details).toBe(true);
- if (serverErrors !== null) {
- expect(
- serverErrors,
- `the server logged errors while this ran:\n ${describeErrors(serverErrors, observer.tag)}\n`
- ).toEqual([]);
- }
+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
});
}
- // Last, so it sees whatever the run left the server in.
+ // Last, so they see whatever the run left the server in.
it('the server is still ready after the run', async (context) => {
if (!lane.target) return context.skip(lane.skip);
await lane.target.switchClient.ready();
});
+
+ it('the server logged no errors outside the scenarios', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ expectNoServerErrors(await watch.sweep(), null, 'during setup or between scenarios');
+ });
});