Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Browser lane: wait for each page's own requests, and summarise its runs

Petr Bauchab06fce9bb3d
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index f5f6c8151..c7729b940 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -136,8 +136,12 @@ Each scenario gets a fresh browser context, signed out. What it needs in the
server it creates over HTTP through the same fixtures as the fast lane. Beyond
its own assertions, a scenario fails on anything that went wrong on its pages:
an uncaught page error, a console error, a `/gateway` request the server
-answered with a 5xx (or any 4xx once signed in), or an error banner. A failed
-scenario's last page is saved under `screenshots/` in the output directory.
+answered with a 5xx (or any 4xx once signed in), or an error banner. A
+request is blamed on the page that made it, and one the browser cancelled
+itself (the page moved on mid-request) is not a problem. A page counts as
+loaded once no `/gateway` request has been in flight for half a second; one
+still waiting after 15 s is a problem too. A failed scenario's last page is
+saved under `screenshots/` in the output directory.
| Scenario | What it drives |
| --- | --- |
diff --git a/console/tooling-e2e/src/browser/browser.e2e.test.ts b/console/tooling-e2e/src/browser/browser.e2e.test.ts
index 08dd29422..a263a2861 100644
--- a/console/tooling-e2e/src/browser/browser.e2e.test.ts
+++ b/console/tooling-e2e/src/browser/browser.e2e.test.ts
@@ -7,10 +7,9 @@ import { RunContext } from '../core/run.ts';
import { admit, assertValidHeaders } from '../core/scenario.ts';
import { IssuedSecrets } from '../core/secrets.ts';
import {
- describeErrors,
describeServerLogCheck,
- ScenarioObserver,
- type ServerLogError,
+ expectNoServerErrors,
+ ServerLogWatch,
} from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { Fixtures } from '../fixtures/fixtures.ts';
@@ -34,12 +33,14 @@ assertValidHeaders(BROWSER_SCENARIOS.map((scenario) => scenario.header));
const run = new RunContext();
let lane: LaneResolution;
let browser: Browser | null = null;
+let watch: ServerLogWatch;
beforeAll(async () => {
lane = await resolveLane('browser');
if (!lane.target) return;
console.log(`browser lane run ${run.runId}: ${describeEnv(lane.target.env)}`);
console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);
+ watch = new ServerLogWatch(lane.target.env.serverLog);
if (lane.target.env.gatewayUiUrl !== null) browser = await chromium.launch();
}, 120_000);
@@ -69,7 +70,11 @@ describe('browser lane', () => {
const names = run.scope();
const fixtures = new Fixtures(env, names, switchClient, mattermost);
const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);
- const browserContext = await browser.newContext();
+ // The page's own /gateway requests carry the scope's tag too, so an
+ // error the server logs serving one is attributed to this scenario.
+ const browserContext = await browser.newContext({
+ extraHTTPHeaders: { 'X-Request-Id': `${names.tag}-ui` },
+ });
const page = await browserContext.newPage();
const ui = new GatewayUi(
browserContext,
@@ -78,40 +83,40 @@ describe('browser lane', () => {
env.switch,
new IssuedSecrets(env.secretsFile)
);
- const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);
+ const observer = watch.begin(switchClient, names.tag);
let failure: unknown = null;
+ let problems: string[];
+ let leftovers: string[];
try {
- await scenario.run({ env, names, switchClient, fixtures, ui, log });
- } catch (error) {
- failure = error;
- }
- const problems = [...ui.recorded()];
- if ((failure !== null || problems.length > 0) && env.outputDir !== null) {
- const dir = path.join(env.outputDir, 'screenshots');
- mkdirSync(dir, { recursive: true });
- const file = path.join(dir, `${scenario.header.id}.png`);
- await ui.screenshot(file);
- log(`screenshot of the last page: ${file}`);
+ try {
+ await scenario.run({ env, names, switchClient, fixtures, ui, log });
+ } catch (error) {
+ failure = error;
+ }
+ problems = [...ui.recorded()];
+ if ((failure !== null || problems.length > 0) && env.outputDir !== null) {
+ const dir = path.join(env.outputDir, 'screenshots');
+ mkdirSync(dir, { recursive: true });
+ const file = path.join(dir, `${scenario.header.id}.png`);
+ await ui.screenshot(file);
+ log(`screenshot of the last page: ${file}`);
+ }
+ await browserContext.close();
+ leftovers = await fixtures.teardown();
+ } finally {
+ observer.stop();
}
- await browserContext.close();
- const leftovers = await fixtures.teardown();
for (const line of leftovers) log(`teardown: ${line}`);
- const serverErrors = await observer.finish();
- if (serverErrors !== null && serverErrors.length > 0) {
- log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);
- }
+ const serverErrors = observer.errors(failure, log);
if (failure !== null) throw failure;
expect(problems, `the pages went wrong:\n ${problems.join('\n ')}\n`).toEqual([]);
if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);
- expectNoServerErrors(serverErrors, observer.tag);
+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
});
}
-});
-function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {
- if (errors === null) return;
- expect(
- errors,
- `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`
- ).toEqual([]);
-}
+ it('the server logged no errors outside the scenarios', async (context) => {
+ if (!lane.target) return context.skip(lane.skip);
+ expectNoServerErrors(await watch.sweep(), null, 'outside any scenario');
+ });
+});
diff --git a/console/tooling-e2e/src/browser/gateway-ui.ts b/console/tooling-e2e/src/browser/gateway-ui.ts
index 2c464f8b4..ceb1b5300 100644
--- a/console/tooling-e2e/src/browser/gateway-ui.ts
+++ b/console/tooling-e2e/src/browser/gateway-ui.ts
@@ -1,4 +1,4 @@
-import type { BrowserContext, ConsoleMessage, Locator, Page, Response } from 'playwright';
+import type { BrowserContext, ConsoleMessage, Locator, Page, Request, Response } from 'playwright';
import { expect } from 'vitest';
import type { SwitchEnv } from '../core/env.ts';
import type { IssuedSecrets } from '../core/secrets.ts';
@@ -6,6 +6,24 @@ import type { IssuedSecrets } from '../core/secrets.ts';
/** The session cookie the gateway sets on sign-in. */
const SESSION_COOKIE = 'switch_auth';
+/** How long a page must go without a `/gateway` request in flight to count as settled. */
+const QUIET_MS = 500;
+/** How long a page may take to settle before what it is still waiting for is a problem. */
+const SETTLE_TIMEOUT_MS = 15_000;
+
+/**
+ * Requests that stay open for as long as the page does, and so never finish:
+ * waiting for them would never settle.
+ */
+const STREAMING = new Set(['eventsource', 'websocket']);
+
+/**
+ * How each browser reports a request it cancelled itself: the page navigated
+ * away mid-request, or a component unmounted and aborted its fetch. Nothing
+ * went wrong on the server.
+ */
+const ABORTED = /net::ERR_ABORTED|NS_BINDING_ABORTED|cancelled/i;
+
/**
* The gateway UI in one browser context, and everything that went wrong on its
* pages while a scenario drove it.
@@ -27,6 +45,9 @@ export class GatewayUi {
private readonly credentials: SwitchEnv;
private readonly secrets: IssuedSecrets;
private readonly problems: string[] = [];
+ /** Each `/gateway` request still in flight, with the page that made it. */
+ private readonly inFlight = new Map<Request, string>();
+ private lastActivityMs = Date.now();
private signedIn = false;
constructor(
@@ -43,15 +64,31 @@ export class GatewayUi {
this.secrets = secrets;
page.on('pageerror', (error) => this.problems.push(`page error: ${error.message}`));
page.on('console', (entry) => this.onConsole(entry));
+ page.on('request', (request) => {
+ if (!this.isGatewayApi(request.url()) || STREAMING.has(request.resourceType())) return;
+ this.inFlight.set(request, this.path());
+ this.lastActivityMs = Date.now();
+ });
page.on('response', (response) => this.onResponse(response));
+ page.on('requestfinished', (request) => this.settle(request));
page.on('requestfailed', (request) => {
+ const from = this.settle(request);
if (!this.isGatewayApi(request.url())) return;
+ const reason = request.failure()?.errorText ?? 'unknown';
+ if (ABORTED.test(reason)) return;
this.problems.push(
- `${request.method()} ${this.pathOf(request.url())} failed: ${request.failure()?.errorText ?? 'unknown'}`
+ `${request.method()} ${this.pathOf(request.url())} failed on ${from}: ${reason}`
);
});
}
+ /** The request is done: forget it, and return the page that made it. */
+ private settle(request: Request): string {
+ const from = this.inFlight.get(request) ?? this.path();
+ if (this.inFlight.delete(request)) this.lastActivityMs = Date.now();
+ return from;
+ }
+
/** Every problem recorded on the pages so far. */
recorded(): readonly string[] {
return this.problems;
@@ -105,11 +142,28 @@ export class GatewayUi {
/**
* Wait until the page has stopped fetching, then fail on an error it shows.
- * MUI renders every error banner as an `alert`; a page with nothing wrong
- * shows none with error severity.
+ *
+ * A route change in the app is not a page load, so the browser's own
+ * `networkidle` was reached long ago and says nothing about the new page.
+ * Settled is instead no `/gateway` request in flight for `QUIET_MS`, which
+ * also covers a page whose effects fetch a moment after it renders. MUI
+ * renders every error banner as an `alert`; a page with nothing wrong shows
+ * none with error severity.
*/
async settled(): Promise<void> {
- await this.page.waitForLoadState('networkidle');
+ const deadline = Date.now() + SETTLE_TIMEOUT_MS;
+ while (this.inFlight.size > 0 || Date.now() - this.lastActivityMs < QUIET_MS) {
+ if (Date.now() > deadline) {
+ const waiting = [...this.inFlight.keys()].map(
+ (request) => `${request.method()} ${this.pathOf(request.url())}`
+ );
+ this.problems.push(
+ `${this.path()} had not settled after ${SETTLE_TIMEOUT_MS / 1000} s, still waiting on: ${waiting.join(', ')}`
+ );
+ break;
+ }
+ await this.page.waitForTimeout(50);
+ }
const alerts = this.page.locator('[role="alert"].MuiAlert-colorError');
const shown = await alerts.allInnerTexts();
for (const text of shown) this.problems.push(`${this.path()} shows an error: ${text.trim()}`);
@@ -147,8 +201,9 @@ export class GatewayUi {
const status = response.status();
const failed = status >= 500 || (this.signedIn && status >= 400);
if (!failed) return;
+ const from = this.inFlight.get(response.request()) ?? this.path();
this.problems.push(
- `${response.request().method()} ${this.pathOf(response.url())} answered ${status} on ${this.path()}`
+ `${response.request().method()} ${this.pathOf(response.url())} answered ${status} on ${from}`
);
}
diff --git a/console/tooling-e2e/src/ci-summary.test.ts b/console/tooling-e2e/src/ci-summary.test.ts
index 33607ef20..e95879de1 100644
--- a/console/tooling-e2e/src/ci-summary.test.ts
+++ b/console/tooling-e2e/src/ci-summary.test.ts
@@ -1,5 +1,9 @@
-import { describe, expect, it } from 'vitest';
-import { parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { describe, expect, it, vi } from 'vitest';
+import { main, parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';
+import { LANES } from './core/env.ts';
const REPORT = `<?xml version="1.0" encoding="UTF-8" ?>
<testsuites name="vitest tests" tests="4">
@@ -44,3 +48,23 @@ describe('CI run summary', () => {
expect(summaryMarkdown('fast on dev', [])).toContain('holds no end-to-end scenarios');
});
});
+
+describe('CI run summary, as the workflow calls it', () => {
+ it('takes every lane, and refuses one that is not', () => {
+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-ci-summary-'));
+ const report = path.join(dir, 'junit.xml');
+ writeFileSync(report, REPORT);
+ vi.stubEnv('GITHUB_OUTPUT', '');
+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+ try {
+ for (const lane of LANES) {
+ expect(() => main([report, lane, `${lane} lane on ephemeral`])).not.toThrow();
+ }
+ expect(() => main([report, 'nightly', 'title'])).toThrow(/Usage/);
+ } finally {
+ write.mockRestore();
+ vi.unstubAllEnvs();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/console/tooling-e2e/src/ci-summary.ts b/console/tooling-e2e/src/ci-summary.ts
index a7c235591..ddb70fbe5 100644
--- a/console/tooling-e2e/src/ci-summary.ts
+++ b/console/tooling-e2e/src/ci-summary.ts
@@ -10,7 +10,7 @@
* get as far as its suite, and a summary that read "0 passed" would hide that.
*/
import { appendFileSync, readFileSync } from 'node:fs';
-import type { Lane } from './core/env.ts';
+import { isLane, LANES, type Lane } from './core/env.ts';
import { message } from './core/suite.ts';
export type CaseStatus = 'pass' | 'fail' | 'skip';
@@ -92,10 +92,10 @@ export function summaryMarkdown(title: string, cases: readonly ScenarioCase[]):
].join('\n');
}
-function main(argv: readonly string[]): void {
+export function main(argv: readonly string[]): void {
const [reportPath, lane, title] = argv;
- if (!reportPath || (lane !== 'fast' && lane !== 'provider') || !title) {
- throw new Error('Usage: ci-summary.ts <junit.xml> <fast|provider> <title>');
+ if (!reportPath || lane === undefined || !isLane(lane) || !title) {
+ throw new Error(`Usage: ci-summary.ts <junit.xml> <${LANES.join('|')}> <title>`);
}
let xml: string;
try {
diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.ts
index 59d5e0f09..1ef5947f8 100644
--- a/console/tooling-e2e/src/core/env.ts
+++ b/console/tooling-e2e/src/core/env.ts
@@ -22,6 +22,13 @@ import { fileURLToPath } from 'node:url';
*/
export type Lane = 'fast' | 'provider' | 'browser';
+/** Every lane: the one list anything that takes a lane by name checks against. */
+export const LANES: readonly Lane[] = ['fast', 'provider', 'browser'];
+
+export function isLane(raw: string): raw is Lane {
+ return (LANES as readonly string[]).includes(raw);
+}
+
/**
* What the target is. Scenarios declare which they may run against.
*
@@ -185,8 +192,8 @@ export function laneFrom(
): Lane | null {
const raw = reader(source, file)('SWITCH_E2E_LANE');
if (raw === undefined) return null;
- if (raw === 'fast' || raw === 'provider' || raw === 'browser') return raw;
- throw new Error(`SWITCH_E2E_LANE must be 'fast', 'provider' or 'browser' — got '${raw}'.`);
+ if (isLane(raw)) return raw;
+ throw new Error(`SWITCH_E2E_LANE must be one of ${LANES.join(', ')} — got '${raw}'.`);
}
/**