Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Browser lane: wait for each page's own requests, and summarise its runs
Petr Bauchab06fce9bb3d
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.mdindex f5f6c8151..c7729b940 100644--- a/console/tooling-e2e/README.md+++ b/console/tooling-e2e/README.md@@ -136,8 +136,12 @@ Each scenario gets a fresh browser context, signed out. What it needs in theserver it creates over HTTP through the same fixtures as the fast lane. Beyondits own assertions, a scenario fails on anything that went wrong on its pages:an uncaught page error, a console error, a `/gateway` request the server-answered with a 5xx (or any 4xx once signed in), or an error banner. A failed-scenario's last page is saved under `screenshots/` in the output directory.+answered with a 5xx (or any 4xx once signed in), or an error banner. A+request is blamed on the page that made it, and one the browser cancelled+itself (the page moved on mid-request) is not a problem. A page counts as+loaded once no `/gateway` request has been in flight for half a second; one+still waiting after 15 s is a problem too. A failed scenario's last page is+saved under `screenshots/` in the output directory.| Scenario | What it drives || --- | --- |diff --git a/console/tooling-e2e/src/browser/browser.e2e.test.ts b/console/tooling-e2e/src/browser/browser.e2e.test.tsindex 08dd29422..a263a2861 100644--- a/console/tooling-e2e/src/browser/browser.e2e.test.ts+++ b/console/tooling-e2e/src/browser/browser.e2e.test.ts@@ -7,10 +7,9 @@ import { RunContext } from '../core/run.ts';import { admit, assertValidHeaders } from '../core/scenario.ts';import { IssuedSecrets } from '../core/secrets.ts';import {- describeErrors,describeServerLogCheck,- ScenarioObserver,- type ServerLogError,+ expectNoServerErrors,+ ServerLogWatch,} from '../core/server-log.ts';import { resolveLane, type LaneResolution } from '../core/suite.ts';import { Fixtures } from '../fixtures/fixtures.ts';@@ -34,12 +33,14 @@ assertValidHeaders(BROWSER_SCENARIOS.map((scenario) => scenario.header));const run = new RunContext();let lane: LaneResolution;let browser: Browser | null = null;+let watch: ServerLogWatch;beforeAll(async () => {lane = await resolveLane('browser');if (!lane.target) return;console.log(`browser lane run ${run.runId}: ${describeEnv(lane.target.env)}`);console.log(` ${describeServerLogCheck(lane.target.env.serverLog)}`);+ watch = new ServerLogWatch(lane.target.env.serverLog);if (lane.target.env.gatewayUiUrl !== null) browser = await chromium.launch();}, 120_000);@@ -69,7 +70,11 @@ describe('browser lane', () => {const names = run.scope();const fixtures = new Fixtures(env, names, switchClient, mattermost);const log = (message: string): void => console.log(`[${scenario.header.id}] ${message}`);- const browserContext = await browser.newContext();+ // The page's own /gateway requests carry the scope's tag too, so an+ // error the server logs serving one is attributed to this scenario.+ const browserContext = await browser.newContext({+ extraHTTPHeaders: { 'X-Request-Id': `${names.tag}-ui` },+ });const page = await browserContext.newPage();const ui = new GatewayUi(browserContext,@@ -78,40 +83,40 @@ describe('browser lane', () => {env.switch,new IssuedSecrets(env.secretsFile));- const observer = new ScenarioObserver(env.serverLog, switchClient, names.tag);+ const observer = watch.begin(switchClient, names.tag);let failure: unknown = null;+ let problems: string[];+ let leftovers: string[];try {- await scenario.run({ env, names, switchClient, fixtures, ui, log });- } catch (error) {- failure = error;- }- const problems = [...ui.recorded()];- if ((failure !== null || problems.length > 0) && env.outputDir !== null) {- const dir = path.join(env.outputDir, 'screenshots');- mkdirSync(dir, { recursive: true });- const file = path.join(dir, `${scenario.header.id}.png`);- await ui.screenshot(file);- log(`screenshot of the last page: ${file}`);+ try {+ await scenario.run({ env, names, switchClient, fixtures, ui, log });+ } catch (error) {+ failure = error;+ }+ problems = [...ui.recorded()];+ if ((failure !== null || problems.length > 0) && env.outputDir !== null) {+ const dir = path.join(env.outputDir, 'screenshots');+ mkdirSync(dir, { recursive: true });+ const file = path.join(dir, `${scenario.header.id}.png`);+ await ui.screenshot(file);+ log(`screenshot of the last page: ${file}`);+ }+ await browserContext.close();+ leftovers = await fixtures.teardown();+ } finally {+ observer.stop();}- await browserContext.close();- const leftovers = await fixtures.teardown();for (const line of leftovers) log(`teardown: ${line}`);- const serverErrors = await observer.finish();- if (serverErrors !== null && serverErrors.length > 0) {- log(`the server logged errors:\n ${describeErrors(serverErrors, observer.tag)}`);- }+ const serverErrors = observer.errors(failure, log);if (failure !== null) throw failure;expect(problems, `the pages went wrong:\n ${problems.join('\n ')}\n`).toEqual([]);if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);- expectNoServerErrors(serverErrors, observer.tag);+ expectNoServerErrors(serverErrors, observer.tag, 'while this ran');});}-});-function expectNoServerErrors(errors: ServerLogError[] | null, tag: string): void {- if (errors === null) return;- expect(- errors,- `the server logged errors while this ran:\n ${describeErrors(errors, tag)}\n`- ).toEqual([]);-}+ it('the server logged no errors outside the scenarios', async (context) => {+ if (!lane.target) return context.skip(lane.skip);+ expectNoServerErrors(await watch.sweep(), null, 'outside any scenario');+ });+});diff --git a/console/tooling-e2e/src/browser/gateway-ui.ts b/console/tooling-e2e/src/browser/gateway-ui.tsindex 2c464f8b4..ceb1b5300 100644--- a/console/tooling-e2e/src/browser/gateway-ui.ts+++ b/console/tooling-e2e/src/browser/gateway-ui.ts@@ -1,4 +1,4 @@-import type { BrowserContext, ConsoleMessage, Locator, Page, Response } from 'playwright';+import type { BrowserContext, ConsoleMessage, Locator, Page, Request, Response } from 'playwright';import { expect } from 'vitest';import type { SwitchEnv } from '../core/env.ts';import type { IssuedSecrets } from '../core/secrets.ts';@@ -6,6 +6,24 @@ import type { IssuedSecrets } from '../core/secrets.ts';/** The session cookie the gateway sets on sign-in. */const SESSION_COOKIE = 'switch_auth';+/** How long a page must go without a `/gateway` request in flight to count as settled. */+const QUIET_MS = 500;+/** How long a page may take to settle before what it is still waiting for is a problem. */+const SETTLE_TIMEOUT_MS = 15_000;++/**+ * Requests that stay open for as long as the page does, and so never finish:+ * waiting for them would never settle.+ */+const STREAMING = new Set(['eventsource', 'websocket']);++/**+ * How each browser reports a request it cancelled itself: the page navigated+ * away mid-request, or a component unmounted and aborted its fetch. Nothing+ * went wrong on the server.+ */+const ABORTED = /net::ERR_ABORTED|NS_BINDING_ABORTED|cancelled/i;+/*** The gateway UI in one browser context, and everything that went wrong on its* pages while a scenario drove it.@@ -27,6 +45,9 @@ export class GatewayUi {private readonly credentials: SwitchEnv;private readonly secrets: IssuedSecrets;private readonly problems: string[] = [];+ /** Each `/gateway` request still in flight, with the page that made it. */+ private readonly inFlight = new Map<Request, string>();+ private lastActivityMs = Date.now();private signedIn = false;constructor(@@ -43,15 +64,31 @@ export class GatewayUi {this.secrets = secrets;page.on('pageerror', (error) => this.problems.push(`page error: ${error.message}`));page.on('console', (entry) => this.onConsole(entry));+ page.on('request', (request) => {+ if (!this.isGatewayApi(request.url()) || STREAMING.has(request.resourceType())) return;+ this.inFlight.set(request, this.path());+ this.lastActivityMs = Date.now();+ });page.on('response', (response) => this.onResponse(response));+ page.on('requestfinished', (request) => this.settle(request));page.on('requestfailed', (request) => {+ const from = this.settle(request);if (!this.isGatewayApi(request.url())) return;+ const reason = request.failure()?.errorText ?? 'unknown';+ if (ABORTED.test(reason)) return;this.problems.push(- `${request.method()} ${this.pathOf(request.url())} failed: ${request.failure()?.errorText ?? 'unknown'}`+ `${request.method()} ${this.pathOf(request.url())} failed on ${from}: ${reason}`);});}+ /** The request is done: forget it, and return the page that made it. */+ private settle(request: Request): string {+ const from = this.inFlight.get(request) ?? this.path();+ if (this.inFlight.delete(request)) this.lastActivityMs = Date.now();+ return from;+ }+/** Every problem recorded on the pages so far. */recorded(): readonly string[] {return this.problems;@@ -105,11 +142,28 @@ export class GatewayUi {/*** Wait until the page has stopped fetching, then fail on an error it shows.- * MUI renders every error banner as an `alert`; a page with nothing wrong- * shows none with error severity.+ *+ * A route change in the app is not a page load, so the browser's own+ * `networkidle` was reached long ago and says nothing about the new page.+ * Settled is instead no `/gateway` request in flight for `QUIET_MS`, which+ * also covers a page whose effects fetch a moment after it renders. MUI+ * renders every error banner as an `alert`; a page with nothing wrong shows+ * none with error severity.*/async settled(): Promise<void> {- await this.page.waitForLoadState('networkidle');+ const deadline = Date.now() + SETTLE_TIMEOUT_MS;+ while (this.inFlight.size > 0 || Date.now() - this.lastActivityMs < QUIET_MS) {+ if (Date.now() > deadline) {+ const waiting = [...this.inFlight.keys()].map(+ (request) => `${request.method()} ${this.pathOf(request.url())}`+ );+ this.problems.push(+ `${this.path()} had not settled after ${SETTLE_TIMEOUT_MS / 1000} s, still waiting on: ${waiting.join(', ')}`+ );+ break;+ }+ await this.page.waitForTimeout(50);+ }const alerts = this.page.locator('[role="alert"].MuiAlert-colorError');const shown = await alerts.allInnerTexts();for (const text of shown) this.problems.push(`${this.path()} shows an error: ${text.trim()}`);@@ -147,8 +201,9 @@ export class GatewayUi {const status = response.status();const failed = status >= 500 || (this.signedIn && status >= 400);if (!failed) return;+ const from = this.inFlight.get(response.request()) ?? this.path();this.problems.push(- `${response.request().method()} ${this.pathOf(response.url())} answered ${status} on ${this.path()}`+ `${response.request().method()} ${this.pathOf(response.url())} answered ${status} on ${from}`);}diff --git a/console/tooling-e2e/src/ci-summary.test.ts b/console/tooling-e2e/src/ci-summary.test.tsindex 33607ef20..e95879de1 100644--- a/console/tooling-e2e/src/ci-summary.test.ts+++ b/console/tooling-e2e/src/ci-summary.test.ts@@ -1,5 +1,9 @@-import { describe, expect, it } from 'vitest';-import { parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import path from 'node:path';+import { describe, expect, it, vi } from 'vitest';+import { main, parseScenarioCases, summaryLine, summaryMarkdown } from './ci-summary.ts';+import { LANES } from './core/env.ts';const REPORT = `<?xml version="1.0" encoding="UTF-8" ?><testsuites name="vitest tests" tests="4">@@ -44,3 +48,23 @@ describe('CI run summary', () => {expect(summaryMarkdown('fast on dev', [])).toContain('holds no end-to-end scenarios');});});++describe('CI run summary, as the workflow calls it', () => {+ it('takes every lane, and refuses one that is not', () => {+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-ci-summary-'));+ const report = path.join(dir, 'junit.xml');+ writeFileSync(report, REPORT);+ vi.stubEnv('GITHUB_OUTPUT', '');+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);+ try {+ for (const lane of LANES) {+ expect(() => main([report, lane, `${lane} lane on ephemeral`])).not.toThrow();+ }+ expect(() => main([report, 'nightly', 'title'])).toThrow(/Usage/);+ } finally {+ write.mockRestore();+ vi.unstubAllEnvs();+ rmSync(dir, { recursive: true, force: true });+ }+ });+});diff --git a/console/tooling-e2e/src/ci-summary.ts b/console/tooling-e2e/src/ci-summary.tsindex a7c235591..ddb70fbe5 100644--- a/console/tooling-e2e/src/ci-summary.ts+++ b/console/tooling-e2e/src/ci-summary.ts@@ -10,7 +10,7 @@* get as far as its suite, and a summary that read "0 passed" would hide that.*/import { appendFileSync, readFileSync } from 'node:fs';-import type { Lane } from './core/env.ts';+import { isLane, LANES, type Lane } from './core/env.ts';import { message } from './core/suite.ts';export type CaseStatus = 'pass' | 'fail' | 'skip';@@ -92,10 +92,10 @@ export function summaryMarkdown(title: string, cases: readonly ScenarioCase[]):].join('\n');}-function main(argv: readonly string[]): void {+export function main(argv: readonly string[]): void {const [reportPath, lane, title] = argv;- if (!reportPath || (lane !== 'fast' && lane !== 'provider') || !title) {- throw new Error('Usage: ci-summary.ts <junit.xml> <fast|provider> <title>');+ if (!reportPath || lane === undefined || !isLane(lane) || !title) {+ throw new Error(`Usage: ci-summary.ts <junit.xml> <${LANES.join('|')}> <title>`);}let xml: string;try {diff --git a/console/tooling-e2e/src/core/env.ts b/console/tooling-e2e/src/core/env.tsindex 59d5e0f09..1ef5947f8 100644--- a/console/tooling-e2e/src/core/env.ts+++ b/console/tooling-e2e/src/core/env.ts@@ -22,6 +22,13 @@ import { fileURLToPath } from 'node:url';*/export type Lane = 'fast' | 'provider' | 'browser';+/** Every lane: the one list anything that takes a lane by name checks against. */+export const LANES: readonly Lane[] = ['fast', 'provider', 'browser'];++export function isLane(raw: string): raw is Lane {+ return (LANES as readonly string[]).includes(raw);+}+/*** What the target is. Scenarios declare which they may run against.*@@ -185,8 +192,8 @@ export function laneFrom(): Lane | null {const raw = reader(source, file)('SWITCH_E2E_LANE');if (raw === undefined) return null;- if (raw === 'fast' || raw === 'provider' || raw === 'browser') return raw;- throw new Error(`SWITCH_E2E_LANE must be 'fast', 'provider' or 'browser' — got '${raw}'.`);+ if (isLane(raw)) return raw;+ throw new Error(`SWITCH_E2E_LANE must be one of ${LANES.join(', ')} — got '${raw}'.`);}/**