Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

E2E: address review of the facts, own rooms and route checks

Petr Bauchc605a2873dc1
diff --git a/console/tooling-e2e/README.md b/console/tooling-e2e/README.md
index cd7f02c2d..4273b7949 100644
--- a/console/tooling-e2e/README.md
+++ b/console/tooling-e2e/README.md
@@ -305,7 +305,17 @@ a room of its own, provisioned through the gateway's `POST /rooms/from-yaml`
from the same document an agent's `create_room_from_yaml` takes. A scenario
may set `description`, `instructions`, `roles` and `docs`; the harness names
the room and picks its bridge and members (the agent, the bridge's witness
-bot, the human). The room is removed afterwards.
+bot, the human). The room is removed afterwards. Its text may not contain a
+`{name}` placeholder: the server fills those in a room document and cannot
+escape one, so the agent would read something other than what the facts and
+the judge are checked against.
+
+A fact's pattern is matched anywhere in a reply, so give it word boundaries
+(`\bten\b`, not `ten`, which matches "often"), and prefer `any` to `last`
+where the agent may answer in an earlier reply than the final one.
+
+The delivery check matches each driver message to a room message of its own,
+in order, after folding spacing and quote style; a leading mention is allowed.
- **The driver** is a model playing the human. After each reply it writes the
next message from the persona, the goal and the conversation so far, or
@@ -444,9 +454,11 @@ on every PR with no server.
(`tenant_id`, `room_id`, `agent_id`, `agent_name`) with that workspace's
values. 403 or 404 passes; 2xx, 3xx, 5xx or an HTML page fails. Other 4xx
answers (mostly a body the probe did not send) are listed as
- inconclusive. The prober is not the harness admin: a deployment admin
- may cross workspaces by design. Workspaces and users cannot be deleted,
- so this runs only on `ephemeral` and `local`.
+ inconclusive. Reads go first, then writes, then deletes, so a write or
+ delete that leaked cannot turn the probes after it into 404s. The prober
+ is not the harness admin: a deployment admin may cross workspaces by
+ design. The room and the agent are removed afterwards, but workspaces
+ and users cannot be deleted, so this runs only on `ephemeral`.
## In CI
diff --git a/console/tooling-e2e/agentic/clarify-before-acting.yaml b/console/tooling-e2e/agentic/clarify-before-acting.yaml
index 77c1f3b43..ef701f450 100644
--- a/console/tooling-e2e/agentic/clarify-before-acting.yaml
+++ b/console/tooling-e2e/agentic/clarify-before-acting.yaml
@@ -20,7 +20,7 @@ facts:
matches: '\?'
- id: plan-mentions-local-time
turn: any
- matches: 'local|time ?zone'
+ matches: '\blocal|\btime ?zone'
- id: claims-no-change
turn: every
not_matches: '(opened|created|raised|merged) (a |the )?(pr|pull request)|i(''|’)?ve (fixed|shipped|deployed|pushed)|i have (fixed|shipped|deployed|pushed)'
diff --git a/console/tooling-e2e/agentic/explain-then-follow-up.yaml b/console/tooling-e2e/agentic/explain-then-follow-up.yaml
index c5d92cfdc..01627d126 100644
--- a/console/tooling-e2e/agentic/explain-then-follow-up.yaml
+++ b/console/tooling-e2e/agentic/explain-then-follow-up.yaml
@@ -15,7 +15,7 @@ max_turns: 3
facts:
- id: explains-timing
turn: first
- matches: 'concurren|simultaneous|same time|order|timing|interleav'
+ matches: '\b(concurren|simultaneous|same time|race|ordering|order of|timing|interleav)'
- id: shows-python
turn: any
matches: '```|import (threading|asyncio|multiprocessing)|def [a-z_]+\('
diff --git a/console/tooling-e2e/agentic/follow-room-instructions.yaml b/console/tooling-e2e/agentic/follow-room-instructions.yaml
index 8b31344fa..ed3b90581 100644
--- a/console/tooling-e2e/agentic/follow-room-instructions.yaml
+++ b/console/tooling-e2e/agentic/follow-room-instructions.yaml
@@ -23,7 +23,7 @@ facts:
matches: '-- ?payments-release'
- id: names-a-weekday
turn: first
- matches: 'monday|tuesday|wednesday|thursday'
+ matches: '\b(monday|tuesday|wednesday|thursday)'
rubric:
- id: follows-the-room
criterion: Every reply from the agent ends with the line "-- payments-release", as the room's instructions require.
diff --git a/console/tooling-e2e/agentic/summarise-the-channel.yaml b/console/tooling-e2e/agentic/summarise-the-channel.yaml
index e1337a82f..b0703000e 100644
--- a/console/tooling-e2e/agentic/summarise-the-channel.yaml
+++ b/console/tooling-e2e/agentic/summarise-the-channel.yaml
@@ -17,17 +17,17 @@ opening: Just got back. What was decided above about the weekly sync?
max_turns: 2
facts:
- id: names-the-day
- turn: last
- matches: 'wednesday'
+ turn: any
+ matches: '\bwednesday'
- id: names-the-time
- turn: last
- matches: '10(:00)?|ten'
+ turn: any
+ matches: '\b10(:00)?(\s*a\.?m\.?)?\b|\bten\b'
- id: names-the-invite
- turn: last
- matches: 'calendar|invite'
+ turn: any
+ matches: '\b(calendar|invite)'
- id: invents-no-other-day
turn: every
- not_matches: 'monday|tuesday|friday'
+ not_matches: '\b(monday|tuesday|friday)'
rubric:
- id: names-the-decision
criterion: The agent says the sync moves to Wednesday at 10:00, starting next week.
diff --git a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
index 05870a2dc..10af75ca0 100644
--- a/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
+++ b/console/tooling-e2e/src/agentic/agentic.e2e.test.ts
@@ -226,7 +226,9 @@ describe('agentic lane', () => {
const history = await h.switch.roomHistory(h.witness, active.room.id, 100);
const lost = undelivered(
turns,
- history.map((event) => event.body)
+ history
+ .toSorted((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0))
+ .map((event) => event.body)
);
if (lost.length > 0) {
throw new Error(
diff --git a/console/tooling-e2e/src/agentic/agentic.test.ts b/console/tooling-e2e/src/agentic/agentic.test.ts
index e3f8de773..67e3769d9 100644
--- a/console/tooling-e2e/src/agentic/agentic.test.ts
+++ b/console/tooling-e2e/src/agentic/agentic.test.ts
@@ -61,6 +61,15 @@ describe('agentic scenario files', () => {
).toThrow(/'room' may not set 'name'[\s\S]*'room' may not set 'agents'/);
});
+ it("refuses a room text the server's templating would change", () => {
+ const templated = `${SCENARIO}room:\n instructions: Report to {$creator} and {team}.\n docs:\n - name: '{x}'\n`;
+ expect(() => parseScenario('some-scenario.yaml', templated)).toThrow(
+ /placeholder '\{\$creator\}'[\s\S]*placeholder '\{team\}'[\s\S]*placeholder '\{x\}'/
+ );
+ const braces = `${SCENARIO}room:\n instructions: 'Reply in JSON, like {"ok": true}.'\n`;
+ expect(parseScenario('some-scenario.yaml', braces).room).not.toBeNull();
+ });
+
it('refuses a scenario with no facts, and facts that cannot be checked', () => {
const noFacts = SCENARIO.replace(/facts:[\s\S]*?rubric:/, 'rubric:');
expect(() => parseScenario('some-scenario.yaml', noFacts)).toThrow(/at least one fact/);
@@ -174,6 +183,23 @@ describe('facts', () => {
];
expect(undelivered(turns, history)).toEqual(['The payments one.']);
});
+
+ it('does not count a short message as delivered because a longer or earlier body contains it', () => {
+ const short: Turn[] = [
+ { from: 'human', text: 'Yes.' },
+ { from: 'agent', text: 'Done.' },
+ { from: 'human', text: 'Yes.' },
+ ];
+ expect(undelivered(short, ['@e2e-abc123b-claude Yes.'])).toEqual(['Yes.']);
+ expect(undelivered(short, ['Yes. Ship it.', '@e2e-abc123b-claude Yes.'])).toEqual(['Yes.']);
+ });
+
+ it("matches across a platform's spacing and quote style", () => {
+ const quoted: Turn[] = [{ from: 'human', text: 'It\u2019s the "payments" one.' }];
+ expect(
+ undelivered(quoted, ["@e2e-abc123b-claude It's the \u201Cpayments\u201D one.\n"])
+ ).toEqual([]);
+ });
});
describe('prompt versions', () => {
diff --git a/console/tooling-e2e/src/agentic/facts.ts b/console/tooling-e2e/src/agentic/facts.ts
index c2b275845..67c4f34d6 100644
--- a/console/tooling-e2e/src/agentic/facts.ts
+++ b/console/tooling-e2e/src/agentic/facts.ts
@@ -64,10 +64,41 @@ function checkFact(fact: Fact, replies: readonly string[]): FactResult {
}
}
-/** The human messages that are not in the room's history: written by the driver, never delivered. */
+/**
+ * The human messages that are not in the room's history: written by the
+ * driver, never delivered.
+ *
+ * `history` is the room's message bodies, oldest first. Each driver message
+ * has to match a body of its own, later than the one the message before it
+ * matched, so a lost "Yes." is not covered by an earlier "Yes." or by a longer
+ * message that contains it. A body matches when it is the message, or the
+ * message after a leading mention (the bridge carries the human's
+ * `@agent ...` as written). Both are compared after `normalise`, so a
+ * platform's spacing or quote style does not read as a lost message.
+ */
export function undelivered(turns: readonly Turn[], history: readonly string[]): string[] {
- return turns
- .filter((turn) => turn.from === 'human' || turn.from === 'channel')
- .map((turn) => turn.text)
- .filter((text) => !history.some((body) => body.includes(text)));
+ const bodies = history.map(normalise);
+ const lost: string[] = [];
+ let next = 0;
+ for (const turn of turns) {
+ if (turn.from !== 'human' && turn.from !== 'channel') continue;
+ const text = normalise(turn.text);
+ const index = bodies.findIndex(
+ (body, i) =>
+ i >= next && (body === text || (body.startsWith('@') && body.endsWith(` ${text}`)))
+ );
+ if (index === -1) lost.push(turn.text);
+ else next = index + 1;
+ }
+ return lost;
+}
+
+/** Unicode compatibility forms folded, curly quotes made straight, runs of whitespace made one space. */
+export function normalise(text: string): string {
+ return text
+ .normalize('NFKC')
+ .replace(/[\u2018\u2019\u201A\u2032]/g, "'")
+ .replace(/[\u201C\u201D\u201E\u2033]/g, '"')
+ .replace(/\s+/g, ' ')
+ .trim();
}
diff --git a/console/tooling-e2e/src/agentic/room.ts b/console/tooling-e2e/src/agentic/room.ts
index 95d50435b..14e7e9418 100644
--- a/console/tooling-e2e/src/agentic/room.ts
+++ b/console/tooling-e2e/src/agentic/room.ts
@@ -45,12 +45,17 @@ export async function provisionScenarioRoom(
const remove = async (log: (message: string) => void): Promise<void> => {
try {
await switchClient.deleteRoom(room.id);
- if (channelId !== null) {
- const left = await mattermost.getChannel(channelId);
- if (!left.delete_at) await mattermost.archiveChannel(channelId);
- }
} catch (error) {
- log(`could not remove room ${room.name} (${room.id}) — delete it by hand: ${message(error)}`);
+ log(`could not delete room ${room.name} (${room.id}) — delete it by hand: ${message(error)}`);
+ }
+ if (channelId === null) return;
+ try {
+ const left = await mattermost.getChannel(channelId);
+ if (!left.delete_at) await mattermost.archiveChannel(channelId);
+ } catch (error) {
+ log(
+ `could not archive Mattermost channel ${channelId} of room ${room.name} — archive it by hand: ${message(error)}`
+ );
}
};
try {
diff --git a/console/tooling-e2e/src/agentic/scenario-file.ts b/console/tooling-e2e/src/agentic/scenario-file.ts
index d811d252b..24618f6f1 100644
--- a/console/tooling-e2e/src/agentic/scenario-file.ts
+++ b/console/tooling-e2e/src/agentic/scenario-file.ts
@@ -65,6 +65,21 @@ export interface Fact {
/** What a scenario's `room:` may say. The rest of a room document is the harness's to fill in. */
const ROOM_KEYS = new Set(['description', 'instructions', 'roles', 'docs']);
+/** The server's room-template placeholder, `{name}` or `{$name}` (`PLACEHOLDER_RE` in `rooms_yaml.py`). */
+const PLACEHOLDER = /\{\$?[A-Za-z_][A-Za-z0-9_]*\}/g;
+
+function placeholdersIn(node: unknown): string[] {
+ if (typeof node === 'string') return node.match(PLACEHOLDER) ?? [];
+ if (Array.isArray(node)) return node.flatMap(placeholdersIn);
+ if (typeof node === 'object' && node !== null) {
+ return Object.entries(node).flatMap(([key, value]) => [
+ ...placeholdersIn(key),
+ ...placeholdersIn(value),
+ ]);
+ }
+ return [];
+}
+
/** Where the scenario files live: `console/tooling-e2e/agentic/`. */
export const AGENTIC_SCENARIO_DIR = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
@@ -167,6 +182,11 @@ export function parseScenario(file: string, raw: string): AgenticScenario {
);
}
}
+ for (const placeholder of new Set(placeholdersIn(room))) {
+ problems.push(
+ `'room' contains the placeholder '${placeholder}': the server fills placeholders in a room document and has no way to escape one, so the agent would read other text than the scenario wrote`
+ );
+ }
}
}
diff --git a/console/tooling-e2e/src/browser/index.ts b/console/tooling-e2e/src/browser/index.ts
index edce6b885..1b3e88232 100644
--- a/console/tooling-e2e/src/browser/index.ts
+++ b/console/tooling-e2e/src/browser/index.ts
@@ -75,12 +75,24 @@ const createRoom: BrowserScenario = {
await ui.page.getByRole('textbox', { name: /^Description/ }).fill(description);
await ui.page.getByRole('combobox', { name: 'Agents' }).fill(agent.name);
await ui.page.getByRole('option').filter({ hasText: agent.name }).click();
+ // The room is known from the create call's answer, not from where the page
+ // goes next, so it is removed even if the page never gets there.
+ const created = ui.page.waitForResponse(
+ (response) =>
+ response.request().method() === 'POST' &&
+ new URL(response.url()).pathname.endsWith('/gateway/rooms'),
+ { timeout: 15_000 }
+ );
await ui.page.getByRole('button', { name: 'Create', exact: true }).click();
-
- await expect.poll(() => ui.path(), { timeout: 15_000 }).toMatch(/^\/rooms\/[0-9a-f-]{36}$/);
- const roomId = ui.path().slice('/rooms/'.length);
+ const response = await created;
+ expect(response.status(), 'the create call failed').toBeLessThan(300);
+ const roomId = ((await response.json()) as { id?: unknown }).id;
+ if (typeof roomId !== 'string') {
+ throw new Error(`the create call answered without a room id: ${await response.text()}`);
+ }
fixtures.defer(`room ${name} (${roomId})`, () => switchClient.deleteRoom(roomId));
log(`created room ${name} (${roomId}) from the form`);
+ await expect.poll(() => ui.path(), { timeout: 15_000 }).toBe(`/rooms/${roomId}`);
const room = await switchClient.getRoom(roomId);
expect(room.name).toBe(name);
diff --git a/console/tooling-e2e/src/clients/switch-client.ts b/console/tooling-e2e/src/clients/switch-client.ts
index 39bb9bb73..65a1715c3 100644
--- a/console/tooling-e2e/src/clients/switch-client.ts
+++ b/console/tooling-e2e/src/clients/switch-client.ts
@@ -111,6 +111,8 @@ export function eventText(event: AgentEvent): string {
export interface RouteProbe {
status: number;
contentType: string;
+ /** The `X-Request-Id` the request carried, which the server's log lines for it name; null untagged. */
+ requestId: string | null;
}
export class SwitchHttpError extends Error {
@@ -162,8 +164,9 @@ export class SwitchClient {
private readonly environment: Environment;
private readonly issued: IssuedSecrets;
private gatewayCookie: string | null = null;
- private requestTag: string | null = null;
- private requestCount = 0;
+ // Shared with every client `as()` makes, so one scenario's requests carry
+ // one tag, numbered without repeats, whichever user sent them.
+ private tagging: { tag: string | null; count: number } = { tag: null, count: 0 };
constructor(env: SwitchEnv, environment: Environment, issued: IssuedSecrets) {
this.apiUrl = env.apiUrl;
@@ -181,14 +184,14 @@ export class SwitchClient {
* server stamps on each log line it writes while serving it. `null` stops.
*/
tagRequests(tag: string | null): void {
- this.requestTag = tag;
- this.requestCount = 0;
+ this.tagging.tag = tag;
+ this.tagging.count = 0;
}
private requestId(): Record<string, string> {
- if (this.requestTag === null) return {};
- this.requestCount += 1;
- return { 'X-Request-Id': `${this.requestTag}-${this.requestCount}` };
+ if (this.tagging.tag === null) return {};
+ this.tagging.count += 1;
+ return { 'X-Request-Id': `${this.tagging.tag}-${this.tagging.count}` };
}
private async request<T>(method: string, path: string, init: SwitchRequest = {}): Promise<T> {
@@ -242,26 +245,33 @@ export class SwitchClient {
private async probe(method: string, path: string, cookie: string | null): Promise<RouteProbe> {
assertAllowed(this.environment, method, path);
+ const tag = this.requestId();
const response = await fetch(`${this.apiUrl}${path}`, {
method,
headers: {
Accept: 'application/json',
...(cookie === null ? {} : { Cookie: cookie }),
- ...this.requestId(),
+ ...tag,
},
redirect: 'manual',
signal: AbortSignal.timeout(30_000),
});
await response.body?.cancel();
- return { status: response.status, contentType: response.headers.get('content-type') ?? '' };
+ return {
+ status: response.status,
+ contentType: response.headers.get('content-type') ?? '',
+ requestId: tag['X-Request-Id'] ?? null,
+ };
}
/**
* The same server, signed in as someone else. Shares this client's record of
- * issued credentials, so the other session is scanned for too.
+ * issued credentials, so the other session is scanned for too, and its
+ * request tagging, so a scenario observing this client sees the other's
+ * requests as its own.
*/
as(email: string, password: string): SwitchClient {
- return new SwitchClient(
+ const other = new SwitchClient(
{
apiUrl: this.apiUrl,
agentRegistrationToken: this.registrationToken,
@@ -271,6 +281,8 @@ export class SwitchClient {
this.environment,
this.issued
);
+ other.tagging = this.tagging;
+ return other;
}
/** `POST /gateway/users` — a user in the signed-in admin's workspace. `role: 'admin'` makes a deployment operator. */
@@ -567,14 +579,38 @@ export class SwitchClient {
if (result.room_id === undefined) {
throw new Error(`from-yaml did not make one room: ${JSON.stringify(result)}`);
}
+ const roomId = result.room_id;
if (result.failed_attachments !== undefined && result.failed_attachments.length > 0) {
- await this.deleteRoom(result.room_id);
- throw new Error(
- `from-yaml made room ${result.room_id} without part of what it was given, so it was deleted: ` +
- JSON.stringify(result.failed_attachments)
+ throw await this.withRoomDeleted(
+ roomId,
+ new Error(
+ `from-yaml made room ${roomId} without part of what it was given: ` +
+ JSON.stringify(result.failed_attachments)
+ )
+ );
+ }
+ try {
+ return await this.getRoom(roomId);
+ } catch (error) {
+ throw await this.withRoomDeleted(roomId, error);
+ }
+ }
+
+ /** Delete a room a call made before failing, and return the failure, saying whether the room went too. */
+ private async withRoomDeleted(roomId: string, failure: unknown): Promise<Error> {
+ const reason = failure instanceof Error ? failure.message : String(failure);
+ try {
+ await this.deleteRoom(roomId);
+ return new Error(`${reason} (room ${roomId} was deleted)`, { cause: failure });
+ } catch (cleanup) {
+ const why = cleanup instanceof Error ? cleanup.message : String(cleanup);
+ return new Error(
+ `${reason} (and room ${roomId} could not be deleted, delete it by hand: ${why})`,
+ {
+ cause: failure,
+ }
);
}
- return this.getRoom(result.room_id);
}
/**
diff --git a/console/tooling-e2e/src/fast/cross-tenant.ts b/console/tooling-e2e/src/fast/cross-tenant.ts
index ad45a6bad..4e4a62c6e 100644
--- a/console/tooling-e2e/src/fast/cross-tenant.ts
+++ b/console/tooling-e2e/src/fast/cross-tenant.ts
@@ -1,5 +1,6 @@
import type { AgentRef, RoomDetail, SwitchClient } from '../clients/switch-client.ts';
import type { NameScope } from '../core/run.ts';
+import type { Fixtures } from '../fixtures/fixtures.ts';
import type { ServerSurface } from '../gate/gate.ts';
import { isHtml } from './session-required.ts';
@@ -25,7 +26,8 @@ import { isHtml } from './session-required.ts';
* not failed: a validation error answered before the tenant is checked is
* not a leak.
*
- * Workspaces cannot be deleted, so this runs only on a throwaway stack.
+ * Workspaces and users cannot be deleted, so this runs only on a stack that
+ * is thrown away after the run. The room and the agent are removed.
*/
export interface CrossTenantProbe {
@@ -37,6 +39,8 @@ export interface CrossTenantProbe {
inconclusive: string[];
/** Served, changed or broke on another workspace's resource, with what. */
wrong: string[];
+ /** The route each request id was sent to, to name the route behind a server log line. */
+ routeOf: Map<string, string>;
}
export interface OtherTenant {
@@ -45,11 +49,16 @@ export interface OtherTenant {
agent: AgentRef;
}
-/** Make the second workspace, its room and its agent, and the plain member who probes it. */
+/**
+ * Make the second workspace, its room and its agent, and the plain member who
+ * probes it. The room and the agent are deferred to `fixtures` as each is
+ * made, so a failure part way still removes what was made.
+ */
export async function otherTenantAndProber(
admin: SwitchClient,
- names: NameScope
-): Promise<{ other: OtherTenant; owner: SwitchClient; prober: SwitchClient }> {
+ names: NameScope,
+ fixtures: Fixtures
+): Promise<{ other: OtherTenant; prober: SwitchClient }> {
const password = `e2e-${names.runId}-${crypto.randomUUID()}`;
const ownerEmail = `${names.name('owner')}@e2e.invalid`;
const proberEmail = `${names.name('prober')}@e2e.invalid`;
@@ -66,6 +75,9 @@ export async function otherTenantAndProber(
name: names.name('theirs'),
description: `Switch end-to-end harness, another workspace's agent (run ${names.runId}).`,
});
+ fixtures.defer(`the other workspace's agent ${agent.name}`, () =>
+ owner.deleteAgentByName(agent.name)
+ );
const room = await owner.createRoom({
name: names.name('their-room'),
description: `Switch end-to-end harness, another workspace's room (run ${names.runId}).`,
@@ -73,7 +85,17 @@ export async function otherTenantAndProber(
agentNames: [agent.name],
userNames: [],
});
- return { other: { tenantId, room, agent }, owner, prober: admin.as(proberEmail, password) };
+ fixtures.defer(`the other workspace's room ${room.name} (${room.id})`, () =>
+ owner.deleteRoom(room.id)
+ );
+ return { other: { tenantId, room, agent }, prober: admin.as(proberEmail, password) };
+}
+
+/** 0 for a read, 1 for a write, 2 for a delete; ties keep the snapshot's order. */
+export function probeOrder(route: string): number {
+ const method = route.split(' ', 1)[0];
+ if (method === 'GET' || method === 'HEAD') return 0;
+ return method === 'DELETE' ? 2 : 1;
}
export function crossTenantPath(
@@ -105,22 +127,33 @@ export async function probeCrossTenant(
enabledFlags: ReadonlySet<string>,
other: OtherTenant
): Promise<CrossTenantProbe> {
- const result: CrossTenantProbe = { refused: 0, unprobed: 0, inconclusive: [], wrong: [] };
- // Reads first: a leaking DELETE would otherwise turn every later probe of the
- // same resource into a 404 that looks like a refusal.
+ const result: CrossTenantProbe = {
+ refused: 0,
+ unprobed: 0,
+ inconclusive: [],
+ wrong: [],
+ routeOf: new Map(),
+ };
+ // Reads, then writes, then deletes: a write that leaked could archive or
+ // move the resource, and a delete that leaked removes it, either way turning
+ // every later probe of it into a 404 that looks like a refusal.
const routes = surface.gatewayRoutes
.filter(
({ auth, route, flags }) =>
auth === 'session' && /\{/.test(route) && flags.every((f) => enabledFlags.has(f))
)
- .sort((a, b) => Number(a.route.startsWith('DELETE ')) - Number(b.route.startsWith('DELETE ')));
+ .sort((a, b) => probeOrder(a.route) - probeOrder(b.route));
for (const { route } of routes) {
const target = crossTenantPath(route, other);
if (target === null) {
result.unprobed += 1;
continue;
}
- const { status, contentType } = await prober.statusWithSession(target.method, target.path);
+ const { status, contentType, requestId } = await prober.statusWithSession(
+ target.method,
+ target.path
+ );
+ if (requestId !== null) result.routeOf.set(requestId, route);
const answered = `${route} -> ${status} ${contentType || '(no content type)'}`;
if (isHtml(contentType) || status < 400 || status >= 500) result.wrong.push(answered);
else if (status === 403 || status === 404) result.refused += 1;
diff --git a/console/tooling-e2e/src/fast/fast.e2e.test.ts b/console/tooling-e2e/src/fast/fast.e2e.test.ts
index 2c04a8ffe..c3d898e3b 100644
--- a/console/tooling-e2e/src/fast/fast.e2e.test.ts
+++ b/console/tooling-e2e/src/fast/fast.e2e.test.ts
@@ -10,7 +10,7 @@ import {
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { Fixtures } from '../fixtures/fixtures.ts';
import { readSurface } from '../gate/gate.ts';
-import { otherTenantAndProber, probeCrossTenant } from './cross-tenant.ts';
+import { otherTenantAndProber, probeCrossTenant, type CrossTenantProbe } from './cross-tenant.ts';
import { FAST_SCENARIOS } from './index.ts';
import { probeSessionRoutes } from './session-required.ts';
@@ -107,45 +107,59 @@ describe('fast lane, derived checks', () => {
}, 120_000);
// Makes a workspace and two users, none of which can be deleted, so only on
- // a stack that is thrown away after the run.
+ // a stack that is thrown away after the run: not on `local`, which is a
+ // developer's own and lives on.
it("a workspace member is refused another workspace's rooms and agents", async (context) => {
if (!lane.target) return context.skip(lane.skip);
- const { environment } = lane.target.env;
- if (environment !== 'ephemeral' && environment !== 'local') {
+ const { env, switchClient, mattermost } = lane.target;
+ if (env.environment !== 'ephemeral') {
return context.skip(
- `leaves a workspace and users behind, which cannot be deleted; not on ${environment}`
+ `leaves a workspace and users behind, which cannot be deleted; only on ephemeral, not ${env.environment}`
);
}
- const { switchClient } = lane.target;
const names = run.scope();
+ const fixtures = new Fixtures(env, names, switchClient, mattermost);
+ const log = (message: string): void => console.log(`[cross-tenant] ${message}`);
const observer = watch.begin(switchClient, names.tag);
- let probe: Awaited<ReturnType<typeof probeCrossTenant>>;
- let removeRoom: () => Promise<void>;
+ let failure: unknown = null;
+ let probe: CrossTenantProbe | null = null;
+ let leftovers: string[];
try {
- const { other, owner, prober } = await otherTenantAndProber(switchClient, names);
- probe = await probeCrossTenant(
- prober,
- readSurface(),
- await switchClient.enabledFeatureFlags(),
- other
- );
- removeRoom = () => owner.deleteRoom(other.room.id);
+ try {
+ const { other, prober } = await otherTenantAndProber(switchClient, names, fixtures);
+ probe = await probeCrossTenant(
+ prober,
+ readSurface(),
+ await switchClient.enabledFeatureFlags(),
+ other
+ );
+ } catch (error) {
+ failure = error;
+ }
+ leftovers = await fixtures.teardown();
} finally {
observer.stop();
}
- const serverErrors = observer.errors(null, console.log);
- console.log(
- `[cross-tenant] ${probe.refused} routes refused; ${probe.inconclusive.length} answered neither way; ` +
+ for (const line of leftovers) log(`teardown: ${line}`);
+ const serverErrors = observer.errors(failure, log);
+ for (const error of serverErrors ?? []) {
+ const route = error.requestId === null ? undefined : probe?.routeOf.get(error.requestId);
+ if (route !== undefined) log(`the error at ${error.timestamp} was logged serving ${route}`);
+ }
+ if (failure !== null) throw failure;
+ if (probe === null) throw new Error('the probe neither ran nor failed');
+ log(
+ `${probe.refused} routes refused; ${probe.inconclusive.length} answered neither way; ` +
`${probe.unprobed} take a parameter the other workspace has nothing for`
);
- for (const line of probe.inconclusive) console.log(`[cross-tenant] inconclusive: ${line}`);
+ for (const line of probe.inconclusive) log(`inconclusive: ${line}`);
expect(probe.refused, 'no route refused the other workspace').toBeGreaterThan(0);
expect(
probe.wrong,
`served another workspace's resource:\n ${probe.wrong.join('\n ')}\n`
).toEqual([]);
+ if (!env.keepArtifacts) expect(leftovers, 'teardown left artefacts behind').toEqual([]);
expectNoServerErrors(serverErrors, observer.tag, 'while this ran');
- await removeRoom();
}, 180_000);
// Last, so they see whatever the run left the server in.
diff --git a/console/tooling-e2e/src/fast/route-checks.test.ts b/console/tooling-e2e/src/fast/route-checks.test.ts
index 0d53db5dd..4af8e4036 100644
--- a/console/tooling-e2e/src/fast/route-checks.test.ts
+++ b/console/tooling-e2e/src/fast/route-checks.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { AgentRef, RoomDetail } from '../clients/switch-client.ts';
-import { crossTenantPath, type OtherTenant } from './cross-tenant.ts';
+import { crossTenantPath, probeOrder, type OtherTenant } from './cross-tenant.ts';
import { isHtml } from './session-required.ts';
const OTHER: OtherTenant = {
@@ -33,6 +33,27 @@ describe('crossTenantPath', () => {
});
});
+describe('probeOrder', () => {
+ it('puts reads first, then every write, then deletes', () => {
+ const routes = [
+ 'DELETE /gateway/rooms/{room_id}',
+ 'POST /gateway/rooms/{room_id}/archive',
+ 'GET /gateway/rooms/{room_id}',
+ 'PATCH /gateway/rooms/{room_id}',
+ 'PUT /gateway/tenants/{tenant_id}/members/{user_id}',
+ 'GET /gateway/agents/{agent_id}',
+ ];
+ expect(routes.toSorted((a, b) => probeOrder(a) - probeOrder(b))).toEqual([
+ 'GET /gateway/rooms/{room_id}',
+ 'GET /gateway/agents/{agent_id}',
+ 'POST /gateway/rooms/{room_id}/archive',
+ 'PATCH /gateway/rooms/{room_id}',
+ 'PUT /gateway/tenants/{tenant_id}/members/{user_id}',
+ 'DELETE /gateway/rooms/{room_id}',
+ ]);
+ });
+});
+
describe('isHtml', () => {
it('tells a web page from an API answer', () => {
expect(isHtml('text/html; charset=utf-8')).toBe(true);