Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
import {
copyFileSync,
createWriteStream,
existsSync,
mkdirSync,
mkdtempSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { createRequire } from 'node:module';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { SwitchHttpError, type SwitchClient } from '../clients/switch-client.ts';
import {
describeEnv,
durationMs,
exploreTimeLimitFrom,
laneFrom,
readDotEnvFile,
type HarnessEnv,
} from '../core/env.ts';
import { RunContext, type NameScope } from '../core/run.ts';
import {
describeServerLogCheck,
expectNoServerErrors,
ServerLogWatch,
type ScenarioObserver,
type ServerLogError,
} from '../core/server-log.ts';
import { resolveLane, type LaneResolution } from '../core/suite.ts';
import { Fixtures } from '../fixtures/fixtures.ts';
import { PROPERTIES } from './properties.ts';
import { readTrace, type TraceSummary } from './trace.ts';
/**
* The explore lane: Bombadil drives the gateway UI at random for
* `SWITCH_E2E_EXPLORE_TIME_LIMIT`, checking the properties in `spec/gateway.ts`
* in every state it reaches. Runs when `SWITCH_E2E_LANE=explore`, with
* `SWITCH_GATEWAY_UI_URL` naming where the UI is served.
*
* The explorer deletes, archives and renames whatever it finds, so the lane
* runs only on an ephemeral stack. It starts on the login page and signs in
* from there (the spec's `signIn`), so no session cookie is ever passed on
* Bombadil's command line, which it prints back. The admin's credentials go
* beside a copy of the spec in a private temporary directory, never into the
* output.
*
* One case explores; then one case per property reports what the trace says
* about it, with the first violating state's page and screenshot; then the
* server log is checked for the whole exploration, as the browser lane checks
* it per scenario. Bombadil's own log and trace go to
* `$SWITCH_E2E_OUTPUT_DIR/explore`, readable with `bombadil browser inspect`.
*/
const SPEC_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), 'spec');
/** Bombadil's exit codes (`bombadil browser test`). */
const EXIT_CLEAN = 0;
const EXIT_STALLED = 1;
const EXIT_VIOLATIONS = 2;
/** Over the time limit, what the explorer may take to start Chromium and write its trace. */
const SLACK_MS = 3 * 60_000;
/** How long Bombadil has to stop on SIGINT, writing its trace, before it is killed. */
const KILL_GRACE_MS = 10_000;
/** The Bombadil binary the npm package ships for each platform it supports. */
const BOMBADIL_BINARIES: Record<string, string> = {
'linux-x64': 'bombadil-linux-x64',
'linux-arm64': 'bombadil-linux-arm64',
'darwin-arm64': 'bombadil-darwin-arm64',
};
const run = new RunContext();
// Read only when this lane is asked for: a value the other lanes have no use
// for must not fail them.
const timeLimit =
laneFrom(process.env, readDotEnvFile()) === 'explore' ? exploreTimeLimitFrom() : null;
let lane: LaneResolution;
let watch: ServerLogWatch;
let fixtures: Fixtures | null = null;
let names: NameScope;
let observer: ScenarioObserver | null = null;
let outputDir: string;
let trace: TraceSummary | null = null;
let serverErrors: ServerLogError[] | null = null;
let explorer: ChildProcess | null = null;
beforeAll(async () => {
lane = await resolveLane('explore');
if (!lane.target) return;
const { env, switchClient, mattermost } = lane.target;
if (env.environment !== 'ephemeral') {
// The explorer deletes what it finds; on any other stack that is someone's data.
const reason = `the explore lane runs only on an ephemeral stack, not '${env.environment}'`;
if (env.strict) throw new Error(`The explore lane cannot run (SWITCH_E2E_STRICT=1): ${reason}`);
lane = { target: null, skip: reason };
return;
}
if (env.gatewayUiUrl === null) {
throw new Error('The explore lane needs SWITCH_GATEWAY_UI_URL: where the gateway UI is served.');
}
if (env.outputDir === null) {
throw new Error(
'The explore lane needs SWITCH_E2E_OUTPUT_DIR: where Bombadil writes the trace a failure is read from.'
);
}
console.log(`explore lane run ${run.runId}: ${describeEnv(env)}`);
console.log(` ${describeServerLogCheck(env.serverLog)}`);
watch = new ServerLogWatch(env.serverLog);
outputDir = path.join(env.outputDir, 'explore');
names = run.scope();
fixtures = new Fixtures(env, names, switchClient, mattermost);
await seed(fixtures, switchClient, names);
}, 120_000);
afterAll(async () => {
if (explorer !== null) await stop(explorer);
observer?.stop();
if (fixtures === null) return;
const leftovers = await fixtures.teardown();
for (const left of leftovers) console.log(`[explore] teardown: ${left}`);
});
describe('explore lane', () => {
const timeout = timeLimit === null ? undefined : durationMs(timeLimit) + SLACK_MS;
it(`explores the gateway UI`, { timeout }, async (context) => {
if (!lane.target || timeLimit === null) return context.skip(lane.skip ?? 'not asked for');
const { env, switchClient } = lane.target;
observer = watch.begin(switchClient, names.tag, 'explore');
const exit = await explore(env, names.tag, timeLimit, context.signal);
serverErrors = observer.errors(null, (message) => console.log(`[explore] ${message}`));
observer.stop();
observer = null;
const traceFile = path.join(outputDir, 'trace.jsonl');
if (!existsSync(traceFile)) {
throw new Error(`Bombadil exited ${exit} and wrote no trace; see ${logFile()}.`);
}
trace = readTrace(traceFile);
console.log(
`[explore] ${trace.states} states, ${trace.violations.size} properties violated; ` +
`inspect with: pnpm exec bombadil browser inspect ${outputDir}`
);
if (exit === EXIT_STALLED) {
throw new Error(`The browser stalled: Bombadil stopped getting new states; see ${logFile()}.`);
}
if (exit !== EXIT_CLEAN && exit !== EXIT_VIOLATIONS) {
throw new Error(`Bombadil exited ${exit}; see ${logFile()}.`);
}
const unknown = [...trace.violations.keys()].filter(
(name) => !(PROPERTIES as readonly string[]).includes(name)
);
expect(unknown, 'violated properties missing from PROPERTIES').toEqual([]);
expect(exit === EXIT_VIOLATIONS, 'Bombadil exit code agrees with the trace').toBe(
trace.violations.size > 0
);
});
for (const property of PROPERTIES) {
it(property, (context) => {
if (!lane.target) return context.skip(lane.skip);
if (trace === null) throw new Error('The exploration did not finish, so nothing was checked.');
const violation = trace.violations.get(property);
if (violation === undefined) return;
throw new Error(
`${property} was violated ${(violation.firstAtMicros / 1e6).toFixed(1)} s in, on ${violation.url}, ` +
`and in ${violation.states} of ${trace.states} states: ${violation.detail}\n` +
` screenshot: ${violation.screenshot}`
);
});
}
it('the server logged no errors while it explored', (context) => {
if (!lane.target) return context.skip(lane.skip);
if (trace === null) throw new Error('The exploration did not finish, so nothing was checked.');
expectNoServerErrors(serverErrors, names.tag, 'while the explorer ran');
});
it('the server logged no errors outside the exploration', async (context) => {
if (!lane.target) return context.skip(lane.skip);
expectNoServerErrors(await watch.sweep(), null, 'outside the exploration');
});
});
/**
* Something on the pages: an agent, in two rooms (an internal room needs one).
* The explorer may delete them before teardown does, so one already gone is
* not a leftover.
*/
async function seed(fixtures: Fixtures, client: SwitchClient, names: NameScope): Promise<void> {
const agentName = names.name('agent');
await client.registerKnownAgent({
agentType: 'opencode',
name: agentName,
description: `Switch end-to-end harness agent (run ${names.runId}). Safe to delete.`,
options: { auto_session: false },
});
fixtures.defer(`agent ${agentName}`, () => unlessGone(client.deleteAgentByName(agentName)));
for (const role of ['room', 'other-room']) {
const name = names.name(role);
const room = await client.createRoom({
name,
description: `Switch end-to-end harness (run ${names.runId}). Safe to delete.`,
bridgeId: null,
agentNames: [agentName],
userNames: [],
});
fixtures.defer(`room ${name} (${room.id})`, () => unlessGone(client.deleteRoom(room.id)));
}
}
function logFile(): string {
return path.join(outputDir, 'bombadil.log');
}
async function unlessGone(removal: Promise<void>): Promise<void> {
try {
await removal;
} catch (error) {
if (error instanceof SwitchHttpError && error.status === 404) return;
throw error;
}
}
/**
* Run Bombadil against the gateway UI, and return its exit code.
*
* Its output goes to a log file rather than this process's: it prints a line
* per action, thousands in a run. Every request the page makes carries the
* scope's tag, so an error the server logs serving one is attributed here.
*
* When the case times out or the run is cancelled (`signal`), Bombadil and the
* Chromium it started are stopped (see `stop`), rather than going on deleting
* things on the stack while teardown runs.
*/
async function explore(
env: HarnessEnv,
tag: string,
timeLimit: string,
signal: AbortSignal
): Promise<number> {
rmSync(outputDir, { recursive: true, force: true });
mkdirSync(outputDir, { recursive: true });
const specDir = mkdtempSync(path.join(tmpdir(), 'switch-explore-spec-'));
try {
copyFileSync(path.join(SPEC_DIR, 'gateway.ts'), path.join(specDir, 'gateway.ts'));
writeFileSync(
path.join(specDir, 'credentials.json'),
JSON.stringify({
email: env.switch.gatewayAdminEmail,
password: env.switch.gatewayAdminPassword,
}),
{ mode: 0o600 }
);
const args = [
'browser',
'test',
'--headless',
'--no-sandbox',
'--time-limit',
timeLimit,
'--output-path',
outputDir,
'--width',
'1440',
'--height',
'900',
'--header',
`X-Request-Id=${tag}-ui`,
`${env.gatewayUiUrl}/login`,
path.join(specDir, 'gateway.ts'),
];
console.log(`[explore] exploring for ${timeLimit}; Bombadil's log: ${logFile()}`);
const log = createWriteStream(logFile());
const child = spawn(bombadilBinary(), args, {
// Bombadil finds no browser of its own; Playwright's is installed for this lane.
env: { ...process.env, CHROME: chromium.executablePath() },
stdio: ['ignore', 'pipe', 'pipe'],
detached: true,
});
explorer = child;
child.stdout.pipe(log);
child.stderr.pipe(log);
const onAbort = () => void stop(child);
signal.addEventListener('abort', onAbort, { once: true });
const removeGuard = guardAgainstExit(child, specDir);
try {
return await new Promise<number>((resolve, reject) => {
child.on('error', reject);
child.on('close', (exitCode, killedBy) =>
exitCode === null
? reject(new Error(`Bombadil was stopped by ${killedBy}.`))
: resolve(exitCode)
);
});
} finally {
signal.removeEventListener('abort', onAbort);
removeGuard();
explorer = null;
await new Promise((resolve) => log.end(resolve));
}
} finally {
rmSync(specDir, { recursive: true, force: true });
}
}
const EXIT_SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP'] as const;
/**
* Kill Bombadil's process group and remove the spec copy (which holds the
* admin's password) if this worker goes away first: cancelling a run with
* Ctrl-C ends it without aborting the case's signal, and the group, detached
* so it can be stopped as one, would otherwise outlive it. Everything here is
* synchronous, as an `exit` handler must be. Returns the undo.
*/
function guardAgainstExit(child: ChildProcess, specDir: string): () => void {
const kill = () => {
killExplorer(child);
rmSync(specDir, { recursive: true, force: true });
};
const onSignal = (received: NodeJS.Signals) => {
kill();
process.kill(process.pid, received);
};
process.once('exit', kill);
process.once('disconnect', kill);
for (const name of EXIT_SIGNALS) process.once(name, onSignal);
return () => {
process.off('exit', kill);
process.off('disconnect', kill);
for (const name of EXIT_SIGNALS) process.off(name, onSignal);
};
}
/**
* Stop Bombadil the way Ctrl-C does (SIGINT, its only handled signal): it
* finishes, writes its trace and closes Chromium. Killed if it has not exited
* in time.
*/
async function stop(child: ChildProcess): Promise<void> {
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) return;
const exited = new Promise((resolve) => child.once('close', resolve));
process.kill(child.pid, 'SIGINT');
const timer = new Promise((resolve) => setTimeout(resolve, KILL_GRACE_MS, 'timeout'));
if ((await Promise.race([exited, timer])) === 'timeout') {
console.log(`[explore] Bombadil did not stop within ${KILL_GRACE_MS / 1000} s; killing it.`);
killExplorer(child);
await exited;
}
}
/**
* SIGKILL Bombadil's process group and each Chromium's. Bombadil starts
* Chromium in a group of its own and closes it only on a clean exit, so a
* killed Bombadil would leave it running: its groups are found first, from
* Bombadil's children. Synchronous, for the exit guard.
*/
function killExplorer(child: ChildProcess): void {
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) return;
for (const leader of [...childPids(child.pid), child.pid]) {
try {
process.kill(-leader, 'SIGKILL');
} catch (error) {
// A child that leads no group of its own went with Bombadil's.
if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
}
}
}
function childPids(pid: number): number[] {
try {
return execFileSync('pgrep', ['-P', String(pid)], { encoding: 'utf8' })
.split('\n')
.filter((line) => line.trim() !== '')
.map(Number);
} catch (error) {
// pgrep exits 1 when nothing matches.
if ((error as { status?: number }).status === 1) return [];
throw error;
}
}
/** The binary the npm package ships for this platform, run without its Node launcher. */
function bombadilBinary(): string {
const key = `${process.platform}-${process.arch}`;
const binary = BOMBADIL_BINARIES[key];
if (binary === undefined) throw new Error(`Bombadil ships no binary for ${key}.`);
// The package's exports name only its types, so look for it where Node would.
const searched = createRequire(import.meta.url).resolve.paths('@antithesishq/bombadil') ?? [];
for (const dir of searched) {
const candidate = path.join(dir, '@antithesishq', 'bombadil', 'binaries', binary);
if (existsSync(candidate)) return candidate;
}
throw new Error(`No ${binary} in @antithesishq/bombadil; is it installed (pnpm install)?`);
}