Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
/**
* Configuration for the harness. Every value is a secret or a deployment
* address, so nothing is hard-coded: it comes from the process environment, or
* from a gitignored `.env` beside this package. Deployed environments' URLs are
* deliberately absent from source.
*/
/**
* Which runner mode a run is in.
*
* `fast` needs nothing but a Switch server (and Mattermost, for the scenarios
* that cross the bridge): no provider, no Console, no LLM key. It is the lane
* that can gate pull requests, because it carries no secrets a fork lacks.
* `provider` needs a real agent session attending the room, driven by Switch
* Console or its sidecar. `browser` drives the gateway UI in a headless
* Chromium against the same server: a smoke test of what an operator sees,
* which no HTTP-level scenario can see break. `agentic` runs the same real
* session as `provider`, but a model plays the human from a scenario file and
* another model judges the conversation. `explore` drives the same gateway UI
* as `browser`, but at random for a fixed time (Bombadil), checking properties
* every page must keep rather than following a script.
*/
export type Lane = 'fast' | 'provider' | 'browser' | 'agentic' | 'explore';
/** Every lane: the one list anything that takes a lane by name checks against. */
export const LANES: readonly Lane[] = ['fast', 'provider', 'browser', 'agentic', 'explore'];
export function isLane(raw: string): raw is Lane {
return (LANES as readonly string[]).includes(raw);
}
/**
* What the target is. Scenarios declare which they may run against.
*
* `ephemeral` is a stack created for this run and destroyed after it, so a
* scenario may change server-global state there (feature flags, for one).
* `local` is a developer's own long-lived stack. `prod` is read-only: the
* clients refuse every authenticated write against it.
*/
export type Environment = 'ephemeral' | 'local' | 'dev' | 'staging' | 'prod';
/** The Switch known-agent type the provider lane registers its throwaway agent as. */
export type HarnessAgentType = 'opencode' | 'claude-code' | 'codex' | 'antigravity' | 'cursor';
/**
* How the `question` scenario expects the agent to ask.
*
* `numbered` is the real thing: the provider offers a native ask-the-user tool,
* the console relays it into the room as a numbered list, and the answer goes
* back by number. It is the default for every agent type. `prose` is the
* explicit fallback for a provider whose SDK sessions are offered no such tool:
* the agent asks in an ordinary message and is answered with the word. The
* round trip being tested (the session is still alive and still connected when
* the second message arrives) is the same; the relay path is not exercised.
*/
export type QuestionMode = 'numbered' | 'prose';
export interface SwitchEnv {
apiUrl: string;
gatewayAdminEmail: string;
gatewayAdminPassword: string;
agentRegistrationToken: string;
}
export interface MattermostEnv {
url: string;
/** Token of the **human** the harness posts as. */
token: string;
/** Team slug (e.g. `switch`), not its display name. */
team: string;
}
/** Settings only the provider lane reads. */
export interface ProviderEnv {
/** Working directory the console session under test runs from. */
agentRepoDir: string | null;
/**
* Where a setup records the agent and room it created, so a later process can
* reuse them instead of registering its own. Needed because a Switch Console
* session can only be pointed at an agent that already exists: seed first,
* start the console, then run the scenarios against the same agent.
*/
manifestPath: string | null;
/** Which agent type to register, and so which provider's runtime is exercised. */
agentType: HarnessAgentType;
/** How `question` expects the clarifying question to arrive. */
questionMode: QuestionMode;
/**
* A headless agent controller to run the agent on. When set, setup places
* the agent on it through the management API and writes no Switch Console
* files; when not, a Console session for the agent is expected instead.
*/
controllerId: string | null;
}
/**
* Settings only the agentic lane reads. Each is null when unset, and the lane's
* scenarios then need it (`agentic-models`): a model is never picked for a run
* that did not name one, and a run never spends without a limit.
*/
export interface AgenticEnv {
/** Exact model id that plays the human. */
driverModel: string | null;
/** Exact model id that judges the conversation. */
judgeModel: string | null;
/** What the lane may spend, driver, judge and agent together, in USD. */
budgetUsd: number | null;
/**
* A file of provider keys (`ANTHROPIC_API_KEY=…`), the one the agent
* controller is handed too. Without it, the driver and judge use the
* Claude Code CLI's own sign-in.
*/
providerEnvFile: string | null;
}
export interface HarnessEnv {
lane: Lane;
environment: Environment;
/**
* In strict mode a scenario whose prerequisites are missing fails instead of
* skipping. CI runs strict, so a misconfigured job goes red rather than
* reporting a suite that ran nothing as green.
*/
strict: boolean;
/** Leave created agents, rooms and channels behind after the run (for debugging). */
keepArtifacts: boolean;
/** Where to record the credentials the server issues during the run; see `IssuedSecrets`. */
secretsFile: string | null;
/** The server's JSON log, when the harness can read it; see `ServerLog`. */
serverLog: string | null;
/**
* The gateway UI, served against `switch.apiUrl` (its `/gateway` requests
* reach that server). Only the browser and explore lanes read it.
*/
gatewayUiUrl: string | null;
/**
* Where the browser lane saves a screenshot of a page that failed, and the
* explore lane its trace, if anywhere.
*/
outputDir: string | null;
switch: SwitchEnv;
/** `null` when no Mattermost is configured; scenarios that need one then skip or fail. */
mattermost: MattermostEnv | null;
/**
* The environment deploys no Mattermost, by design (`SWITCH_E2E_NO_MATTERMOST=1`).
* Scenarios that need one skip even in strict mode, and say why.
*/
noMattermost: boolean;
provider: ProviderEnv;
agentic: AgenticEnv;
}
const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
/**
* Parse `KEY=VALUE` lines. Blank lines and `#` comments are skipped; a value may
* be wrapped in single or double quotes. Deliberately tiny — this is the only
* reason the package would otherwise need a dotenv dependency.
*/
export function parseDotEnv(raw: string): Record<string, string> {
const out: Record<string, string> = {};
for (const line of raw.split('\n')) {
const trimmed = line.trim();
if (trimmed === '' || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq <= 0) continue;
const key = trimmed.slice(0, eq).trim();
let value = trimmed.slice(eq + 1).trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
out[key] = value;
}
return out;
}
/** Values from `.env` beside this package, or `{}` when there is no such file. */
export function readDotEnvFile(file = path.join(PACKAGE_ROOT, '.env')): Record<string, string> {
let raw: string;
try {
raw = readFileSync(file, 'utf8');
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return {};
throw error;
}
return parseDotEnv(raw);
}
export class MissingConfigError extends Error {
readonly missing: string[];
constructor(missing: string[]) {
super(
`Missing required configuration: ${missing.join(', ')}. ` +
`Set them in the environment or in console/tooling-e2e/.env (see README.md).`
);
this.name = 'MissingConfigError';
this.missing = missing;
}
}
type Source = Record<string, string | undefined>;
function reader(source: Source, file: Record<string, string>): (key: string) => string | undefined {
return (key) => {
const value = source[key] ?? file[key];
return value !== undefined && value.trim() !== '' ? value.trim() : undefined;
};
}
/**
* The lane this process was asked to run, or `null` when none was — in which
* case every end-to-end suite skips, which is what keeps `pnpm -r test` on a
* developer machine from reaching for a server.
*/
export function laneFrom(
source: Source = process.env,
file: Record<string, string> = readDotEnvFile()
): Lane | null {
const raw = reader(source, file)('SWITCH_E2E_LANE');
if (raw === undefined) return null;
if (isLane(raw)) return raw;
throw new Error(`SWITCH_E2E_LANE must be one of ${LANES.join(', ')} — got '${raw}'.`);
}
/**
* Whether a run that cannot go ahead fails rather than skips. Read from the
* same sources as everything else, so `.env` and the environment never
* disagree about it.
*/
export function strictFrom(
source: Source = process.env,
file: Record<string, string> = readDotEnvFile()
): boolean {
return reader(source, file)('SWITCH_E2E_STRICT') === '1';
}
/**
* How long the explore lane explores, as Bombadil's `--time-limit` takes it
* (`90s`, `5m`, `1h`): `SWITCH_E2E_EXPLORE_TIME_LIMIT`, default `5m`. Not part of
* `loadEnv`, so a malformed value fails only the explore lane, which reads it
* when it collects its cases, to size the exploring case's timeout.
*/
export function exploreTimeLimitFrom(
source: Source = process.env,
file: Record<string, string> = readDotEnvFile()
): string {
const raw = reader(source, file)('SWITCH_E2E_EXPLORE_TIME_LIMIT') ?? '5m';
durationMs(raw);
return raw;
}
const DURATION_UNITS_MS = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 } as const;
/** A duration as Bombadil's `--time-limit` takes it (`90s`, `5m`, `1h`, `1d`), in milliseconds. */
export function durationMs(raw: string): number {
const match = /^([1-9][0-9]*)([smhd])$/.exec(raw);
if (match === null) {
throw new Error(
`SWITCH_E2E_EXPLORE_TIME_LIMIT must be a whole number with a unit, s, m, h or d (e.g. 5m) — got '${raw}'.`
);
}
return Number(match[1]) * DURATION_UNITS_MS[match[2] as keyof typeof DURATION_UNITS_MS];
}
/**
* Resolve the harness configuration from `process.env` layered over `.env`.
* Throws {@link MissingConfigError} naming every absent variable at once rather
* than failing on the first — a first run is usually missing several.
*
* Mattermost is all-or-nothing: none of its three variables means "no
* Mattermost here", some of them is a mistake and is reported as one.
*/
export function loadEnv(
lane: Lane,
source: Source = process.env,
file: Record<string, string> = readDotEnvFile()
): HarnessEnv {
const get = reader(source, file);
const required = {
SWITCH_API_URL: get('SWITCH_API_URL'),
SWITCH_GATEWAY_ADMIN_EMAIL: get('SWITCH_GATEWAY_ADMIN_EMAIL'),
SWITCH_GATEWAY_ADMIN_PASSWORD: get('SWITCH_GATEWAY_ADMIN_PASSWORD'),
SWITCH_AGENT_REGISTRATION_TOKEN: get('SWITCH_AGENT_REGISTRATION_TOKEN'),
SWITCH_E2E_ENVIRONMENT: get('SWITCH_E2E_ENVIRONMENT'),
};
const mattermost = {
MATTERMOST_URL: get('MATTERMOST_URL'),
MATTERMOST_TOKEN: get('MATTERMOST_TOKEN'),
MATTERMOST_TEAM: get('MATTERMOST_TEAM'),
};
const mattermostGiven = Object.values(mattermost).some((value) => value !== undefined);
const missing = [
...Object.entries(required),
...(mattermostGiven ? Object.entries(mattermost) : []),
]
.filter(([, value]) => value === undefined)
.map(([key]) => key);
if (missing.length > 0) throw new MissingConfigError(missing);
const noMattermost = get('SWITCH_E2E_NO_MATTERMOST') === '1';
if (noMattermost && mattermostGiven) {
throw new Error(
'SWITCH_E2E_NO_MATTERMOST=1 says this environment has no Mattermost, but MATTERMOST_* is set too. Unset one.'
);
}
return {
lane,
environment: environmentFrom(required.SWITCH_E2E_ENVIRONMENT as string),
strict: strictFrom(source, file),
keepArtifacts: get('SWITCH_E2E_KEEP') === '1',
secretsFile: get('SWITCH_E2E_SECRETS_FILE') ?? null,
serverLog: get('SWITCH_E2E_SERVER_LOG') ?? null,
gatewayUiUrl: optionalUrl(get('SWITCH_GATEWAY_UI_URL')),
outputDir: get('SWITCH_E2E_OUTPUT_DIR') ?? null,
switch: {
apiUrl: stripTrailingSlash(required.SWITCH_API_URL as string),
gatewayAdminEmail: required.SWITCH_GATEWAY_ADMIN_EMAIL as string,
gatewayAdminPassword: required.SWITCH_GATEWAY_ADMIN_PASSWORD as string,
agentRegistrationToken: required.SWITCH_AGENT_REGISTRATION_TOKEN as string,
},
mattermost: mattermostGiven
? {
url: stripTrailingSlash(mattermost.MATTERMOST_URL as string),
token: mattermost.MATTERMOST_TOKEN as string,
team: mattermost.MATTERMOST_TEAM as string,
}
: null,
noMattermost,
provider: {
agentRepoDir: get('SWITCH_E2E_AGENT_DIR') ?? null,
manifestPath: get('SWITCH_E2E_MANIFEST') ?? null,
agentType: agentTypeFrom(get('SWITCH_E2E_AGENT_TYPE')),
questionMode: questionModeFrom(get('SWITCH_E2E_QUESTION_MODE')),
controllerId: get('SWITCH_E2E_CONTROLLER_ID') ?? null,
},
agentic: {
driverModel: get('SWITCH_E2E_DRIVER_MODEL') ?? null,
judgeModel: get('SWITCH_E2E_JUDGE_MODEL') ?? null,
budgetUsd: budgetFrom(get('SWITCH_E2E_AGENTIC_BUDGET_USD')),
providerEnvFile: get('SWITCH_E2E_PROVIDER_ENV_FILE') ?? null,
},
};
}
const ENVIRONMENTS: readonly Environment[] = ['ephemeral', 'local', 'dev', 'staging', 'prod'];
function environmentFrom(raw: string): Environment {
const found = ENVIRONMENTS.find((environment) => environment === raw);
if (!found) {
throw new Error(
`SWITCH_E2E_ENVIRONMENT must be one of ${ENVIRONMENTS.join(', ')} — got '${raw}'.`
);
}
return found;
}
const AGENT_TYPES: readonly HarnessAgentType[] = [
'opencode',
'claude-code',
'codex',
'antigravity',
'cursor',
];
function agentTypeFrom(raw: string | undefined): HarnessAgentType {
if (raw === undefined) return 'opencode';
const found = AGENT_TYPES.find((type) => type === raw);
if (!found) {
throw new Error(
`SWITCH_E2E_AGENT_TYPE must be one of ${AGENT_TYPES.join(', ')} — got '${raw}'.`
);
}
return found;
}
function questionModeFrom(raw: string | undefined): QuestionMode {
if (raw === undefined || raw === 'numbered') return 'numbered';
if (raw === 'prose') return 'prose';
throw new Error(`SWITCH_E2E_QUESTION_MODE must be 'numbered' or 'prose' — got '${raw}'.`);
}
function budgetFrom(raw: string | undefined): number | null {
if (raw === undefined) return null;
const budget = Number(raw);
if (!Number.isFinite(budget) || budget <= 0) {
throw new Error(
`SWITCH_E2E_AGENTIC_BUDGET_USD must be a positive number of dollars — got '${raw}'.`
);
}
return budget;
}
function optionalUrl(raw: string | undefined): string | null {
return raw === undefined ? null : stripTrailingSlash(raw);
}
export function stripTrailingSlash(url: string): string {
return url.endsWith('/') ? url.slice(0, -1) : url;
}
/**
* A message that names the running config without disclosing any of it. Used in
* skip reasons and the result table, which are printed to a terminal and may be
* pasted into a ticket.
*/
export function describeEnv(env: HarnessEnv): string {
const mattermost = env.mattermost
? `mattermost=${env.mattermost.url} team=${env.mattermost.team}`
: 'mattermost=none';
const ui = env.lane === 'browser' || env.lane === 'explore' ? ` gateway-ui=${env.gatewayUiUrl ?? 'none'}` : '';
return `lane=${env.lane} environment=${env.environment} switch=${env.switch.apiUrl} ${mattermost}${ui}`;
}