Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import { execFile } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdir, open, readFile, rm, stat, unlink } from 'node:fs/promises';
import { homedir } from 'node:os';
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { promisify } from 'node:util';
import {
bundleIdentity,
clearTakenOver,
commandLineNames,
ensureSharedProcess,
hostedCredentialSchema,
inProcessSupervision,
killProcessTree,
materializeHostedProvider,
type OpenAgentStream,
type ProviderReadiness,
processCommandLine,
providerReadinessSchema,
readTakenOver,
readWatchFlags,
recordWatcherHealth,
replaceFile,
requestStop,
runAgentHost,
SessionLinks,
sharedConfigSchema,
type SharedHostConfig,
type Supervision,
type TakenOver,
WATCH_FLAGS_FILE,
WATCHER_HEALTH_FILE,
type WatchFlags,
WatcherControl,
type WatcherHealthFile,
watcherHealthFileSchema,
watchFlagsSchema,
} from '@switch-console/agent-providers';
import { ReasonedError } from './errors';
import { errorMessage, type Logger } from './log';
import { agentWorkspace, type DataLayout } from './paths';
import type { Isolation, Provider } from './schemas';
import type { GivenLogin } from './sealed-logins';
const execute = promisify(execFile);
/** What is on disk and alive for one agent's agent host. */
export type AgentObservation = {
/** The agent host or its supervisor is running. */
alive: boolean;
/** The provider and working directory its saved configuration was created with. */
configured: { provider: string; cwd: string } | null;
flags: WatchFlags | null;
/** `current` is false when the process that wrote it is gone. */
health: (WatcherHealthFile & { current: boolean }) | null;
/** `supervisor/failure.json`: why the agent host stopped and was not restarted. */
failure: string | null;
takenOver: TakenOver | null;
/**
* The agent host outlives this controller (it runs in a process of its own)
* and runs another shared-host bundle than the one this controller launches
* agent hosts from, or one too old to say which. Read from a live agent
* host's current health only; false otherwise.
*/
outdated: boolean;
};
/** What an agent with no agent host root at all looks like. */
export function emptyObservation(): AgentObservation {
return {
alive: false,
configured: null,
flags: null,
health: null,
failure: null,
takenOver: null,
outdated: false,
};
}
export type LaunchOptions = {
/** Where the agent host runs: in this controller's process, or in one of its own. */
isolation: Isolation;
/**
* Stop a running agent host first. Without it, a running agent host is
* handed the new configuration and brings its sessions in step itself.
*/
restart: boolean;
/** Restart into a different provider or working directory: the saved configuration goes. */
replaceIdentity: boolean;
/** Someone asked for this agent host on purpose: a standing-down marker is cleared. */
clearTakenOver: boolean;
};
/**
* What an agent host reads to reach Switch: the controller's relay, a token
* for it, and the hub an agent host in a process of its own hears its events on.
*/
export type RelayCredentials = {
endpoint: string;
token: string;
hub: string;
/** A login Switch gave the machine for the agent's provider, which it has none of its own for. */
providerLogin: GivenLogin | null;
};
/** Runs agent hosts one way: in this controller's process, or each in a process of its own. */
export interface AgentRunner {
observe(agentId: string): Promise<AgentObservation>;
launch(agentId: string, template: SharedHostConfig, options: LaunchOptions): Promise<void>;
/** Turns the agent host off; with `wait`, returns once it and its sessions are gone. */
stop(agentId: string, options: { wait: boolean }): Promise<void>;
/** The controller is exiting. */
close(): Promise<void>;
}
/**
* How the controller runs agents on this machine (`AgentRuntimes`, over an
* `InProcessRuntime` and a runner for isolated agents); tests substitute a
* fake.
*/
export interface AgentRuntime extends AgentRunner, WarmsRooms {
credentialsPath(agentId: string): string;
/** The credentials file as written, or null when there is none or it cannot be read. */
readCredentials(agentId: string): Promise<RelayCredentials | null>;
writeCredentials(agentId: string, credentials: RelayCredentials): Promise<void>;
deleteCredentials(agentId: string): Promise<void>;
/**
* `directory` from the definition, or a workspace of the agent's own when it
* names none. A directory inside the workspaces folder is made when
* missing; any other must already exist.
*/
workingDirectory(agentId: string, name: string, directory: string | null): Promise<string>;
/**
* Whether the provider signs in as an agent would run it: with the
* machine's own login, or with `login`, one Switch gave the machine.
*/
probe(
provider: Provider,
binaryPath: string,
cwd: string,
login: GivenLogin | null
): Promise<ProviderReadiness>;
/** The agent host's state root as the agent host sees it: where a given login's files go. */
agentStateRoot(agentId: string): string;
}
/** An agent host this controller runs, asked to start a room's session early. */
export interface WarmsRooms {
/** Start the session a message to `roomId` would start; throws when the agent host is not running here. */
warm(agentId: string, roomId: string): Promise<void>;
}
/**
* The environment to probe `login` with: its files written under `dir`, made
* afresh, and the variables pointing at them or holding its token.
*/
export async function loginProbeEnvironment(
dir: string,
login: GivenLogin,
binaryPath: string
): Promise<Record<string, string>> {
await rm(dir, { recursive: true, force: true });
await mkdir(dir, { recursive: true, mode: 0o700 });
const env: Record<string, string> = {};
await materializeHostedProvider(dir, env, login, binaryPath);
return env;
}
/** How long an agent host asked to stop is given; each session host is allowed 20 s of it. */
export const STOP_TIMEOUT_MS = 30_000;
const PROBE_TIMEOUT_MS = 90_000;
/**
* Asks a provider's CLI, through the shared-host bundle's `--probe`, whether
* it is signed in, with the environment the agents would have.
*/
export async function probeProvider(
bundlePath: string,
provider: Provider,
binaryPath: string,
cwd: string,
env: NodeJS.ProcessEnv
): Promise<ProviderReadiness> {
const { stdout } = await execute(
process.execPath,
[bundlePath, '--probe', provider, cwd, binaryPath],
{ timeout: PROBE_TIMEOUT_MS, maxBuffer: 4 * 1024 * 1024, env, windowsHide: true }
);
const line = stdout.trim().split('\n').at(-1) ?? '';
return providerReadinessSchema.parse(JSON.parse(line));
}
/** What an agent host's credentials file holds; see `readSharedCredentials`. */
export function relayCredentialsBody(agentId: string, credentials: RelayCredentials): string {
return JSON.stringify({
env: {
SWITCH_API_ENDPOINT: credentials.endpoint,
SWITCH_API_TOKEN: credentials.token,
SWITCH_AGENT_ID: agentId,
SWITCH_AGENT_HUB: credentials.hub,
},
providerLogin: credentials.providerLogin,
});
}
/** A credentials file's relay, token and hub; null when it is not one, or is another agent's. */
export function parseRelayCredentials(text: string, agentId: string): RelayCredentials | null {
try {
const parsed = JSON.parse(text) as { env?: Record<string, unknown>; providerLogin?: unknown };
const env = parsed.env ?? {};
const endpoint = env.SWITCH_API_ENDPOINT;
const token = env.SWITCH_API_TOKEN;
// Written before the hub: an empty one, which no relay names, so the file is written again.
const hub = typeof env.SWITCH_AGENT_HUB === 'string' ? env.SWITCH_AGENT_HUB : '';
if (env.SWITCH_AGENT_ID !== agentId) return null;
const login = hostedCredentialSchema.nullable().safeParse(parsed.providerLogin ?? null);
// A login it cannot read is none: the file is written again with the one wanted.
const providerLogin = login.success && login.data?.status === 'connected' ? login.data : null;
return typeof endpoint === 'string' && typeof token === 'string'
? { endpoint, token, hub, providerLogin }
: null;
} catch {
return null;
}
}
/** Failures an agent host is started again after, within `CRASH_WINDOW_MS`. */
const MAX_CRASHES = 3;
const CRASH_WINDOW_MS = 10 * 60 * 1000;
/**
* What an agent host run in the controller's process records as its build: never
* the shared-host bundle a detached agent host records, so one left by an
* earlier controller is replaced rather than taken for this one.
*/
const IN_PROCESS_BUILD = 'switch-agent-controller:in-process';
export async function writeAtomic(path: string, body: string): Promise<void> {
const temporary = `${path}.${randomUUID()}`;
const file = await open(temporary, 'wx', 0o600);
try {
await file.writeFile(body);
await file.sync();
} finally {
await file.close();
}
await replaceFile(temporary, path);
}
export async function readOptional(path: string): Promise<string | null> {
try {
return await readFile(path, 'utf8');
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
throw error;
}
}
export async function removeOptional(path: string): Promise<void> {
await unlink(path).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'ENOENT') throw error;
});
}
/** `path` is strictly below `root`, after resolving `..` segments. */
export function isInside(root: string, path: string): boolean {
const rest = relative(resolve(root), resolve(path));
return rest !== '' && rest !== '..' && !rest.startsWith(`..${sep}`) && !isAbsolute(rest);
}
export function alive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false;
throw error;
}
}
/** A saved PID counts only while it still names a process started for this root. */
export async function ownsRoot(pid: number, root: string): Promise<boolean> {
if (!alive(pid)) return false;
const commandLine = await processCommandLine(pid);
return commandLine !== null && commandLineNames(commandLine, root);
}
export async function recordedPid(path: string): Promise<number | null> {
const text = await readOptional(path);
if (text === null) return null;
const pid = (JSON.parse(text) as { pid?: unknown }).pid;
return typeof pid === 'number' && Number.isSafeInteger(pid) && pid > 0 ? pid : null;
}
/**
* The shared-host bundle a runner launches agent hosts from, by
* `bundleIdentity`: read again only when the file at its path changes, as an
* update installed over it or a moved `current` link does.
*/
export class LaunchedBundle {
private known: { key: string; identity: string } | null = null;
constructor(private readonly path: string) {}
async identity(): Promise<string> {
const info = await stat(this.path);
const key = `${info.dev}:${info.ino}:${info.size}:${info.mtimeMs}`;
if (this.known?.key !== key) this.known = { key, identity: await bundleIdentity(this.path) };
return this.known.identity;
}
/**
* Whether the live agent host whose health this is runs another bundle, or
* one too old to say which. Health that is not current says nothing.
*/
async outdates(health: AgentObservation['health']): Promise<boolean> {
if (!health?.current) return false;
if (health.bundle === undefined) return true;
return health.bundle !== (await this.identity());
}
}
/** The owner records an agent host writes under its root: the worker's, then a detached one's supervisor's. */
export const OWNER_RECORDS = ['shared-owner.lock', join('supervisor', 'owner.json')];
/** What is on disk for one agent's agent host, read by a process that does not run it (`status`). */
export async function observeOnDisk(
layout: DataLayout,
agentId: string
): Promise<AgentObservation> {
const root = layout.watcherRoot(agentId);
const pid = await recordedPid(join(root, 'shared-owner.lock'));
const running = pid !== null && alive(pid);
const healthText = await readOptional(join(root, WATCHER_HEALTH_FILE));
const health =
healthText === null
? null
: { ...watcherHealthFileSchema.parse(JSON.parse(healthText)), current: running };
return { ...(await readRoot(root)), alive: running, health, outdated: false };
}
/** Everything about an agent host root but whether it runs, and on which bundle. */
export async function readRoot(
root: string
): Promise<Omit<AgentObservation, 'alive' | 'outdated'>> {
const config = await readOptional(join(root, 'config.json'));
let configured: AgentObservation['configured'] = null;
if (config !== null) {
const parsed = JSON.parse(config) as SharedHostConfig;
configured = { provider: parsed.start.provider, cwd: parsed.start.input.cwd };
}
let flags: WatchFlags | null = null;
try {
flags = await readWatchFlags(root);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
const failureText = await readOptional(join(root, 'supervisor', 'failure.json'));
const failure =
failureText === null
? null
: String((JSON.parse(failureText) as { message?: unknown }).message ?? failureText);
return { configured, flags, health: null, failure, takenOver: await readTakenOver(root) };
}
type RunningAgentHost = {
stop: AbortController;
done: Promise<void>;
control: WatcherControl;
sessions: Supervision & { close: () => Promise<void> };
};
/**
* Runs each agent's agent host inside the controller's process, as Switch
* Console runs its own agents' agent hosts: the shared host's `runAgentHost`,
* fed its events by `openStream` rather than by a connection of its own, in a
* state root of its own driven through the files it reads (`watch.json`,
* `config.json`). Its sessions are the controller's child processes, so
* nothing an agent runs outlives the controller.
*
* An agent host that fails is started again after a short wait, at most
* `MAX_CRASHES` times in `CRASH_WINDOW_MS`; past that its failure is recorded
* and it stays down until a new revision or an explicit restart.
*/
export class InProcessRuntime implements AgentRuntime {
private readonly hosts = new Map<string, RunningAgentHost>();
private readonly crashes = new Map<string, number[]>();
private readonly links = new SessionLinks();
private readonly lifetime = new AbortController();
constructor(
private readonly deps: {
layout: DataLayout;
/** Where agents with no directory of their own work. */
workspaces: string;
/** The `shared-host-daemon.mjs` bundle from agent-providers: what each session runs. */
bundlePath: string;
/** The stream the agent's agent host hears its events on. */
openStream: (agentId: string) => OpenAgentStream;
log: Logger;
/** How long a failed agent host waits before it is started again. */
crashBackoffMs: number;
}
) {}
credentialsPath(agentId: string): string {
return this.deps.layout.agentCredentials(agentId);
}
async readCredentials(agentId: string): Promise<RelayCredentials | null> {
const text = await readOptional(this.credentialsPath(agentId));
return text === null ? null : parseRelayCredentials(text, agentId);
}
async writeCredentials(agentId: string, credentials: RelayCredentials): Promise<void> {
await mkdir(this.deps.layout.agentDir(agentId), { recursive: true, mode: 0o700 });
await writeAtomic(this.credentialsPath(agentId), relayCredentialsBody(agentId, credentials));
}
async deleteCredentials(agentId: string): Promise<void> {
await removeOptional(this.credentialsPath(agentId));
}
async workingDirectory(
_agentId: string,
name: string,
directory: string | null
): Promise<string> {
if (directory === null) {
const path = agentWorkspace(this.deps.workspaces, name);
await mkdir(path, { recursive: true });
return path;
}
const expanded =
directory === '~' ||
directory.startsWith('~/') ||
(process.platform === 'win32' && directory.startsWith('~\\'))
? join(homedir(), directory.slice(1))
: directory;
if (!isAbsolute(expanded))
throw new ReasonedError(
'definition_invalid',
`The working directory '${directory}' is not an absolute path.`
);
if (isInside(this.deps.workspaces, expanded)) {
const path = resolve(expanded);
await mkdir(path, { recursive: true });
return path;
}
let isDirectory: boolean;
try {
isDirectory = (await stat(expanded)).isDirectory();
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
throw new ReasonedError(
'definition_invalid',
`The working directory ${expanded} does not exist on this machine.`
);
}
if (!isDirectory)
throw new ReasonedError('definition_invalid', `${expanded} is not a directory.`);
return expanded;
}
async observe(agentId: string): Promise<AgentObservation> {
const root = this.deps.layout.watcherRoot(agentId);
const running = this.hosts.get(agentId);
const observation = await readRoot(root);
return {
...observation,
alive: running !== undefined,
health: running
? {
...running.control.health(),
pid: process.pid,
updatedAt: new Date().toISOString(),
current: true,
}
: null,
outdated: false,
};
}
async launch(agentId: string, template: SharedHostConfig, options: LaunchOptions): Promise<void> {
const root = this.deps.layout.watcherRoot(agentId);
await mkdir(root, { recursive: true, mode: 0o700 });
if (options.restart || options.replaceIdentity) await this.stop(agentId, { wait: true });
await this.stopDetached(root);
if (options.replaceIdentity) await removeOptional(join(root, 'config.json'));
if (options.clearTakenOver) await clearTakenOver(root);
await this.writeFlags(root, { enabled: true, spawn: true });
this.crashes.delete(agentId);
await this.ensure(agentId, root, template);
}
async stop(agentId: string, options: { wait: boolean }): Promise<void> {
const root = this.deps.layout.watcherRoot(agentId);
await mkdir(root, { recursive: true, mode: 0o700 });
await this.writeFlags(root, { enabled: false, spawn: false });
const running = this.hosts.get(agentId);
if (!running) return;
running.stop.abort();
if (!options.wait) return;
const stopped = await Promise.race([
running.done.then(() => true),
delay(STOP_TIMEOUT_MS).then(() => false),
]);
if (!stopped)
throw new Error(
`The agent host for agent ${agentId} has not stopped ${STOP_TIMEOUT_MS / 1000} s after being turned off.`
);
}
/** Stops every agent host and every session: the controller is exiting. */
async close(): Promise<void> {
this.lifetime.abort();
await Promise.allSettled([...this.hosts.values()].map((running) => running.done));
}
/** Writes the agent host's configuration from `template` and starts it, unless it runs. */
private async ensure(agentId: string, root: string, template: SharedHostConfig): Promise<void> {
await ensureSharedProcess({
root,
config: template,
resuming: false,
watcher: true,
restart: false,
startSource: null,
supervision: {
links: null,
build: IN_PROCESS_BUILD,
start: async ({ root: prepared, configPath }) => {
// What was written, not what was asked for: an earlier run's
// configuration keeps its session identity.
const written = sharedConfigSchema.parse(JSON.parse(await readFile(configPath, 'utf8')));
this.start(agentId, prepared, written);
},
stop: async () => {
await this.stop(agentId, { wait: true });
},
},
});
}
private start(agentId: string, root: string, config: SharedHostConfig): void {
if (this.hosts.has(agentId) || this.lifetime.signal.aborted) return;
const stop = new AbortController();
const signal = AbortSignal.any([stop.signal, this.lifetime.signal]);
const control = new WatcherControl();
const sessions = inProcessSupervision(this.deps.bundlePath, this.links);
// For `status`, which runs in another process and reads only disk.
const stopRecording = recordWatcherHealth(root, control, null);
const forget = () => {
if (this.hosts.get(agentId)?.done === done) this.hosts.delete(agentId);
};
const done = (async () => {
try {
await runAgentHost(root, config, signal, sessions, control, this.deps.openStream(agentId));
} finally {
stopRecording();
await sessions.close();
}
})()
.then(
() => {
this.deps.log.info('Agent host stopped', { agentId });
return false;
},
(error: unknown) => this.crashed(agentId, root, signal, error, forget)
)
.then((again) => {
forget();
if (again) this.start(agentId, root, config);
});
this.hosts.set(agentId, { stop, done, control, sessions });
this.deps.log.info('Agent host started', { agentId });
}
/**
* Whether an agent host that failed is started again. One that is not is
* forgotten before its failure is recorded, so nothing that reads the
* failure can still see it alive.
*/
private async crashed(
agentId: string,
root: string,
signal: AbortSignal,
error: unknown,
forget: () => void
): Promise<boolean> {
const message = errorMessage(error);
if (signal.aborted) {
this.deps.log.info('Agent host stopped', { agentId, error: message });
return false;
}
const now = Date.now();
const recent = (this.crashes.get(agentId) ?? []).filter((at) => now - at < CRASH_WINDOW_MS);
recent.push(now);
this.crashes.set(agentId, recent);
if (recent.length > MAX_CRASHES) {
this.deps.log.error('Agent host failed too often; it stays down until restarted', {
agentId,
error: message,
});
forget();
await mkdir(join(root, 'supervisor'), { recursive: true, mode: 0o700 });
await writeAtomic(join(root, 'supervisor', 'failure.json'), JSON.stringify({ message }));
return false;
}
this.deps.log.warn('Agent host failed; starting it again', {
agentId,
error: message,
attempt: recent.length,
});
await delay(this.deps.crashBackoffMs * 2 ** (recent.length - 1), undefined, {
signal: this.lifetime.signal,
}).catch(() => {});
if (this.lifetime.signal.aborted) return false;
const flags = await readWatchFlags(root).catch(() => null);
return flags?.enabled === true;
}
/**
* Stops an agent host an earlier version of the controller left running as a
* detached process of its own, so this one can take its root.
*/
private async stopDetached(root: string): Promise<void> {
for (const record of OWNER_RECORDS) {
const path = join(root, record);
const pid = await recordedPid(path);
if (pid === null || pid === process.pid || !(await ownsRoot(pid, root))) continue;
this.deps.log.warn('Stopping an agent host left running by an earlier controller', {
root,
pid,
});
await requestStop(root, pid);
const deadline = Date.now() + STOP_TIMEOUT_MS;
while (alive(pid)) {
if (Date.now() > deadline) {
await killProcessTree(pid);
break;
}
await delay(200);
}
}
}
async probe(
provider: Provider,
binaryPath: string,
cwd: string,
login: GivenLogin | null
): Promise<ProviderReadiness> {
const given = login
? await loginProbeEnvironment(
join(this.deps.layout.root, 'login-check', provider),
login,
binaryPath
)
: {};
return probeProvider(this.deps.bundlePath, provider, binaryPath, cwd, {
...process.env,
...given,
});
}
agentStateRoot(agentId: string): string {
return this.deps.layout.watcherRoot(agentId);
}
/** False when the agent host does not run in this process. */
async warmHere(agentId: string, roomId: string): Promise<boolean> {
const host = this.hosts.get(agentId);
if (!host) return false;
await host.control.warm(roomId);
return true;
}
async warm(agentId: string, roomId: string): Promise<void> {
if (!(await this.warmHere(agentId, roomId)))
throw new Error(`The agent host of agent ${agentId} is not running in this controller.`);
}
private async writeFlags(root: string, flags: WatchFlags): Promise<void> {
await writeAtomic(join(root, WATCH_FLAGS_FILE), JSON.stringify(watchFlagsSchema.parse(flags)));
}
}