Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
import { existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
AccessTokens,
ControllerApiError,
ControllerClient,
type OpenedSocket,
type SocketLike,
} from './api';
import { DEFAULT_TIMING } from './controller';
import { silentLogger } from './log';
import type { AgentCursor, ControllerConnection } from './schemas';
import { SocketCallLost, SocketCalls } from './socket-calls';
import { type ControllerFrame, type ControllerStreamOptions, runControllerStream } from './stream';
import { FakeCore } from './testing/fake-core';
type Message = { event: string; data: unknown };
/**
* A socket that opens, delivers `messages` one per turn of the event loop,
* then closes with `end` (or stays open). What the controller sends is kept
* in `sent`, and `deliver` adds more as the test goes.
*/
class ScriptedSocket extends EventTarget {
readyState = 0;
readonly sent: unknown[] = [];
constructor(messages: (Message | string)[], end: number | 'open') {
super();
setImmediate(() => {
this.readyState = 1;
this.dispatchEvent(new Event('open'));
for (const message of messages) this.deliver(message);
if (end !== 'open') this.end(end);
});
}
deliver(message: Message | string): void {
const data = typeof message === 'string' ? message : JSON.stringify(message);
setImmediate(() => {
if (this.readyState !== 3) this.dispatchEvent(Object.assign(new Event('message'), { data }));
});
}
end(code: number): void {
setImmediate(() => {
if (this.readyState === 3) return;
this.readyState = 3;
this.dispatchEvent(Object.assign(new Event('close'), { code }));
});
}
send(data: string): void {
this.sent.push(JSON.parse(data));
}
close(code = 1000): void {
this.end(code);
}
}
function scriptedSocket(messages: (Message | string)[], end: number | 'open'): ScriptedSocket {
return new ScriptedSocket(messages, end);
}
const state: Message = { event: 'connection_state', data: { connection_id: 'connection-1' } };
function refused(status: number, code: string): Message {
return { event: 'refused', data: { status, detail: { code, message: code } } };
}
async function waitFor(condition: () => boolean, what: string, timeoutMs = 15_000): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (!condition()) {
if (Date.now() > deadline) throw new Error(`Timed out waiting for ${what}.`);
await delay(5);
}
}
const opened: ControllerConnection = {
connection_id: 'connection-1',
generation: 1,
heartbeat_interval_s: 0.02,
agents: ['agent-1'],
};
/** A client that answers from scripts, recording what it was asked. */
class ScriptedClient {
opens: Record<string, AgentCursor>[] = [];
attaches: { connectionId: string; generation: number }[] = [];
opened: ScriptedSocket[] = [];
staleTokens = 0;
open: () => Promise<ControllerConnection> = async () => opened;
sockets: () => ScriptedSocket = () => scriptedSocket([], 'open');
readonly client: ControllerStreamOptions['client'] = {
openConnection: async (cursors) => {
this.opens.push(cursors);
return this.open();
},
openSocket: async (connection): Promise<OpenedSocket> => {
this.attaches.push(connection);
const socket = this.sockets();
this.opened.push(socket);
return {
socket: socket as unknown as SocketLike,
tokenRefused: () => void this.staleTokens++,
};
},
};
}
function run(
scripted: ScriptedClient,
signal: AbortSignal,
overrides: Partial<ControllerStreamOptions> = {}
) {
const frames: ControllerFrame[] = [];
const states: string[] = [];
const ending = runControllerStream({
client: scripted.client,
cursors: () => ({ 'agent-1': 7 }),
confirmed: () => ({ 'agent-1': 7 }),
onOpened: (connection) => void states.push(`opened ${connection.connection_id}`),
onConnected: () => void states.push('connected'),
onDisconnected: () => void states.push('disconnected'),
onFrame: async (frame) => void frames.push(frame),
signal,
log: silentLogger,
idleTimeoutMs: 200,
initialBackoffMs: 5,
maxBackoffMs: 20,
random: () => 0,
...overrides,
});
return { frames, states, ending };
}
describe('runControllerStream', () => {
it('takes calls on a socket that offers them, answers each as it arrives, and fails them when it closes', async () => {
const scripted = new ScriptedClient();
const offering = {
event: 'connection_state',
data: { connection_id: 'connection-1', calls: { max_in_flight: 2 } },
};
scripted.sockets = () => scriptedSocket([offering], 'open');
const calls = new SocketCalls();
const stop = new AbortController();
const { frames, states, ending } = run(scripted, stop.signal, { calls });
await waitFor(() => states.includes('connected'), 'the socket to attach');
const call = {
agentId: 'agent-1',
roomId: 'room-a',
operation: 'post_message',
args: { body: 'hi' },
requestId: null,
};
const answered = calls.call(call)!;
const socket = scripted.opened[0]!;
expect(socket.sent).toContainEqual(expect.objectContaining({ type: 'call', id: 1 }));
socket.deliver({ event: 'call.result', data: { id: 1, status: 200, body: { result: 'ok' } } });
expect(await answered).toEqual({ status: 200, body: { result: 'ok' } });
const unanswered = calls.call(call)!;
socket.end(1011);
await expect(unanswered).rejects.toBeInstanceOf(SocketCallLost);
stop.abort();
expect(await ending).toBe('stopped');
expect(frames.map((frame) => frame.type)).not.toContain('call.result');
});
it('takes no calls on a socket from a server that does not offer them', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () => scriptedSocket([state], 'open');
const calls = new SocketCalls();
const stop = new AbortController();
const { states, ending } = run(scripted, stop.signal, { calls });
await waitFor(() => states.includes('connected'), 'the socket to attach');
expect(
calls.call({ agentId: 'agent-1', roomId: null, operation: 'x', args: {}, requestId: null })
).toBeNull();
stop.abort();
expect(await ending).toBe('stopped');
});
it('opens with the cursors, attaches, and hands over typed frames in order, skipping bad ones', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () =>
scriptedSocket(
[
{ event: 'ping', data: {} },
'not json',
{
event: 'agent.attached',
data: { agent_id: 'agent-1', from_seq: 7, rooms: ['room-a'] },
},
{ event: 'agent.event', data: { agent_id: 'agent-1' } },
{ event: 'agent.unknown', data: { agent_id: 'agent-1' } },
{
event: 'agent.event',
data: {
agent_id: 'agent-1',
seq: 8,
event: {
type: 'message',
room_id: 'room-a',
sequence: 8,
payload: { addressed: true },
future: 1,
},
},
},
{ event: 'assignment.changed', data: { revision: 2 } },
{
event: 'operation.pending',
data: { operation_id: 'op-1', kind: 'agent.restart', agent_id: null },
},
],
'open'
);
const stop = new AbortController();
const { frames, states, ending } = run(scripted, stop.signal);
await waitFor(() => frames.length === 4, 'four frames');
stop.abort();
expect(await ending).toBe('stopped');
expect(scripted.opens).toEqual([{ 'agent-1': 7 }]);
expect(scripted.attaches[0]).toMatchObject({ connectionId: 'connection-1', generation: 1 });
expect(states.slice(0, 2)).toEqual(['opened connection-1', 'connected']);
expect(frames.map((frame) => frame.type)).toEqual([
'agent.attached',
'agent.event',
'assignment.changed',
'operation.pending',
]);
expect(frames[1]!.data).toMatchObject({ seq: 8, event: { future: 1 } });
});
it('answers every ping with a pong naming the confirmed cursors: its beat', async () => {
const scripted = new ScriptedClient();
const ping = { event: 'ping', data: {} };
scripted.sockets = () => scriptedSocket([state, ping, ping], 'open');
const stop = new AbortController();
const { ending } = run(scripted, stop.signal);
await waitFor(() => (scripted.opened[0]?.sent.length ?? 0) >= 2, 'two pongs');
stop.abort();
await ending;
expect(scripted.opened[0]!.sent[0]).toEqual({ type: 'pong', cursors: { 'agent-1': 7 } });
});
it('reattaches to the same connection after the socket closes', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () => scriptedSocket([state], 1000);
const stop = new AbortController();
const { states, ending } = run(scripted, stop.signal);
await waitFor(() => scripted.attaches.length >= 3, 'three attaches');
stop.abort();
await ending;
expect(scripted.opens).toHaveLength(1);
expect(new Set(scripted.attaches.map((a) => a.connectionId))).toEqual(
new Set(['connection-1'])
);
expect(states).toContain('disconnected');
});
it('comes back within a second, without backing off, when the server is restarting', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () => scriptedSocket([state], 1012);
const stop = new AbortController();
const started = Date.now();
const { ending } = run(scripted, stop.signal, { initialBackoffMs: 60_000 });
await waitFor(() => scripted.attaches.length >= 3, 'three attaches', 3_000);
stop.abort();
await ending;
expect(Date.now() - started).toBeGreaterThanOrEqual(490);
});
it('reconnects when nothing arrives within the idle timeout', async () => {
const scripted = new ScriptedClient();
const started = Date.now();
const stop = new AbortController();
const { ending } = run(scripted, stop.signal);
await waitFor(() => scripted.attaches.length >= 2, 'a second attach');
stop.abort();
await ending;
expect(Date.now() - started).toBeGreaterThanOrEqual(190);
});
it('waits three of the server’s beats before calling a quiet socket dead', async () => {
const scripted = new ScriptedClient();
// Switch beats every 0.2 s here, as slowly as the 200 ms idle timeout: three beats it is.
scripted.sockets = () =>
scriptedSocket(
[
{
event: 'connection_state',
data: { connection_id: 'connection-1', heartbeat_interval_s: 0.2 },
},
],
'open'
);
const stop = new AbortController();
const { ending } = run(scripted, stop.signal);
await waitFor(() => scripted.attaches.length === 1, 'an attach');
await delay(450);
expect(scripted.attaches).toHaveLength(1);
await waitFor(() => scripted.attaches.length >= 2, 'a second attach once three beats passed');
stop.abort();
await ending;
});
it('opens a new connection, from the cursors as they stand, when Switch no longer knows it', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () =>
scriptedSocket(
scripted.attaches.length === 2 ? [refused(404, 'unknown_connection')] : [state],
scripted.attaches.length === 2 ? 4404 : 1000
);
let cursor = 7;
const stop = new AbortController();
const { ending } = run(scripted, stop.signal, {
cursors: () => ({ 'agent-1': cursor++ }),
});
await waitFor(() => scripted.opens.length === 2, 'a second open');
stop.abort();
await ending;
expect(scripted.opens).toEqual([{ 'agent-1': 7 }, { 'agent-1': 8 }]);
});
it('opens a new connection when an attach is refused as stale', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () => scriptedSocket([refused(409, 'stale_generation')], 4409);
const stop = new AbortController();
const { ending } = run(scripted, stop.signal);
await waitFor(() => scripted.opens.length >= 2, 'a second open');
stop.abort();
await ending;
});
it('opens a new connection when Switch evicts it for any reason but a takeover', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () =>
scriptedSocket(
[state, { event: 'evicted', data: { code: 'no_stream', reason: 'beat refused' } }],
1000
);
const stop = new AbortController();
const { ending } = run(scripted, stop.signal);
await waitFor(() => scripted.opens.length >= 2, 'a second open');
stop.abort();
expect(await ending).toBe('stopped');
});
it('stops for good when another instance took the connection over', async () => {
const evicted = new ScriptedClient();
evicted.sockets = () =>
scriptedSocket(
[state, { event: 'evicted', data: { code: 'taken_over', reason: 'x' } }],
1000
);
expect(await run(evicted, new AbortController().signal).ending).toBe('taken_over');
const refusedAttach = new ScriptedClient();
refusedAttach.sockets = () => scriptedSocket([refused(409, 'taken_over')], 4409);
expect(await run(refusedAttach, new AbortController().signal).ending).toBe('taken_over');
});
it('ends as revoked on the frame, on a refused open, and on a refused attach', async () => {
const framed = new ScriptedClient();
framed.sockets = () => scriptedSocket([{ event: 'credential.revoked', data: {} }], 'open');
const first = run(framed, new AbortController().signal);
expect(await first.ending).toBe('revoked');
expect(first.frames.map((frame) => frame.type)).toEqual(['credential.revoked']);
const refusedOpen = new ScriptedClient();
refusedOpen.open = async () => {
throw new ControllerApiError(401, 'controller_revoked', 'revoked', false, null);
};
expect(await run(refusedOpen, new AbortController().signal).ending).toBe('revoked');
const refusedAttach = new ScriptedClient();
refusedAttach.sockets = () => scriptedSocket([refused(401, 'controller_revoked')], 4401);
const third = run(refusedAttach, new AbortController().signal);
expect(await third.ending).toBe('revoked');
expect(refusedAttach.staleTokens).toBe(0);
});
it('stops for good when the server refuses its protocol, on open and on attach', async () => {
const refusedOpen = new ScriptedClient();
let opens = 0;
refusedOpen.open = async () => {
opens++;
throw new ControllerApiError(426, 'protocol_unsupported', 'protocol 1', false, null);
};
expect(await run(refusedOpen, new AbortController().signal).ending).toBe('upgrade_required');
expect(opens).toBe(1);
const refusedAttach = new ScriptedClient();
refusedAttach.sockets = () => scriptedSocket([refused(426, 'protocol_unsupported')], 4426);
expect(await run(refusedAttach, new AbortController().signal).ending).toBe('upgrade_required');
});
it('stops for good when the server knows no controller by its credential', async () => {
const scripted = new ScriptedClient();
let opens = 0;
scripted.open = async () => {
opens++;
throw new ControllerApiError(401, 'invalid_credential', 'not valid', false, null);
};
expect(await run(scripted, new AbortController().signal).ending).toBe('credential_invalid');
expect(opens).toBe(1);
});
it('drops a token the socket refused as stale, and attaches again', async () => {
const scripted = new ScriptedClient();
scripted.sockets = () =>
scripted.attaches.length === 1
? scriptedSocket([refused(401, 'token_expired')], 4401)
: scriptedSocket([state], 'open');
const stop = new AbortController();
const { states, ending } = run(scripted, stop.signal);
await waitFor(() => states.includes('connected'), 'a connection');
stop.abort();
await ending;
expect(scripted.staleTokens).toBe(1);
expect(scripted.opens).toHaveLength(1);
});
it('keeps retrying an open that fails', async () => {
const scripted = new ScriptedClient();
let opens = 0;
scripted.open = async () => {
opens++;
if (opens < 3) throw new TypeError('fetch failed');
return opened;
};
scripted.sockets = () => scriptedSocket([state], 'open');
const stop = new AbortController();
const { states, ending } = run(scripted, stop.signal);
await waitFor(() => states.includes('connected'), 'a connection');
stop.abort();
await ending;
expect(opens).toBe(3);
});
it('opens again when a busy server says, not on its own backoff', async () => {
const scripted = new ScriptedClient();
let opens = 0;
scripted.open = async () => {
opens++;
if (opens === 1) throw new ControllerApiError(503, 'busy', 'opening at capacity', true, 0.3);
return opened;
};
scripted.sockets = () => scriptedSocket([state], 'open');
const stop = new AbortController();
const started = Date.now();
const { states, ending } = run(scripted, stop.signal);
await waitFor(() => states.includes('connected'), 'a connection', 3_000);
stop.abort();
await ending;
expect(opens).toBe(2);
// The backoff alone would have opened again after 5 ms.
expect(Date.now() - started).toBeGreaterThanOrEqual(290);
});
it('waits less each time a socket attached, however briefly, and never long', async () => {
const scripted = new ScriptedClient();
let opens = 0;
scripted.open = async () => {
opens++;
if (opens <= 3) throw new TypeError('fetch failed');
return opened;
};
scripted.sockets = () =>
scripted.attaches.length === 1 ? scriptedSocket([state], 1000) : scriptedSocket([], 1006);
const waits: number[] = [];
const log = {
...silentLogger,
warn: (_message: string, fields?: Record<string, unknown>) => {
if (typeof fields?.retryInMs === 'number') waits.push(fields.retryInMs);
},
};
const stop = new AbortController();
const { ending } = run(scripted, stop.signal, {
log,
initialBackoffMs: 8,
maxBackoffMs: 64,
});
await waitFor(() => waits.length >= 5, 'five reconnect waits');
stop.abort();
await ending;
// Doubling from the start after the socket that attached, capped, with
// jitter taking half off (random() is 0 here).
expect(waits.slice(0, 5)).toEqual([4, 8, 16, 8, 16]);
});
it('caps the reconnect wait at seconds, not a minute', () => {
expect(DEFAULT_TIMING.streamInitialBackoffMs).toBeLessThanOrEqual(1_000);
expect(DEFAULT_TIMING.streamMaxBackoffMs).toBeLessThanOrEqual(10_000);
});
const fixture = join(
import.meta.dirname,
'..',
'..',
'..',
'..',
'core',
'tests',
'switch_core',
'fixtures',
'agent_controllers',
'stream_frames.json'
);
it.skipIf(!existsSync(fixture))('reads every frame in Core’s stream fixture', async () => {
const fixtureFrames = JSON.parse(readFileSync(fixture, 'utf8')) as {
event: string;
data: unknown;
}[];
const scripted = new ScriptedClient();
scripted.sockets = () => scriptedSocket(fixtureFrames, 'open');
const stop = new AbortController();
const { frames, ending } = run(scripted, stop.signal);
await waitFor(
() =>
frames.length === fixtureFrames.length - 1 || frames.at(-1)?.type === 'credential.revoked',
'every fixture frame'
);
stop.abort();
await ending;
// An answer to a call goes to the calls waiting on it, not to `onFrame`.
const streamed = fixtureFrames.filter((frame) => frame.event !== 'call.result');
expect(frames.map((frame) => frame.type)).toEqual(
streamed
.map((frame) => frame.event)
.slice(
0,
streamed.findIndex((frame) => frame.event === 'credential.revoked') + 1 || undefined
)
);
});
});
describe('runControllerStream against the controller routes', () => {
let core: FakeCore;
beforeEach(async () => {
core = new FakeCore();
await core.start();
});
afterEach(async () => {
await core.stop();
});
function client(): ControllerClient {
return new ControllerClient({
fetch,
server: core.url,
controllerId: core.controllerId,
version: '0.1.0',
openWebSocket: core.openWebSocket,
tokens: new AccessTokens({
fetch,
server: core.url,
controllerId: core.controllerId,
credential: async () => core.credential,
now: Date.now,
log: silentLogger,
}),
});
}
it('opens, reads, pongs, reattaches, and reopens with cursors after the connection lapses', async () => {
let confirmed = 3;
const frames: ControllerFrame[] = [];
const stop = new AbortController();
const ending = runControllerStream({
client: client(),
cursors: () => ({ 'agent-1': confirmed }),
confirmed: () => ({ 'agent-1': confirmed }),
onOpened: () => {},
onConnected: () => {},
onDisconnected: () => {},
onFrame: async (frame) => void frames.push(frame),
signal: stop.signal,
log: silentLogger,
// Long enough that a loaded machine does not lapse the socket before the test does.
idleTimeoutMs: 30_000,
initialBackoffMs: 5,
maxBackoffMs: 20,
random: () => 0,
});
await waitFor(() => core.streamCount === 1, 'the socket');
core.push('assignment.changed', { revision: 4 });
await waitFor(
() => frames.some((frame) => frame.type === 'assignment.changed'),
'the pushed frame'
);
await waitFor(() => core.beats.length > 0, 'a pong');
expect(core.beats[0]).toEqual({ 'agent-1': 3 });
core.closeStreams();
await waitFor(() => core.streamCount === 1, 'the socket reattached');
expect(core.opens).toHaveLength(1);
confirmed = 9;
core.forgetConnection();
await waitFor(() => core.opens.length === 2, 'a new connection');
expect(core.opens[1]).toEqual({ 'agent-1': 9 });
stop.abort();
expect(await ending).toBe('stopped');
});
});