Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Move the agent connection to one WebSocket, server and runtime (CHOO-3013) (#652)

Abel Dantas8a419be67612
diff --git a/artifacts.yaml b/artifacts.yaml
index e465cb4da..a6090ab16 100644
--- a/artifacts.yaml
+++ b/artifacts.yaml
@@ -181,9 +181,24 @@ contracts:
# attached worker alone is sent `worker_attached`, `relay`, `relay_cancel`,
# `wake`, `mailbox_cancel`, `operation` and `credential`. Nothing changes
# for a local agent, so `accepts` stays at 1.
+ #
+ # Revision 8 moves the connection onto one WebSocket,
+ # `/agents/{id}/connection/ws`, which carries the frames as JSON
+ # (`event`, `data`, `id`) and the heartbeat: the server sends `ping` every
+ # heartbeat interval and the client answers `pong` with its cursor. The
+ # token is checked once, when the socket opens. A hosted worker opens it
+ # with the headers and under the rules revision 7 set for the stream. It
+ # is meant to replace the Server-Sent Events stream and
+ # `POST /connection/beat`, which would make it
+ # the first revision that is not additive, but that removal is deferred:
+ # the stream and the beat are kept beside the socket for a compatibility
+ # window, so `accepts` stays at 1 on both sides. switch-core serves either
+ # transport, and agent-runtime prefers the socket and falls back to the
+ # stream and its beat against a server with no socket. Both go, and
+ # `accepts` rises to 8, once no client still connects over the stream.
artifacts:
- switch-core: { speaks: 7, accepts: 1 }
- agent-runtime: { speaks: 7, accepts: 1 }
+ switch-core: { speaks: 8, accepts: 1 }
+ agent-runtime: { speaks: 8, accepts: 1 }
gateway-api:
description: >-
diff --git a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts
index f8a3bf8b4..ef695c384 100644
--- a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts
@@ -23,7 +23,7 @@ function makeDeps(opts: {
endpoint?: string;
}) {
const missing = opts.missingBinaries ?? [];
- const nodeVersion = opts.nodeVersion ?? 'v18.19.0';
+ const nodeVersion = opts.nodeVersion ?? 'v22.12.0';
const exec = vi.fn(async (command: string, _args: string[]) => {
// Combined working-dir + required-tools probe (`sh -c <script>`). A missing
// dir or exhausted channel rejects; otherwise missing tools and node's
@@ -119,7 +119,7 @@ describe('preflightRemoteSession', () => {
it('fails loud when node is present but too old', async () => {
const deps = makeDeps({ nodeVersion: 'v12.22.9', credsFile: CREDS_FILE });
await expect(preflightRemoteSession(deps)).rejects.toThrow(
- /has Node v12.22.9, but the Switch Console sidecar needs Node 18 or newer/
+ /has Node v12.22.9, but the Switch Console sidecar needs Node 22 or newer/
);
});
diff --git a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts
index 43a0a0f76..cda59aa53 100644
--- a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts
+++ b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts
@@ -15,9 +15,8 @@ import { parseSwitchAgentCredentials } from '@main/core/switch-rooms/switch-cred
* exec context prepends `cd <workDir> &&`, so a missing dir rejects the whole
* command, while present-but-missing tools and node's version are reported on
* stdout. A too-old node is rejected here rather than surfacing later as an
- * opaque "sidecar exited during startup" — the sidecar bundle (and this
- * module's own reachability probe) rely on `fetch`/optional chaining, which
- * only stabilised in Node 18;
+ * opaque "sidecar exited during startup" — the sidecar holds its agent
+ * connection on Node's built-in WebSocket, stable only from Node 22;
* 2. the agent's Switch creds exist on the remote host — checked at the agent's
* provider-neutral per-agent path (`.switch/agents/<name>.json`) first, then
* the legacy `.claude/settings.local.json` for un-migrated installs
@@ -32,9 +31,9 @@ import { parseSwitchAgentCredentials } from '@main/core/switch-rooms/switch-cred
const REQUIRED_BINARIES = ['node', 'git'] as const;
const REACHABILITY_TIMEOUT_MS = 5000;
-// Global `fetch` and `AbortSignal.timeout` (used by the reachability probe below
-// and throughout the sidecar bundle) are only stable from Node 18.
-const MIN_NODE_MAJOR = 18;
+// The sidecar holds its agent connection on Node's built-in `WebSocket`, which
+// is only stable from Node 22.
+const MIN_NODE_MAJOR = 22;
// Prints `missing <tool>` for each absent binary and, when node is present,
// `node <version>` (e.g. `node v18.19.0`), then exits 0 — so a present working
diff --git a/console/packages/agent-controller/src/relay.test.ts b/console/packages/agent-controller/src/relay.test.ts
index a8fa429ca..423c0bd5d 100644
--- a/console/packages/agent-controller/src/relay.test.ts
+++ b/console/packages/agent-controller/src/relay.test.ts
@@ -212,8 +212,13 @@ async function post(path: string, body: unknown, bearer = token): Promise<Respon
}
describe('the relay as the agent protocol, read by the real SwitchEventStream', () => {
- it('serves the relay’s protocol range as switch-core declares its own', () => {
- expect(RELAY_AGENT_PROTOCOL).toEqual(CONTRACTS['agent-protocol']['switch-core']);
+ it('serves switch-core’s revisions up to the socket, which the runtime still accepts', () => {
+ const core = CONTRACTS['agent-protocol']['switch-core'];
+ const runtime = CONTRACTS['agent-protocol']['agent-runtime'];
+ // The relay serves the event stream, not the socket of agent-protocol 8.
+ expect(RELAY_AGENT_PROTOCOL).toEqual({ speaks: 7, accepts: core.accepts });
+ expect(runtime.accepts).toBeLessThanOrEqual(RELAY_AGENT_PROTOCOL.speaks);
+ expect(RELAY_AGENT_PROTOCOL.accepts).toBeLessThanOrEqual(runtime.speaks);
});
it('connects, then delivers addressed events in order, with their ids and missed counts', async () => {
diff --git a/console/packages/agent-controller/src/relay.ts b/console/packages/agent-controller/src/relay.ts
index dc2d7c00e..f1adbeb0a 100644
--- a/console/packages/agent-controller/src/relay.ts
+++ b/console/packages/agent-controller/src/relay.ts
@@ -28,7 +28,12 @@ import type {
* `AgentHub` instead; the room of its calls comes from there too.
*/
-/** The agent-protocol revisions the relay serves, as switch-core declares its own. */
+/**
+ * The agent-protocol revisions the relay serves: switch-core's, up to the
+ * socket. Revision 8 moved the connection onto a WebSocket, which the relay
+ * does not serve; it serves the event stream and its beat, which the runtime
+ * falls back to against a server that speaks 7 or older.
+ */
export const RELAY_AGENT_PROTOCOL = { speaks: 7, accepts: 1 } as const;
/** From this revision a client names its connection incarnation on every beat and room request. */
const FENCED_PROTOCOL_REVISION = 2;
diff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.ts
index 3b2e38ae9..4fba00fe4 100644
--- a/console/packages/shared/src/artifacts.ts
+++ b/console/packages/shared/src/artifacts.ts
@@ -40,8 +40,8 @@ export function artifactVersion(artifact: ArtifactName): string {
export const CONTRACTS = {
// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
'agent-protocol': {
- 'switch-core': { speaks: 7, accepts: 1 },
- 'agent-runtime': { speaks: 7, accepts: 1 },
+ 'switch-core': { speaks: 8, accepts: 1 },
+ 'agent-runtime': { speaks: 8, accepts: 1 },
},
// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
'gateway-api': {
diff --git a/console/packages/switch-agent-runtime/package.json b/console/packages/switch-agent-runtime/package.json
index d6f434eb3..3b7b32f11 100644
--- a/console/packages/switch-agent-runtime/package.json
+++ b/console/packages/switch-agent-runtime/package.json
@@ -44,6 +44,6 @@
"vitest": "^5.0.1"
},
"engines": {
- "node": ">=20"
+ "node": ">=22"
}
}
diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.ts
index 3b2e38ae9..4fba00fe4 100644
--- a/console/packages/switch-agent-runtime/src/artifacts.ts
+++ b/console/packages/switch-agent-runtime/src/artifacts.ts
@@ -40,8 +40,8 @@ export function artifactVersion(artifact: ArtifactName): string {
export const CONTRACTS = {
// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
'agent-protocol': {
- 'switch-core': { speaks: 7, accepts: 1 },
- 'agent-runtime': { speaks: 7, accepts: 1 },
+ 'switch-core': { speaks: 8, accepts: 1 },
+ 'agent-runtime': { speaks: 8, accepts: 1 },
},
// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
'gateway-api': {
diff --git a/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts b/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts
new file mode 100644
index 000000000..d53477775
--- /dev/null
+++ b/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts
@@ -0,0 +1,2296 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import {
+ BEAT_INTERVAL_MS,
+ BEAT_SETTLE_LIMIT_MS,
+ EVICTION_CREDENTIALS_REJECTED,
+ EVICTION_LAUNCH_SUPERSEDED,
+ EVICTION_TAKEN_OVER,
+ SwitchEventStream,
+ WORKER_CAPABILITY_OBSOLETE,
+ WorkerCallError,
+ type Eviction,
+ type SwitchEventStreamDeps,
+} from './event-stream';
+
+/**
+ * The runtime against a server from before the socket (agent-protocol 7 and
+ * older).
+ *
+ * Such a server has no `/connection/ws`: the upgrade fails before it opens,
+ * which Node's WebSocket reports as an error and a 1006 close and nothing
+ * more. The runtime falls back to the event stream and its own heartbeat, and
+ * has to behave there exactly as it did before the socket, so these are the
+ * tests it had then, run with a socket that never opens. The ones on choosing
+ * the transport are at the end.
+ *
+ * Two ways a client can retry something that can never succeed were measured
+ * on a live deployment: a stream re-declaring a room that had been deleted,
+ * refused on every open, reopening at once; and a heartbeat treating "your
+ * connection is gone" as a normal answer and beating on at full cadence.
+ * Neither could self-heal, and between them they produced most of the error
+ * volume on that deployment.
+ */
+
+const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };
+
+function silentLog() {
+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
+}
+
+function encodeFrame(name: string, data: unknown): Uint8Array {
+ return new TextEncoder().encode(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);
+}
+
+/** The frame a server opens every stream with, naming the incarnation this
+ * client is attached to. No `rooms`, so it says nothing about the declared set
+ * a test may be asserting on. */
+function connected(generation = 0): Uint8Array {
+ return encodeFrame('connection_state', { connection_id: 'conn-1', generation });
+}
+
+/** A stream body carrying one frame and then closing, the way the server ends
+ * a displaced stream. */
+function frameThenClose(name: string, data: unknown): ReadableStream<Uint8Array> {
+ return new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(encodeFrame(name, data));
+ controller.close();
+ },
+ });
+}
+
+/**
+ * A connected stream body that stays open, so the loop neither reconnects nor
+ * spins.
+ *
+ * It announces the connection first because the heartbeat waits for that: a
+ * client that has not been told which incarnation it is cannot fence its own
+ * tick, and a body that never announces is a connection that never beats.
+ */
+function openForever(): ReadableStream<Uint8Array> {
+ return new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected());
+ },
+ });
+}
+
+/**
+ * A connected stream body that ends when the client drops the socket.
+ *
+ * `openForever` cannot: its reader is parked on a read that never returns, so
+ * a reopen is invisible to the test rather than absent. A body that honours
+ * the request's signal is what a real socket does, and what a test asserting
+ * on reopens needs.
+ */
+function openUntilAborted(init: { signal: AbortSignal }): ReadableStream<Uint8Array> {
+ return new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected());
+ init.signal.addEventListener('abort', () => controller.close(), { once: true });
+ },
+ });
+}
+
+/** The incarnation each beat carried, in order. */
+function beatGenerations(fetchMock: { mock: { calls: unknown[][] } }): (number | null)[] {
+ return fetchMock.mock.calls
+ .filter((call) => String(call[0]).includes('connection/beat'))
+ .map(
+ (call) =>
+ (JSON.parse((call[1] as { body: string }).body) as { generation: number | null }).generation
+ );
+}
+
+function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {
+ return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));
+}
+
+/**
+ * The socket an old server's upgrade gives: it never opens. Its error and
+ * close arrive on the microtask queue, the way a real socket's arrive as
+ * separate tasks, once the client has attached its listeners.
+ */
+class SocketThatNeverOpens {
+ static opened: string[] = [];
+ readonly CONNECTING = 0;
+ readonly OPEN = 1;
+ readyState = 0;
+ private readonly listeners = new Map<string, Set<(event: unknown) => void>>();
+
+ constructor(url: string) {
+ SocketThatNeverOpens.opened.push(url);
+ queueMicrotask(() => {
+ this.readyState = 3;
+ for (const listener of [...(this.listeners.get('error') ?? [])]) listener({});
+ for (const listener of [...(this.listeners.get('close') ?? [])])
+ listener({ code: 1006, reason: '' });
+ });
+ }
+
+ addEventListener(type: string, listener: (event: unknown) => void): void {
+ const set = this.listeners.get(type) ?? new Set();
+ set.add(listener);
+ this.listeners.set(type, set);
+ }
+
+ removeEventListener(type: string, listener: (event: unknown) => void): void {
+ this.listeners.get(type)?.delete(listener);
+ }
+
+ send(): void {
+ throw new Error('nothing is sent on a socket that never opened');
+ }
+
+ close(): void {}
+}
+
+function makeStream(
+ fetchMock: ReturnType<typeof vi.fn>,
+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }
+) {
+ vi.stubGlobal('WebSocket', SocketThatNeverOpens);
+ SocketThatNeverOpens.opened = [];
+ vi.stubGlobal('fetch', fetchMock);
+ const abort = new AbortController();
+ const log = silentLog();
+ const stream = new SwitchEventStream({
+ creds,
+ connectionId: 'conn-1',
+ scope: 'single',
+ filter: 'all',
+ onEvent: () => {},
+ onGap: () => {},
+ onEvicted: () => {},
+ worker: null,
+ log,
+ signal: abort.signal,
+ ...deps,
+ });
+ stream.start();
+ return { stream, abort, log };
+}
+
+async function flush(times = 8): Promise<void> {
+ for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));
+}
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+});
+
+describe('a room the server refuses', () => {
+ /** The refusal switch-core returns when a declared room no longer exists. */
+ function roomGone(roomId: string) {
+ return {
+ ok: false,
+ status: 403,
+ body: null,
+ text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),
+ };
+ }
+
+ it('is dropped and not declared again', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events')) {
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ return String(url).includes('rooms=')
+ ? roomGone('room-dead')
+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const rejected: { roomId: string; status: number }[] = [];
+ const reported: string[][] = [];
+ const { abort, log } = makeStream(fetchMock, {
+ rooms: ['room-dead'],
+ onRooms: (rooms) => reported.push(rooms),
+ onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),
+ });
+ await flush();
+
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(2);
+ expect(opens[0]).toContain('rooms=room-dead');
+ expect(opens[1]).not.toContain('rooms=');
+ expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);
+ expect(reported).toEqual([[]]);
+ expect(log.error).toHaveBeenCalled();
+ abort.abort();
+ });
+
+ it('keeps the rooms the refusal does not name', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ const u = String(url);
+ if (!u.includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ return u.includes('room-dead')
+ ? roomGone('room-dead')
+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });
+ await flush();
+
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(2);
+ expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');
+ abort.abort();
+ });
+
+ it('stays a transport error when the body names no declared room', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events')) {
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ return {
+ ok: false,
+ status: 404,
+ body: null,
+ text: async (): Promise<string> => JSON.stringify({ detail: 'No such connection' }),
+ };
+ });
+ const rejected: string[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onRoomRejected: ({ roomId }) => rejected.push(roomId),
+ });
+ await vi.advanceTimersByTimeAsync(0);
+
+ // One open, then the backoff, not an immediate retry, and nothing dropped.
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(rejected).toEqual([]);
+ abort.abort();
+ });
+});
+
+describe('credentials the server rejects', () => {
+ function refusal(status: number, detail: string) {
+ return {
+ ok: false,
+ status,
+ body: null,
+ text: async (): Promise<string> => JSON.stringify({ detail }),
+ };
+ }
+
+ /** Open once, be refused, and let a long while pass. */
+ async function afterRefusal(status: number, detail: string) {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? refusal(status, detail)
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const evicted: Eviction[] = [];
+ const { abort, log } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ await vi.advanceTimersByTimeAsync(0);
+ const settled = fetchMock.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ abort.abort();
+ return { fetchMock, evicted, log, settled };
+ }
+
+ it('ends the stream and the heartbeat on a 401, and says so once', async () => {
+ const { fetchMock, evicted, log, settled } = await afterRefusal(401, 'Invalid agent token');
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ // Nothing beyond the beat already in flight when the refusal landed.
+ expect(fetchMock.mock.calls).toHaveLength(settled);
+ expect(evicted).toHaveLength(1);
+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
+ expect(evicted[0]?.reason).toContain('credentials');
+ expect(evicted[0]?.reason).toContain('401');
+ expect(log.error).toHaveBeenCalled();
+ });
+
+ it('ends the stream on a 403 that names no declared room', async () => {
+ const { fetchMock, evicted, settled } = await afterRefusal(
+ 403,
+ 'Agent is not a member of this room'
+ );
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(fetchMock.mock.calls).toHaveLength(settled);
+ expect(evicted).toHaveLength(1);
+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
+ expect(evicted[0]?.reason).toContain('credentials');
+ expect(evicted[0]?.reason).toContain('403');
+ });
+
+ it('ends the stream and the heartbeat on a 401 heartbeat, and says so once', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
+ : refusal(401, 'Invalid agent token')
+ );
+ const evicted: Eviction[] = [];
+ const { abort, log } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS + 1);
+ const settled = fetchMock.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ abort.abort();
+
+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
+ expect(fetchMock.mock.calls).toHaveLength(settled);
+ expect(evicted).toHaveLength(1);
+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
+ expect(evicted[0]?.reason).toContain('credentials');
+ expect(evicted[0]?.reason).toContain('401');
+ expect(log.error).toHaveBeenCalledTimes(1);
+ });
+
+ it('tells a session start to the room itself only where the server says it leaves that to it', async () => {
+ for (const [said, expected] of [
+ [{ announce_session_starts: true }, true],
+ [{}, false],
+ ] as const) {
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: frameThenClose('connection_state', {
+ connection_id: 'conn-1',
+ generation: 0,
+ ...said,
+ }),
+ text: async (): Promise<string> => '',
+ }
+ : new Promise<never>(() => {})
+ );
+ let connected = false;
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onConnected: () => {
+ connected = true;
+ },
+ });
+ expect(stream.announcesSessionStarts).toBe(false);
+ for (let i = 0; i < 50 && !connected; i += 1) await flush();
+ expect(stream.announcesSessionStarts).toBe(expected);
+ abort.abort();
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it('tells the owner once when the stream and the heartbeat are refused together', async () => {
+ vi.useFakeTimers();
+ let opens = 0;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events')) {
+ // Connected once (the heartbeat has nothing to beat for until then)
+ // and refused on every reopen after that.
+ opens += 1;
+ if (opens === 1) {
+ return {
+ ok: true,
+ status: 200,
+ body: frameThenClose('connection_state', { connection_id: 'conn-1', generation: 0 }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ await new Promise((r) => setTimeout(r, BEAT_INTERVAL_MS));
+ return refusal(401, 'Invalid agent token');
+ }
+ // Slow enough to still be in flight when the reopen is refused: both
+ // doors closing at once is the case this test is about.
+ await new Promise((r) => setTimeout(r, 2 * BEAT_INTERVAL_MS));
+ return refusal(401, 'Invalid agent token');
+ });
+ const evicted: Eviction[] = [];
+ const { abort, log } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ abort.abort();
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
+ expect(evicted).toHaveLength(1);
+ expect(log.error).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([503, 429])('keeps reconnecting after HTTP %i', async (status) => {
+ vi.useFakeTimers();
+ let opens = 0;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ opens += 1;
+ return opens === 1
+ ? { ok: false, status, body: null, text: async (): Promise<string> => 'try later' }
+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+
+ it('keeps reconnecting after a network error', async () => {
+ vi.useFakeTimers();
+ let opens = 0;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ opens += 1;
+ if (opens === 1) throw new TypeError('fetch failed');
+ return { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+});
+
+describe('the heartbeat', () => {
+ /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */
+ async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events'))
+ return {
+ ok: true,
+ status: 200,
+ body: openForever(),
+ text: async (): Promise<string> => '',
+ };
+ return { ok: false, status, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+ await vi.advanceTimersByTimeAsync(windowMs);
+ const beats = urlsFor(fetchMock, 'connection/beat').length;
+ abort.abort();
+ return beats;
+ }
+
+ it('backs off when the connection is rejected, instead of beating at full rate', async () => {
+ const windowMs = 20 * BEAT_INTERVAL_MS;
+ // At the base cadence this window holds ~20 beats. Doubling from the base
+ // gives 4s, 8s, 16s…, four requests in the same window.
+ expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);
+ expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);
+ });
+
+ it('keeps backing off the longer the rejection lasts', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events'))
+ return {
+ ok: true,
+ status: 200,
+ body: openForever(),
+ text: async (): Promise<string> => '',
+ };
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(30_000);
+ const early = urlsFor(fetchMock, 'connection/beat').length;
+ await vi.advanceTimersByTimeAsync(30_000);
+ const late = urlsFor(fetchMock, 'connection/beat').length - early;
+
+ expect(late).toBeLessThan(early);
+ abort.abort();
+ });
+
+ it('sends nothing until the server has said which incarnation it is', async () => {
+ vi.useFakeTimers();
+ const server = { announce: (): void => {} };
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ server.announce = () => controller.enqueue(connected(7));
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+ // An unfenced tick is what a displaced client sends, and the server refuses
+ // it. Beating before the first frame would make every healthy connection
+ // open with one.
+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(0);
+
+ server.announce();
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+
+ const beats = fetchMock.mock.calls.filter((c) => String(c[0]).includes('connection/beat'));
+ expect(beats.length).toBeGreaterThan(0);
+ // And it carries the incarnation that frame named, so the server can fence it.
+ const [, sent] = beats[0] as unknown as [string, { body: string }];
+ expect(JSON.parse(sent.body).generation).toBe(7);
+ abort.abort();
+ });
+
+ it('waits for the new incarnation after its own reconnect, instead of standing down', async () => {
+ vi.useFakeTimers();
+ // A server that fences: every attach is a new incarnation, and a tick
+ // naming an older one is refused the way a displaced client's is.
+ const server = { generation: 0, announce: (): void => {}, drop: (): void => {} };
+ const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
+ if (String(url).includes('/events')) {
+ server.generation += 1;
+ const attached = server.generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ const announce = (): void => controller.enqueue(connected(attached));
+ // The first attach is announced at once; the second is held, so
+ // the beat falls due inside the window this test is about.
+ if (attached === 1) announce();
+ else server.announce = announce;
+ server.drop = () => controller.close();
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
+ if (sent === server.generation)
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
+ };
+ });
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
+
+ await vi.advanceTimersByTimeAsync(2 * BEAT_INTERVAL_MS);
+ const first = beatGenerations(fetchMock);
+ expect(first.length).toBeGreaterThan(0);
+ expect(first.every((generation) => generation === 1)).toBe(true);
+
+ server.drop();
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+ // Nothing went out while the reopen was in flight: the incarnation we hold
+ // is the one before it, and a tick carrying that is a stand-down.
+ expect(beatGenerations(fetchMock)).toHaveLength(first.length);
+ expect(evicted).toEqual([]);
+
+ server.announce();
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ const resumed = beatGenerations(fetchMock).slice(first.length);
+ expect(resumed.length).toBeGreaterThan(0);
+ expect(resumed.every((generation) => generation === 2)).toBe(true);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+
+ it('does not stand down over a beat that was already in flight when it reopened', async () => {
+ vi.useFakeTimers();
+ // The gate stops a beat *starting* inside the reattach window. It cannot
+ // recall one that left before the gate shut, and that beat is answered
+ // against the incarnation the reopen replaced, a truthful `taken_over`
+ // about a takeover this client performed on itself.
+ const server = {
+ generation: 0,
+ announce: (): void => {},
+ drop: (): void => {},
+ answerBeat: (): void => {},
+ beatsHeld: 0,
+ };
+ const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
+ if (String(url).includes('/events')) {
+ const claimed = new URL(String(url)).searchParams.get('expected_generation');
+ if (claimed !== null && Number(claimed) !== server.generation) {
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
+ };
+ }
+ server.generation += 1;
+ const attached = server.generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected(attached));
+ server.drop = () => controller.close();
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
+ // Held rather than answered: the point of the test is an answer that
+ // arrives after the connection has moved on beneath it.
+ server.beatsHeld += 1;
+ await new Promise<void>((resolve) => {
+ server.answerBeat = resolve;
+ });
+ if (sent === server.generation)
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
+ };
+ });
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ expect(server.beatsHeld).toBe(1);
+ expect(beatGenerations(fetchMock)).toEqual([1]);
+
+ // Reopen with that beat still outstanding. The bound expires, the open goes
+ // ahead, and the server is at 2 by the time the old beat is answered.
+ server.drop();
+ await vi.advanceTimersByTimeAsync(BEAT_SETTLE_LIMIT_MS + 10 * BEAT_INTERVAL_MS);
+ expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(1);
+
+ server.answerBeat();
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+
+ // Undecidable, so inert. Standing down here stops a client that nothing
+ // has taken anything from.
+ expect(evicted).toEqual([]);
+ const resumed = beatGenerations(fetchMock).slice(1);
+ expect(resumed.length).toBeGreaterThan(0);
+ expect(resumed.every((generation) => generation === 2)).toBe(true);
+ abort.abort();
+ });
+
+ it('stands down when its reattach is refused, rather than trying again', async () => {
+ vi.useFakeTimers();
+ // The other half: a client that missed its eviction and comes back. The
+ // server refuses the reattach without disturbing the holder, and this is
+ // the only thing left that can tell the loser it lost.
+ const server = { generation: 0, drop: (): void => {} };
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ const claimed = new URL(String(url)).searchParams.get('expected_generation');
+ if (claimed !== null) {
+ // Someone else attached while we were away, so our claim is stale.
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
+ };
+ }
+ server.generation += 1;
+ const attached = server.generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected(attached));
+ server.drop = () => controller.close();
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ });
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ server.drop();
+ await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
+
+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
+ // And it stopped asking: retrying is how the pair trade the connection.
+ const attempts = urlsFor(fetchMock, 'expected_generation=1').length;
+ await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
+ expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(attempts);
+ abort.abort();
+ });
+
+ it('returns to the base cadence once a beat lands', async () => {
+ vi.useFakeTimers();
+ let reject = true;
+ // A socket that ends when the client drops it: each refusal reopens, and
+ // the heartbeat is held from the reopen until the new stream is named.
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (String(url).includes('/events'))
+ return {
+ ok: true,
+ status: 200,
+ body: openUntilAborted(init),
+ text: async (): Promise<string> => '',
+ };
+ if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(30_000);
+ reject = false;
+ await vi.advanceTimersByTimeAsync(30_000);
+ const recovered = urlsFor(fetchMock, 'connection/beat').length;
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+
+ expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);
+ abort.abort();
+ });
+});
+
+describe('the heartbeat after a reconnect', () => {
+ /**
+ * A server that reattaches on every open, can drop the current stream, and
+ * records when each stream was opened and each beat arrived. The server's
+ * own rule is what these tests hold the client to: a new connection that
+ * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.
+ */
+ const HEARTBEAT_TTL_MS = 6000;
+
+ /** Either end may close a stream first; the second close is not an error here. */
+ function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {
+ try {
+ controller.close();
+ } catch {
+ // Already closed by the other side.
+ }
+ }
+
+ function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {
+ const opens: number[] = [];
+ const beats: number[] = [];
+ let generation = 0;
+ let current: ReadableStreamDefaultController<Uint8Array> | null = null;
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (String(url).includes('/events')) {
+ opens.push(Date.now());
+ generation += 1;
+ const gen = generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ current = controller;
+ controller.enqueue(connected(gen));
+ init.signal.addEventListener('abort', () => closeQuietly(controller), {
+ once: true,
+ });
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ beats.push(Date.now());
+ const reply = beatReply();
+ if (reply === 'fail') throw new TypeError('fetch failed');
+ if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ /** The server ends the current stream, as it does a lapsed connection. */
+ const drop = (): void => {
+ if (current) closeQuietly(current);
+ };
+ return { fetchMock, opens, beats, drop };
+ }
+
+ it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {
+ // The lockout: beats fail for a while, so the gap between them climbs to
+ // its 30-second cap. The stream then reopens. The server starts its
+ // 6-second clock on the new connection, and the client used to sit out
+ // the rest of the 30 seconds before saying anything, lose the connection
+ // for silence, and repeat that on every connection it opened afterwards.
+ vi.useFakeTimers();
+ let failing = true;
+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ // Long enough of failed beats for the gap to reach its cap.
+ await vi.advanceTimersByTimeAsync(70_000);
+ failing = false;
+ server.drop();
+ await vi.advanceTimersByTimeAsync(40_000);
+
+ const reopenedAt = server.opens[1];
+ expect(reopenedAt).toBeDefined();
+ const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);
+ expect(firstBeatAfter).toBeDefined();
+ expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);
+ abort.abort();
+ });
+
+ it('keeps beating at the base cadence on the new connection once it lands', async () => {
+ vi.useFakeTimers();
+ let failing = true;
+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(70_000);
+ failing = false;
+ server.drop();
+ await vi.advanceTimersByTimeAsync(5_000);
+ const settled = server.beats.length;
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+
+ expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);
+ abort.abort();
+ });
+
+ it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {
+ // The brake the old ordering provided, which beating promptly on every
+ // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The
+ // wait now sits before each reopen and doubles, so a refusing server sees
+ // a handful of reopens, not one every couple of seconds per agent.
+ vi.useFakeTimers();
+ const server = reattachingServer(() => 404);
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(60_000);
+
+ // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.
+ expect(server.opens.length).toBeLessThanOrEqual(6);
+ abort.abort();
+ });
+
+ it('does not throw away a connection the stream reopened while it waited', async () => {
+ // A lapsed connection's socket is closed by the server, and the stream
+ // loop reopens it by itself. If that lands during the heartbeat's wait
+ // after a refusal, reopening again would discard a healthy connection.
+ vi.useFakeTimers();
+ let refuse = true;
+ const server = reattachingServer(() => (refuse ? 404 : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ expect(server.beats.length).toBe(1);
+ refuse = false;
+ // The server ends the stream now, inside the heartbeat's wait.
+ server.drop();
+ await vi.advanceTimersByTimeAsync(20_000);
+
+ // One open for the start, one for the stream's own reopen, not a third
+ // from the heartbeat discarding it.
+ expect(server.opens.length).toBe(2);
+ abort.abort();
+ });
+});
+
+describe('reopening while an open is still in flight', () => {
+ /**
+ * A server that keeps the incarnation the way the real one does: every open
+ * it receives makes a new one, whether or not the client stays to read the
+ * answer, and a reattach claiming an older one is refused as a takeover.
+ * The next open can be held unanswered, the way a slow connection holds it.
+ */
+ function incarnationServer() {
+ const state = {
+ generation: 0,
+ opens: 0,
+ beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',
+ subscribeDrops: false,
+ holdNext: false,
+ release: (): void => {},
+ drop: (): void => {},
+ };
+ const body = (gen: number, signal: AbortSignal) =>
+ new ReadableStream<Uint8Array>({
+ start(controller) {
+ const close = (): void => {
+ try {
+ controller.close();
+ } catch {
+ // already closed
+ }
+ };
+ state.drop = close;
+ controller.enqueue(connected(gen));
+ signal.addEventListener('abort', close, { once: true });
+ },
+ });
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (String(url).includes('connection/subscribe')) {
+ if (!state.subscribeDrops)
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ state.subscribeDrops = false;
+ // The stream ends while the claim is still in flight.
+ state.holdNext = true;
+ state.drop();
+ await new Promise((r) => setTimeout(r, 3_000));
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ if (!String(url).includes('/events')) {
+ if (state.beat === 'slow-404-and-drop') {
+ // The stream ends while this beat is in flight, and its answer comes
+ // back just after the client has started reopening.
+ state.beat = 'ok';
+ state.holdNext = true;
+ state.drop();
+ await new Promise((r) => setTimeout(r, 1_200));
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ }
+ if (state.beat === 404)
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ state.opens += 1;
+ const claimed = new URL(String(url)).searchParams.get('expected_generation');
+ if (claimed !== null && Number(claimed) < state.generation)
+ return {
+ ok: false,
+ status: 409,
+ body: null,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
+ };
+ // The server acts on the open the moment it arrives.
+ state.generation += 1;
+ const gen = state.generation;
+ const answer = {
+ ok: true,
+ status: 200,
+ body: body(gen, init.signal),
+ text: async (): Promise<string> => '',
+ };
+ if (!state.holdNext) return answer;
+ state.holdNext = false;
+ return new Promise((resolve, reject) => {
+ state.release = () => resolve(answer);
+ init.signal.addEventListener(
+ 'abort',
+ () => reject(new DOMException('aborted', 'AbortError')),
+ { once: true }
+ );
+ });
+ });
+ return { state, fetchMock };
+ }
+
+ it('does not cancel an open the server has already acted on, and so does not stand down', async () => {
+ // The self-takeover seen on the pilot. The server closes the stream, the
+ // client starts reopening, and a beat sent just before comes back 404.
+ // Reopening in answer used to cancel the open already in flight, which
+ // the server had acted on, moving the connection to a new incarnation.
+ // The client never read that answer, reattached claiming the old one, was
+ // refused as a takeover, and stood down for good. The "other client" was
+ // its own cancelled request.
+ vi.useFakeTimers();
+ const { state, fetchMock } = incarnationServer();
+ const onEvicted = vi.fn();
+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });
+
+ await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);
+ expect(state.generation).toBe(1);
+
+ state.beat = 'slow-404-and-drop';
+ // Long enough for the refused beat's wait to end while the open is held.
+ await vi.advanceTimersByTimeAsync(12_000);
+ state.release();
+ await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);
+
+ expect(onEvicted).not.toHaveBeenCalled();
+ expect(state.opens).toBe(2);
+ expect(beatGenerations(fetchMock).at(-1)).toBe(2);
+ abort.abort();
+ });
+
+ it('carries out a repoint asked for mid-open once that open is answered', async () => {
+ // The other way into the same hole: a room change reopens to include the
+ // new room. Mid-open, it must wait for the answer rather than cancel it,
+ // and must still happen, since the open in flight was built without it.
+ vi.useFakeTimers();
+ const { state, fetchMock } = incarnationServer();
+ const onEvicted = vi.fn();
+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+
+ state.subscribeDrops = true;
+ const repointing = stream.repoint('room-b');
+ // The claim returns while the stream's reopen is held unanswered.
+ await vi.advanceTimersByTimeAsync(5_000);
+ state.release();
+ await vi.advanceTimersByTimeAsync(5_000);
+ await repointing;
+
+ expect(onEvicted).not.toHaveBeenCalled();
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens.at(-1)).toContain('rooms=room-b');
+ abort.abort();
+ });
+});
+
+describe('a connection another client takes over', () => {
+ it('stands down when the room it repoints to is refused, without reopening', async () => {
+ // `repoint` claims the room *before* it reopens, so the open's own fence
+ // is too late to protect the winner: by the time it refuses, a displaced
+ // client has already rewritten the winner's rooms. The claim carries the
+ // incarnation for that reason, and a refusal ends this client.
+ const opens: string[] = [];
+ const subscribes: (number | null)[] = [];
+ const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
+ if (String(url).includes('/events')) {
+ opens.push(String(url));
+ return {
+ ok: true,
+ status: 200,
+ body: openForever(),
+ text: async (): Promise<string> => '',
+ };
+ }
+ if (String(url).includes('connection/subscribe')) {
+ const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
+ subscribes.push(sent.generation);
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'not your connection' } }),
+ };
+ }
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ const evicted: Eviction[] = [];
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: ['!old'],
+ onEvicted: (e) => evicted.push(e),
+ });
+ await flush();
+ const opensBefore = opens.length;
+
+ await stream.repoint('!new');
+ await flush();
+
+ // The claim named the incarnation the frame gave us, not nothing.
+ expect(subscribes).toEqual([0]);
+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
+ // No reopen: coming back would be the takeover the refusal just denied.
+ expect(opens.length).toBe(opensBefore);
+ abort.abort();
+ });
+
+ it('does not claim a room before the frame that names its incarnation', async () => {
+ // Claiming nothing is how a client too old to have an incarnation gets
+ // through, so a claim sent in the window before the first frame would go
+ // through the same door, and that window is exactly when this client may
+ // already have been displaced without knowing it.
+ let announce: () => void = () => {};
+ const subscribes: (number | null)[] = [];
+ const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
+ if (String(url).includes('/events'))
+ return {
+ ok: true,
+ status: 200,
+ // Attached, but silent: the server has not said which incarnation
+ // this socket is yet.
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ announce = () => controller.enqueue(connected());
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ if (String(url).includes('connection/subscribe')) {
+ const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
+ subscribes.push(sent.generation);
+ }
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ const { stream, abort } = makeStream(fetchMock, { rooms: ['!old'] });
+ await flush();
+
+ const repointed = stream.repoint('!new');
+ await flush();
+
+ expect(subscribes).toEqual([]);
+
+ announce();
+ await repointed;
+
+ expect(subscribes).toEqual([0]);
+ abort.abort();
+ });
+
+ it('halts instead of reopening, because reopening would be a takeover back', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: frameThenClose('evicted', {
+ code: 'taken_over',
+ reason: 'another stream attached to this connection',
+ room_id: null,
+ }),
+ text: async (): Promise<string> => '',
+ }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+
+ // Two clients reopening each other's connection is the loop this prevents:
+ // whoever reopens wins, so a displaced client that reopens starts it again.
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(evicted).toEqual([
+ {
+ code: EVICTION_TAKEN_OVER,
+ reason: 'another stream attached to this connection',
+ roomId: null,
+ },
+ ]);
+ abort.abort();
+ });
+
+ it('stands down when its heartbeat is refused, rather than taking the connection back', async () => {
+ vi.useFakeTimers();
+ // The case the SSE frame cannot cover: this client's socket dropped before
+ // the eviction reached it, so the refused heartbeat is the only way it ever
+ // hears that it lost.
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
+ : {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({
+ detail: {
+ code: 'taken_over',
+ message: 'connection conn-1 was reopened since incarnation 0',
+ },
+ }),
+ }
+ );
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+
+ // One beat and no second open: reopening is a takeover, so a loser that
+ // reopens pulls the connection straight back off the client that won it.
+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
+ abort.abort();
+ });
+
+ it('still reopens on a refusal that names no code, the way an old server sends it', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: openUntilAborted(init),
+ text: async (): Promise<string> => '',
+ }
+ : {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: 'connection conn-1 has no stream attached' }),
+ }
+ );
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+
+ // Only the takeover is terminal. Everything else a 409 can mean is still
+ // something a reopen fixes, and a server too old to say which is one of them.
+ expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+
+ it('reads an old server’s wording as a takeover when it sends no code', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: frameThenClose('evicted', {
+ reason: 'another stream attached to this connection',
+ }),
+ text: async (): Promise<string> => '',
+ }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+
+ expect(evicted[0]?.code).toBe(EVICTION_TAKEN_OVER);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ abort.abort();
+ });
+});
+
+describe('a stream the server closes cleanly', () => {
+ /** Opens, closes at once, forever: the shape of a contested connection. */
+ function closesAtOnce() {
+ return vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.close();
+ },
+ }),
+ text: async (): Promise<string> => '',
+ }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ }
+
+ it('waits before reopening rather than reconnecting at full rate', async () => {
+ vi.useFakeTimers();
+ const fetchMock = closesAtOnce();
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ // A minute of 1s, 2s, 4s… is seven opens. Without pacing a clean close it
+ // is an unbounded spin, limited only by how fast the server can answer.
+ await vi.advanceTimersByTimeAsync(60_000);
+
+ expect(urlsFor(fetchMock, '/events').length).toBeLessThanOrEqual(8);
+ abort.abort();
+ });
+
+ it('keeps backing off rather than resetting on every handshake', async () => {
+ vi.useFakeTimers();
+ const fetchMock = closesAtOnce();
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(60_000);
+ const early = urlsFor(fetchMock, '/events').length;
+ await vi.advanceTimersByTimeAsync(60_000);
+ const late = urlsFor(fetchMock, '/events').length - early;
+
+ // An open is not evidence of a working stream. Resetting the curve on the
+ // handshake held two contending clients at a reconnect a second forever.
+ expect(late).toBeLessThan(early);
+ abort.abort();
+ });
+});
+
+it('does not acknowledge a delivery before the consumer has saved it', async () => {
+ let finish: () => void = () => {};
+ const saved = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ const onEvent = vi.fn(() => saved);
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!url.includes('/events')) return Response.json({});
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(
+ new TextEncoder().encode(
+ 'id: 7\nevent: message\ndata: {"type":"message","room_id":"room","sequence":7}\n\n'
+ )
+ );
+ },
+ })
+ );
+ });
+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room'], startCursor: 6, onEvent });
+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());
+ expect(stream.position).toBe(6);
+ finish();
+ await vi.waitFor(() => expect(stream.position).toBe(7));
+ abort.abort();
+});
+
+it('replays from an explicitly saved zero cursor instead of starting at head', async () => {
+ const fetchMock = vi
+ .fn()
+ .mockImplementation(async (url: string) =>
+ url.includes('/events?')
+ ? { ok: true, body: openForever() }
+ : { ok: true, json: async () => ({ rooms: [] }) }
+ );
+ const { abort } = makeStream(fetchMock, { rooms: [], startCursor: 0 });
+ try {
+ await flush();
+ expect(new URL(urlsFor(fetchMock, '/events?')[0]).searchParams.get('start_from')).toBe('0');
+ } finally {
+ abort.abort();
+ }
+});
+
+function resetFrames(url: string): Response {
+ if (!url.includes('/events')) return Response.json({});
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(
+ new TextEncoder().encode(
+ 'event: gap\ndata: {"from_sequence":0,"resumed_at":0,"reason":"buffer reset"}\n\n' +
+ 'id: 1\nevent: message\ndata: {"type":"message","room_id":"room","sequence":1}\n\n'
+ )
+ );
+ },
+ })
+ );
+}
+
+it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {
+ let finish!: () => void;
+ const saved = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ const onGap = vi.fn(() => saved);
+ const onEvent = vi.fn();
+ const { stream, abort } = makeStream(
+ vi.fn(async (url: string) => resetFrames(url)),
+ { rooms: ['room'], startCursor: 4, onGap, onEvent }
+ );
+ try {
+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
+ expect(onGap).toHaveBeenCalledWith({
+ fromSequence: 0,
+ resumedAt: 0,
+ cursorReset: true,
+ reason: 'buffer reset',
+ });
+ expect(stream.position).toBe(4);
+ expect(onEvent).not.toHaveBeenCalled();
+ finish();
+ await vi.waitFor(() => expect(stream.position).toBe(1));
+ expect(onEvent).toHaveBeenCalledOnce();
+ } finally {
+ abort.abort();
+ }
+});
+
+it('does not advance or deliver when persisting the reset checkpoint fails', async () => {
+ const onGap = vi.fn(async () => {
+ throw new Error('journal unavailable');
+ });
+ const onEvent = vi.fn();
+ const { stream, abort } = makeStream(
+ vi.fn(async (url: string) => resetFrames(url)),
+ { rooms: ['room'], startCursor: 4, onGap, onEvent }
+ );
+ try {
+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
+ expect(stream.position).toBe(4);
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+});
+
+describe('the permission to start a session', () => {
+ function reopenable() {
+ return vi.fn(async (url: string, init: { signal: AbortSignal }) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: openUntilAborted(init),
+ text: async (): Promise<string> => '',
+ }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ }
+
+ it('is redeclared on the wire when it changes under a live connection', async () => {
+ // Turning automatic sessions off while the controller is connected has to
+ // reach the server, or it goes on promising a session nothing will start.
+ const fetchMock = reopenable();
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ await flush();
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(urlsFor(fetchMock, '/events')[0]).toContain('spawn_capable=true');
+
+ stream.setSpawnCapable(false);
+ await flush();
+
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(2);
+ expect(opens[1]).not.toContain('spawn_capable');
+ // A reattach, not a takeover: the incarnation goes with it, so the server
+ // recognises the same client rather than evicting it in favour of itself.
+ expect(opens[1]).toContain('expected_generation=0');
+ abort.abort();
+ });
+
+ /**
+ * A server that fences reattaches the way the real one does: every accepted
+ * open makes a new incarnation, and an open claiming an older one is refused.
+ * `holdStateOn` withholds the frame naming the incarnation for one open, so a
+ * test can act in the window where the client has not been told yet.
+ */
+ function fencingServer(holdStateOn: number) {
+ let generation = 0;
+ let opens = 0;
+ const announce: Array<() => void> = [];
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ const claimed = new URL(String(url)).searchParams.get('expected_generation');
+ if (claimed !== null && Number(claimed) !== generation)
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: EVICTION_TAKEN_OVER } }),
+ };
+ opens += 1;
+ generation += 1;
+ const held = opens === holdStateOn;
+ const named = generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ let closed = false;
+ const send = () => {
+ if (!closed) controller.enqueue(connected(named));
+ };
+ if (held) announce.push(send);
+ else send();
+ init.signal.addEventListener(
+ 'abort',
+ () => {
+ closed = true;
+ controller.close();
+ },
+ { once: true }
+ );
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ });
+ return { fetchMock, announce };
+ }
+
+ it('holds a second change until the server has answered the first', async () => {
+ // Redeclaring reattaches, which claims the incarnation this client believes
+ // it holds. A second change sent before the first open was answered claims
+ // the one before it, which the server has moved past, and refuses as a
+ // takeover. That would take the agent's only connection off the air over
+ // two clicks in a row.
+ const onEvicted = vi.fn();
+ const { fetchMock, announce } = fencingServer(2);
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true, onEvicted });
+ try {
+ await flush();
+ stream.setSpawnCapable(false);
+ await flush();
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+
+ // The reopen is out but unanswered, so this one has nothing safe to claim.
+ stream.setSpawnCapable(true);
+ await flush();
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+
+ announce[0]();
+ await flush();
+
+ // Without the hold, this is where the connection has already gone: the
+ // third open claims the incarnation the second one replaced.
+ expect(onEvicted).not.toHaveBeenCalled();
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(3);
+ expect(opens[2]).toContain('spawn_capable=true');
+ expect(opens[2]).toContain('expected_generation=2');
+ } finally {
+ abort.abort();
+ }
+ });
+
+ it('carries nothing when a change made in that window is taken back', async () => {
+ const { fetchMock, announce } = fencingServer(2);
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ try {
+ await flush();
+ stream.setSpawnCapable(false);
+ await flush();
+ stream.setSpawnCapable(true);
+ stream.setSpawnCapable(false);
+ await flush();
+
+ announce[0]();
+ await flush();
+
+ // The held open already declares what the setting settled back to.
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ } finally {
+ abort.abort();
+ }
+ });
+
+ it('does not reopen the socket when it is set to what it already is', async () => {
+ const fetchMock = reopenable();
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ await flush();
+
+ stream.setSpawnCapable(true);
+ await flush();
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ abort.abort();
+ });
+});
+
+it('routes command wakeups separately from room events and their cursor', async () => {
+ const onCommands = vi.fn();
+ const onEvent = vi.fn();
+ const fetchMock = vi.fn(async (url: string) =>
+ url.includes('/events')
+ ? new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected());
+ controller.enqueue(encodeFrame('session_commands', { session_ids: ['session'] }));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ )
+ : Response.json({})
+ );
+ const { abort } = makeStream(fetchMock, { rooms: [], scope: 'all', onCommands, onEvent });
+ try {
+ await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+});
+
+function streaming(...frames: Uint8Array[]) {
+ return vi.fn(async (url: string) =>
+ url.includes('/events')
+ ? new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected());
+ for (const frame of frames) controller.enqueue(frame);
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ )
+ : Response.json({})
+ );
+}
+
+it('hands an approval outcome to its own callback, not the room-event path', async () => {
+ const onApprovalOutcome = vi.fn();
+ const onEvent = vi.fn();
+ const outcome = {
+ session_id: 'session',
+ request_id: 'permission',
+ state: 'answered',
+ answer: '0',
+ answered_by: '@person:test',
+ answered_at: '2026-09-24T12:00:00Z',
+ };
+ const { abort } = makeStream(streaming(encodeFrame('approval_outcome', outcome)), {
+ rooms: [],
+ scope: 'all',
+ onApprovalOutcome,
+ onEvent,
+ });
+ try {
+ await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+});
+
+it('drops a frame it does not know that is not a room event', async () => {
+ const onEvent = vi.fn();
+ const { abort } = makeStream(
+ streaming(
+ encodeFrame('something_new', { detail: 'from a later server' }),
+ encodeFrame('approval_outcome', { session_id: 'session' }),
+ encodeFrame('message', { type: 'message', room_id: 'room', payload: {} })
+ ),
+ { rooms: [], scope: 'all', onEvent }
+ );
+ try {
+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));
+ expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });
+ } finally {
+ abort.abort();
+ }
+});
+
+it('hands a relayed session command to its own callback', async () => {
+ const onSessionCommand = vi.fn();
+ const onEvent = vi.fn();
+ const command = {
+ contractVersion: 1,
+ commandId: 'command-1',
+ sessionId: 'session',
+ epoch: 'epoch',
+ origin: { surface: 'console', actorId: 'owner', roomId: null, threadId: null, messageId: null },
+ body: { type: 'turn.interrupt', turnId: 'turn' },
+ };
+ const { abort } = makeStream(
+ streaming(
+ encodeFrame('session_command', { commandId: 'no-session' }),
+ encodeFrame('session_command', command)
+ ),
+ { rooms: [], scope: 'all', onSessionCommand, onEvent }
+ );
+ try {
+ await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));
+ expect(onSessionCommand).toHaveBeenCalledTimes(1);
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+});
+
+it('hands a room another connection took over to its own callback', async () => {
+ const onRoomReleased = vi.fn();
+ const onEvent = vi.fn();
+ const { abort } = makeStream(
+ streaming(
+ encodeFrame('room_released', { session_id: 'no-room' }),
+ encodeFrame('room_released', { room_id: 'room', session_id: 'session' }),
+ encodeFrame('room_released', { room_id: 'other', session_id: null })
+ ),
+ { rooms: [], scope: 'all', onRoomReleased, onEvent }
+ );
+ try {
+ await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));
+ expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });
+ expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+});
+
+it('says when the server has confirmed an open', async () => {
+ const onConnected = vi.fn();
+ const { abort } = makeStream(streaming(), { rooms: [], scope: 'all', onConnected });
+ try {
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
+ } finally {
+ abort.abort();
+ }
+});
+
+it('says when an open stream ended and it is trying again', async () => {
+ const onConnected = vi.fn();
+ const onDisconnected = vi.fn();
+ let opens = 0;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!url.includes('/events')) return Response.json({});
+ opens += 1;
+ if (opens === 1)
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected());
+ controller.close();
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ return new Response(openForever(), { headers: { 'content-type': 'text/event-stream' } });
+ });
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ scope: 'all',
+ onConnected,
+ onDisconnected,
+ });
+ try {
+ await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
+ expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });
+ expect(onConnected).toHaveBeenCalledTimes(1);
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
+ expect(onDisconnected).toHaveBeenCalledTimes(1);
+ } finally {
+ abort.abort();
+ }
+});
+
+it('states placements on the attached incarnation, and raises on a refusal', async () => {
+ const bodies: unknown[] = [];
+ let refuse = false;
+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
+ if (url.includes('/events'))
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected(4));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ if (url.includes('connection/placements')) {
+ bodies.push(JSON.parse(init!.body!));
+ return refuse ? new Response('not a member', { status: 403 }) : Response.json({ ok: true });
+ }
+ return Response.json({});
+ });
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], scope: 'all' });
+ try {
+ await stream.replacePlacements({ session: 'room' });
+ expect(bodies).toEqual([
+ { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },
+ ]);
+ refuse = true;
+ await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');
+ } finally {
+ abort.abort();
+ }
+});
+
+describe('a hosted worker', () => {
+ const worker = { capability: 'cap-1', bootId: 'boot-1', instanceId: 'i-1', stateVersion: 1 };
+
+ function refused(status: number, code: string) {
+ return {
+ ok: false,
+ status,
+ body: null,
+ text: async (): Promise<string> => JSON.stringify({ detail: { code, message: code } }),
+ };
+ }
+
+ it('states its capability and host on the open', async () => {
+ const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({
+ ok: true,
+ status: 200,
+ body: openForever(),
+ text: async (): Promise<string> => '',
+ }));
+ const { abort } = makeStream(fetchMock, { rooms: [], worker });
+ await flush();
+ const open = fetchMock.mock.calls.find((c) => String(c[0]).includes('/events'));
+ expect(open?.[1]?.headers).toMatchObject({
+ 'X-Switch-Worker-Capability': 'cap-1',
+ 'X-Switch-Host-Boot-Id': 'boot-1',
+ 'X-Switch-Host-Instance-Id': 'i-1',
+ 'X-Switch-Worker-State-Version': '1',
+ });
+ abort.abort();
+ });
+
+ it.each([
+ [403, WORKER_CAPABILITY_OBSOLETE],
+ [403, 'worker_capability_required'],
+ [426, 'upgrade_required'],
+ [403, 'hosted_worker_only'],
+ ])('stops for good on a %i %s open', async (status, code) => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async () => refused(status, code));
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ worker,
+ onEvicted: (e) => evicted.push(e),
+ });
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(evicted.map((e) => e.code)).toEqual([code]);
+ abort.abort();
+ });
+
+ it('backs off and retries while another worker is attached', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async () => refused(409, 'worker_already_attached'));
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ worker,
+ onEvicted: (e) => evicted.push(e),
+ });
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);
+ expect(urlsFor(fetchMock, '/events').length).toBeLessThan(6);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+
+ it('halts on a launch_superseded eviction', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? {
+ ok: true,
+ status: 200,
+ body: frameThenClose('evicted', { code: 'launch_superseded', reason: 'bumped' }),
+ text: async (): Promise<string> => '',
+ }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ worker,
+ onEvicted: (e) => evicted.push(e),
+ });
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_LAUNCH_SUPERSEDED]);
+ abort.abort();
+ });
+
+ it('hands worker frames over in order and not as room events', async () => {
+ const body = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected(4));
+ controller.enqueue(encodeFrame('worker_attached', { launch_revision: 3 }));
+ controller.enqueue(encodeFrame('operation', { id: 'op-1' }));
+ },
+ });
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const frames: [string, unknown][] = [];
+ const events: unknown[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ worker,
+ onEvent: (e) => void events.push(e),
+ onWorkerFrame: (name, data) => void frames.push([name, data]),
+ });
+ await flush();
+ expect(frames).toEqual([
+ ['worker_attached', { launch_revision: 3 }],
+ ['operation', { id: 'op-1' }],
+ ]);
+ expect(events).toEqual([]);
+ abort.abort();
+ });
+
+ it('sends an up-call as its connection and incarnation, and raises a refusal with its code', async () => {
+ const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => {
+ const u = String(url);
+ if (u.includes('/events'))
+ return {
+ ok: true,
+ status: 200,
+ body: openForever(),
+ text: async (): Promise<string> => '',
+ };
+ if (u.endsWith('/claim')) return refused(409, 'already_claimed');
+ return {
+ ok: true,
+ status: 200,
+ text: async (): Promise<string> => '{"queued_operations":[]}',
+ };
+ });
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], worker });
+ await expect(
+ stream.workerCall('/agents/agent-1/connection/idle', { busy: false })
+ ).resolves.toEqual({
+ queued_operations: [],
+ });
+ const idle = fetchMock.mock.calls.find((c) => String(c[0]).endsWith('/connection/idle'));
+ expect(String(idle?.[0])).toBe('https://switch.test/agents/agent-1/connection/idle');
+ expect(JSON.parse(String(idle?.[1]?.body))).toEqual({
+ busy: false,
+ connection_id: 'conn-1',
+ generation: 0,
+ });
+ const claim = stream.workerCall('/hosted/operations/op-1/claim', {});
+ await expect(claim).rejects.toBeInstanceOf(WorkerCallError);
+ await expect(claim).rejects.toMatchObject({ status: 409, code: 'already_claimed' });
+ abort.abort();
+ });
+});
+
+describe('test_gap_above_cursor_is_not_cursor_reset', () => {
+ it('reports a restart gap resuming above the cursor as no cursor reset', async () => {
+ const body = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected());
+ controller.enqueue(
+ new TextEncoder().encode(
+ 'id: 7\nevent: message\ndata: {"type":"message","room_id":"r"}\n\n'
+ )
+ );
+ controller.enqueue(
+ encodeFrame('gap', {
+ from_sequence: 7,
+ resumed_at: 2 ** 32 - 1,
+ reason: 'server restarted',
+ rooms: [],
+ })
+ );
+ },
+ });
+ const fetchMock = vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];
+ const { stream, abort } = makeStream(fetchMock, { rooms: [], onGap: (g) => void gaps.push(g) });
+ await flush();
+ expect(gaps).toEqual([expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false })]);
+ expect(stream.position).toBe(2 ** 32 - 1);
+ abort.abort();
+ });
+});
+
+it('does not give its connection up to itself over a request answered across its own reopen', async () => {
+ // Seen on a real host: placements stated under incarnation 4, the stream
+ // reopened (to 5) before the answer came, and the answer, "taken over:
+ // reopened since 4, now at 5", named this very client as the new holder.
+ // It stood down for good, with nobody else anywhere near the connection.
+ const opened: ReadableStreamDefaultController<Uint8Array>[] = [];
+ let answerPlacements: (response: Response) => void = () => {};
+ const fetchMock = vi.fn(async (url: string) => {
+ if (url.includes('/events')) {
+ const generation = 4 + opened.length;
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ opened.push(controller);
+ controller.enqueue(connected(generation));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ }
+ if (url.includes('connection/placements'))
+ return new Promise<Response>((resolve) => {
+ answerPlacements = resolve;
+ });
+ return Response.json({});
+ });
+ const evicted: Eviction[] = [];
+ const onConnected = vi.fn();
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: [],
+ scope: 'all',
+ onConnected,
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ try {
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
+ const stating = stream.replacePlacements({ session: 'room' });
+ await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));
+
+ opened[0]!.close();
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
+ answerPlacements(
+ Response.json(
+ {
+ detail: {
+ code: 'taken_over',
+ message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',
+ },
+ },
+ { status: 409 }
+ )
+ );
+
+ await expect(stating).rejects.toThrow('409');
+ expect(evicted).toEqual([]);
+ } finally {
+ abort.abort();
+ }
+});
+
+it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ if (url.includes('/events'))
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected(4));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ if (url.includes('connection/placements'))
+ return Response.json(
+ { detail: { code: 'taken_over', message: 'another client holds it' } },
+ { status: 409 }
+ );
+ return Response.json({});
+ });
+ const evicted: Eviction[] = [];
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: [],
+ scope: 'all',
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ try {
+ await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');
+ expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);
+ } finally {
+ abort.abort();
+ }
+});
+
+describe('the heartbeat across its own reopen after a refusal', () => {
+ /**
+ * What took every in-Console watcher off the air when a laptop woke
+ * (2026-10-01, four agents within ten seconds). The connection had lapsed
+ * while the machine slept, so the first beat after waking was refused with a
+ * 404. The heartbeat asked the stream to reopen and went straight back to
+ * its gate, which it had found open a moment before the stream loop closed
+ * it for the reopen. One beat went out under the lapsed incarnation right
+ * behind the open that replaced it. The server, holding the new incarnation,
+ * answered that beat `taken_over`, and the client believed it: it stood down
+ * over a connection it had just made itself. In the server log the open and
+ * the stale beat arrived 11 ms apart, every time.
+ */
+ function lapsingServer() {
+ let generation = 0;
+ let lapsed = false;
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {
+ if (String(url).includes('/events')) {
+ generation += 1;
+ lapsed = false;
+ const gen = generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected(gen));
+ init.signal.addEventListener(
+ 'abort',
+ () => {
+ try {
+ controller.error(new DOMException('The operation was aborted.', 'AbortError'));
+ } catch {
+ // Already closed.
+ }
+ },
+ { once: true }
+ );
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;
+ if (claimed !== null && claimed !== generation) {
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({
+ detail: {
+ code: 'taken_over',
+ message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,
+ },
+ }),
+ };
+ }
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ /** The server sweeps the connection for silence; its socket stays up
+ * client-side, the way a socket does across a sleep. */
+ const lapse = (): void => {
+ lapsed = true;
+ };
+ return { fetchMock, lapse };
+ }
+
+ it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {
+ vi.useFakeTimers();
+ const server = lapsingServer();
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(server.fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);
+ server.lapse();
+ await vi.advanceTimersByTimeAsync(30_000);
+
+ // The beat refused with a 404 is the last one under the lapsed
+ // incarnation: what follows it on the wire is the reopen, not another beat
+ // that the server could only take for someone else's.
+ const sent = server.fetchMock.mock.calls.map((call) =>
+ String(call[0]).includes('/events')
+ ? 'open'
+ : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`
+ );
+ const reopen = sent.indexOf('open', 1);
+ expect(reopen).toBeGreaterThan(0);
+ expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);
+ expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+});
+
+describe('choosing the transport', () => {
+ /** A stream body that announces a server speaking `protocol` (none for a
+ * server from before the declaration), then ends. */
+ function announcingThenClose(protocol?: number): ReadableStream<Uint8Array> {
+ return frameThenClose('connection_state', {
+ connection_id: 'conn-1',
+ generation: 0,
+ ...(protocol === undefined ? {} : { protocol }),
+ });
+ }
+
+ function serving(body: () => ReadableStream<Uint8Array>) {
+ return vi.fn(async (url: string) =>
+ String(url).includes('/events')
+ ? { ok: true, status: 200, body: body(), text: async (): Promise<string> => '' }
+ : { ok: true, status: 200, text: async (): Promise<string> => '' }
+ );
+ }
+
+ // 6 is a server before the hosted worker, 7 one with it: neither has a socket.
+ it.each([6, 7])(
+ 'falls back to the event stream, then opens it straight away for a server speaking %i',
+ async (protocol) => {
+ vi.useFakeTimers();
+ const fetchMock = serving(() => announcingThenClose(protocol));
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ // The stream ends at once each time: opens at 0, 1 and 3 seconds.
+ await vi.advanceTimersByTimeAsync(4000);
+
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
+ expect(SocketThatNeverOpens.opened).toHaveLength(1);
+ abort.abort();
+ }
+ );
+
+ it('declares a range a server from before the socket shares', async () => {
+ const fetchMock = serving(() => openForever());
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+ await flush();
+
+ const params = new URL(urlsFor(fetchMock, '/events')[0]!).searchParams;
+ expect(params.get('protocol')).toBe('8');
+ expect(Number(params.get('protocol_accepts'))).toBeLessThanOrEqual(6);
+ abort.abort();
+ });
+
+ it('keeps trying the socket first when neither answers, and says it could not connect', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async () => {
+ throw new TypeError('fetch failed');
+ });
+ const disconnected: string[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: [],
+ onDisconnected: ({ error }) => disconnected.push(error),
+ });
+
+ await vi.advanceTimersByTimeAsync(4000);
+
+ // A network failure decides nothing: every open still starts with the socket.
+ expect(SocketThatNeverOpens.opened).toHaveLength(3);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
+ expect(disconnected[0]).toContain('could not connect');
+ abort.abort();
+ });
+
+ it('goes back to the socket once the server says it speaks a revision with one', async () => {
+ vi.useFakeTimers();
+ // The socket failed to open (a proxy, say) but the server is a new one.
+ const fetchMock = serving(() => announcingThenClose(8));
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(SocketThatNeverOpens.opened).toHaveLength(2);
+ abort.abort();
+ });
+
+ it('moves to the socket at once when the stream it went to straight away has been removed', async () => {
+ vi.useFakeTimers();
+ let opens = 0;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ opens += 1;
+ if (opens === 1)
+ return {
+ ok: true,
+ status: 200,
+ body: announcingThenClose(6),
+ text: async (): Promise<string> => '',
+ };
+ return {
+ ok: false,
+ status: 410,
+ body: null,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'transport_removed', message: 'gone' } }),
+ };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ // The first stream ends at once; the next open, a second later, is the
+ // stream again and is refused as removed.
+ await vi.advanceTimersByTimeAsync(1500);
+ expect(SocketThatNeverOpens.opened).toHaveLength(2);
+
+ // Then the backoff, not a spin between the two.
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(SocketThatNeverOpens.opened).toHaveLength(2);
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
+ abort.abort();
+ });
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
index 516e3c6e2..2fa9b01c1 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -1,106 +1,208 @@
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
- BEAT_INTERVAL_MS,
- BEAT_SETTLE_LIMIT_MS,
EVICTION_CREDENTIALS_REJECTED,
EVICTION_LAUNCH_SUPERSEDED,
EVICTION_TAKEN_OVER,
SwitchEventStream,
WORKER_CAPABILITY_OBSOLETE,
- WorkerCallError,
type Eviction,
type SwitchEventStreamDeps,
} from './event-stream';
/**
- * Two ways a client can retry something that can never succeed.
+ * The client side of the agent connection, held to what the server does.
*
- * Both were measured on a live deployment: a stream re-declaring a room that
- * had been deleted, refused on every open, reopening at once; and a heartbeat
- * treating "your connection is gone" as a normal answer and beating on at full
- * cadence. Neither could self-heal, and between them they produced most of the
- * error volume on that deployment.
+ * Much of what is pinned here was measured on a live deployment: a stream
+ * re-declaring a room that had been deleted, refused on every open, reopening
+ * at once; a client cancelling its own open and then standing down over the
+ * incarnation that open made. Neither could self-heal.
+ *
+ * The server end is a fake WebSocket the test drives by hand: it accepts,
+ * sends frames and pings, refuses, and closes, and records what the client
+ * sent back. Its events are dispatched on the microtask queue, the way a real
+ * socket's arrive as separate tasks, so nothing a test sends is seen before
+ * the client has attached its listeners.
*/
const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };
-function silentLog() {
- return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
+type Listener = (event: { data?: string; code?: number; reason?: string }) => void;
+
+/** What the server does with each socket the client opens, and its index. */
+type Script = (socket: FakeSocket, index: number) => void;
+
+/** Every socket opened in the current test, and the script that answers them. */
+const server: { sockets: FakeSocket[]; script: Script } = {
+ sockets: [],
+ script: () => {},
+};
+
+/** Open and announce incarnation 0: a healthy, attached connection. */
+const attach: Script = (socket) => {
+ socket.open();
+ socket.announce();
+};
+
+class FakeSocket {
+ static readonly CONNECTING = 0;
+ static readonly OPEN = 1;
+ static readonly CLOSING = 2;
+ static readonly CLOSED = 3;
+ readonly CONNECTING = 0;
+ readonly OPEN = 1;
+ readonly CLOSING = 2;
+ readonly CLOSED = 3;
+
+ readyState = 0;
+ readonly url: string;
+ readonly headers: Record<string, string>;
+ /** Every message the client sent, parsed. */
+ readonly sent: unknown[] = [];
+ /** The code the client closed with, if it was the one to close. */
+ closedByClient: number | null = null;
+ private ending = false;
+ private readonly listeners = new Map<string, Set<Listener>>();
+
+ constructor(url: string, init?: { headers?: Record<string, string> }) {
+ this.url = url;
+ this.headers = { ...init?.headers };
+ server.sockets.push(this);
+ server.script(this, server.sockets.length - 1);
+ }
+
+ get params(): URLSearchParams {
+ return new URL(this.url).searchParams;
+ }
+
+ addEventListener(type: string, listener: Listener): void {
+ const set = this.listeners.get(type) ?? new Set<Listener>();
+ set.add(listener);
+ this.listeners.set(type, set);
+ }
+
+ removeEventListener(type: string, listener: Listener): void {
+ this.listeners.get(type)?.delete(listener);
+ }
+
+ send(data: string): void {
+ this.sent.push(JSON.parse(data));
+ }
+
+ close(code = 1000, reason = ''): void {
+ if (this.ending) return;
+ this.closedByClient = code;
+ this.readyState = this.CLOSING;
+ this.end(code, reason);
+ }
+
+ // The server's side.
+
+ open(): void {
+ this.dispatch(
+ 'open',
+ {},
+ () => this.readyState === this.CONNECTING,
+ () => {
+ this.readyState = this.OPEN;
+ }
+ );
+ }
+
+ frame(event: string, data: unknown, id?: number): void {
+ const message = JSON.stringify({ event, data, ...(id === undefined ? {} : { id }) });
+ this.dispatch('message', { data: message }, () => this.readyState === this.OPEN);
+ }
+
+ /** The frame a server opens every connection with, naming its incarnation.
+ * No `rooms`, so it says nothing about the declared set a test may be
+ * asserting on. */
+ announce(generation = 0): void {
+ this.frame('connection_state', { connection_id: 'conn-1', generation });
+ }
+
+ ping(): void {
+ this.frame('ping', {});
+ }
+
+ /** Refuse the open the way the server does: say why, then close with the status. */
+ refuse(status: number, detail: unknown): void {
+ this.open();
+ this.frame('refused', { status, detail });
+ this.drop(4000 + status);
+ }
+
+ /** The server closes the socket. */
+ drop(code = 1000, reason = ''): void {
+ this.end(code, reason);
+ }
+
+ /** The connection fails underneath both ends, before or after it opened. */
+ fail(): void {
+ this.dispatch('error', {}, () => this.readyState !== this.CLOSED);
+ this.end(1006, '');
+ }
+
+ private end(code: number, reason: string): void {
+ if (this.ending) return;
+ this.ending = true;
+ this.dispatch(
+ 'close',
+ { code, reason },
+ () => true,
+ () => {
+ this.readyState = this.CLOSED;
+ }
+ );
+ }
+
+ private dispatch(
+ type: string,
+ event: { data?: string; code?: number; reason?: string },
+ live: () => boolean,
+ apply?: () => void
+ ): void {
+ queueMicrotask(() => {
+ if (!live()) return;
+ apply?.();
+ for (const listener of [...(this.listeners.get(type) ?? [])]) listener(event);
+ });
+ }
}
-function encodeFrame(name: string, data: unknown): Uint8Array {
- return new TextEncoder().encode(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);
+function serve(script: Script): void {
+ server.script = script;
}
-/** The frame a server opens every stream with, naming the incarnation this
- * client is attached to. No `rooms`, so it says nothing about the declared set
- * a test may be asserting on. */
-function connected(generation = 0): Uint8Array {
- return encodeFrame('connection_state', { connection_id: 'conn-1', generation });
+function urls(): string[] {
+ return server.sockets.map((socket) => socket.url);
}
-/** A stream body carrying one frame and then closing, the way the server ends
- * a displaced stream. */
-function frameThenClose(name: string, data: unknown): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(encodeFrame(name, data));
- controller.close();
- },
- });
+function silentLog() {
+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
}
-/**
- * A connected stream body that stays open, so the loop neither reconnects nor
- * spins.
- *
- * It announces the connection first because the heartbeat waits for that: a
- * client that has not been told which incarnation it is cannot fence its own
- * tick, and a body that never announces is a connection that never beats.
- */
-function openForever(): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- },
- });
+/** A request on the side that the server answered. */
+function answer(status = 200, body = '') {
+ return { ok: status >= 200 && status < 300, status, text: async (): Promise<string> => body };
}
-/**
- * A connected stream body that ends when the client drops the socket.
- *
- * `openForever` cannot: its reader is parked on a read that never returns, so
- * a reopen is invisible to the test rather than absent. A body that honours
- * the request's signal is what a real socket does, and what a test asserting
- * on reopens needs.
- */
-function openUntilAborted(init: { signal: AbortSignal }): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- init.signal.addEventListener('abort', () => controller.close(), { once: true });
- },
- });
+function takenOver(message: string) {
+ return answer(409, JSON.stringify({ detail: { code: 'taken_over', message } }));
}
-/** The incarnation each beat carried, in order. */
-function beatGenerations(fetchMock: { mock: { calls: unknown[][] } }): (number | null)[] {
- return fetchMock.mock.calls
- .filter((call) => String(call[0]).includes('connection/beat'))
- .map(
- (call) =>
- (JSON.parse((call[1] as { body: string }).body) as { generation: number | null }).generation
- );
+function okFetch() {
+ return vi.fn(async (_url: string, _init?: { body?: string }) => answer());
}
-function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {
+function postsTo(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {
return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));
}
function makeStream(
- fetchMock: ReturnType<typeof vi.fn>,
- deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }
+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] },
+ fetchMock: ReturnType<typeof vi.fn> = okFetch()
) {
+ vi.stubGlobal('WebSocket', FakeSocket);
vi.stubGlobal('fetch', fetchMock);
const abort = new AbortController();
const log = silentLog();
@@ -118,51 +220,135 @@ function makeStream(
...deps,
});
stream.start();
- return { stream, abort, log };
+ return { stream, abort, log, fetchMock };
}
async function flush(times = 8): Promise<void> {
for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));
}
+beforeEach(() => {
+ server.sockets = [];
+ server.script = attach;
+});
+
afterEach(() => {
vi.unstubAllGlobals();
vi.useRealTimers();
});
+describe('the socket', () => {
+ it('carries the token as a bearer header', async () => {
+ const { abort } = makeStream({ rooms: [] });
+ await flush();
+
+ expect(server.sockets).toHaveLength(1);
+ expect(server.sockets[0]?.headers).toEqual({ Authorization: 'Bearer tok' });
+ abort.abort();
+ });
+
+ it('opens over wss for an https endpoint and ws for an http one', async () => {
+ const secure = makeStream({ rooms: [] });
+ await flush();
+ secure.abort.abort();
+ const plain = makeStream({
+ rooms: [],
+ creds: { ...creds, apiEndpoint: 'http://localhost:8000' },
+ });
+ await flush();
+ plain.abort.abort();
+
+ const [first, second] = urls();
+ expect(first).toMatch(/^wss:\/\/switch\.test\/agents\/agent-1\/connection\/ws\?/);
+ expect(second).toMatch(/^ws:\/\/localhost:8000\/agents\/agent-1\/connection\/ws\?/);
+ expect(new URL(first!).searchParams.get('connection_id')).toBe('conn-1');
+ });
+});
+
+describe('the heartbeat', () => {
+ it('answers a ping at once with a pong carrying the cursor', async () => {
+ serve((socket) => {
+ attach(socket, 0);
+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 3 }, 3);
+ });
+ const { abort } = makeStream({ rooms: ['room'] });
+ await flush();
+
+ server.sockets[0]!.ping();
+ await flush();
+
+ expect(server.sockets[0]!.sent).toEqual([{ type: 'pong', cursor: 3 }]);
+ abort.abort();
+ });
+
+ it('answers a ping while a slow handler is still running, without waiting for it', async () => {
+ // Handling an event can take as long as the agent likes. A pong queued
+ // behind it would have the server sweep a connection that is busy, not gone.
+ let finish: () => void = () => {};
+ const onEvent = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ finish = resolve;
+ })
+ );
+ serve((socket) => {
+ attach(socket, 0);
+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 7 }, 7);
+ });
+ const { abort } = makeStream({ rooms: ['room'], startCursor: 6, onEvent });
+ await flush();
+ expect(onEvent).toHaveBeenCalledOnce();
+
+ server.sockets[0]!.ping();
+ await flush();
+ // The cursor it names is the last event fully handled, not the one in hand.
+ expect(server.sockets[0]!.sent).toEqual([{ type: 'pong', cursor: 6 }]);
+
+ finish();
+ await flush();
+ server.sockets[0]!.ping();
+ await flush();
+ expect(server.sockets[0]!.sent).toEqual([
+ { type: 'pong', cursor: 6 },
+ { type: 'pong', cursor: 7 },
+ ]);
+ abort.abort();
+ });
+
+ it('makes no requests on the side to stay alive', async () => {
+ vi.useFakeTimers();
+ const { abort, fetchMock } = makeStream({ rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(60_000);
+
+ expect(fetchMock).not.toHaveBeenCalled();
+ expect(server.sockets).toHaveLength(1);
+ abort.abort();
+ });
+});
+
describe('a room the server refuses', () => {
- /** The refusal switch-core returns when a declared room no longer exists. */
- function roomGone(roomId: string) {
- return {
- ok: false,
- status: 403,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),
- };
- }
+ /** The refusal switch-core sends when a declared room no longer exists. */
+ const roomGone = (socket: FakeSocket, roomId: string): void =>
+ socket.refuse(403, `Room not found: ${roomId}`);
it('is dropped and not declared again', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events')) {
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- return String(url).includes('rooms=')
- ? roomGone('room-dead')
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
+ serve((socket) =>
+ socket.params.has('rooms') ? roomGone(socket, 'room-dead') : attach(socket, 0)
+ );
const rejected: { roomId: string; status: number }[] = [];
const reported: string[][] = [];
- const { abort, log } = makeStream(fetchMock, {
+ const { abort, log } = makeStream({
rooms: ['room-dead'],
onRooms: (rooms) => reported.push(rooms),
onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),
});
await flush();
- const opens = urlsFor(fetchMock, '/events');
+ const opens = server.sockets.map((socket) => socket.params);
expect(opens).toHaveLength(2);
- expect(opens[0]).toContain('rooms=room-dead');
- expect(opens[1]).not.toContain('rooms=');
+ expect(opens[0]?.get('rooms')).toBe('room-dead');
+ expect(opens[1]?.has('rooms')).toBe(false);
expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);
expect(reported).toEqual([[]]);
expect(log.error).toHaveBeenCalled();
@@ -170,689 +356,159 @@ describe('a room the server refuses', () => {
});
it('keeps the rooms the refusal does not name', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- const u = String(url);
- if (!u.includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return u.includes('room-dead')
- ? roomGone('room-dead')
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });
+ serve((socket) =>
+ socket.params.get('rooms')?.includes('room-dead')
+ ? roomGone(socket, 'room-dead')
+ : attach(socket, 0)
+ );
+ const { abort } = makeStream({ rooms: ['room-dead', 'room-live'] });
await flush();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(2);
- expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');
+ expect(server.sockets).toHaveLength(2);
+ expect(server.sockets[1]?.params.get('rooms')).toBe('room-live');
abort.abort();
});
- it('stays a transport error when the body names no declared room', async () => {
+ it('stays a transport error when the refusal names no declared room', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events')) {
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- return {
- ok: false,
- status: 404,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail: 'No such connection' }),
- };
- });
+ serve((socket) => socket.refuse(404, 'No such connection'));
const rejected: string[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort } = makeStream({
rooms: ['room-live'],
onRoomRejected: ({ roomId }) => rejected.push(roomId),
});
await vi.advanceTimersByTimeAsync(0);
- // One open, then the backoff — not an immediate retry, and nothing dropped.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ // One open, then the backoff: not an immediate retry, and nothing dropped.
+ expect(server.sockets).toHaveLength(1);
expect(rejected).toEqual([]);
abort.abort();
});
});
describe('credentials the server rejects', () => {
- function refusal(status: number, detail: string) {
- return {
- ok: false,
- status,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail }),
- };
- }
-
/** Open once, be refused, and let a long while pass. */
- async function afterRefusal(status: number, detail: string) {
+ async function afterRefusal(refuse: Script) {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? refusal(status, detail)
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
+ serve(refuse);
const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
+ const { abort, log, fetchMock } = makeStream({
rooms: ['room-live'],
onEvicted: (eviction) => evicted.push(eviction),
});
- await vi.advanceTimersByTimeAsync(0);
- const settled = fetchMock.mock.calls.length;
await vi.advanceTimersByTimeAsync(5 * 60_000);
abort.abort();
- return { fetchMock, evicted, log, settled };
+ return { evicted, log, fetchMock };
}
- it('ends the stream and the heartbeat on a 401, and says so once', async () => {
- const { fetchMock, evicted, log, settled } = await afterRefusal(401, 'Invalid agent token');
+ it('ends the stream when the socket is closed for its token, and says so once', async () => {
+ // An auth failure closes with 4401 and no `refused` frame before it.
+ const { evicted, log, fetchMock } = await afterRefusal((socket) => {
+ socket.open();
+ socket.drop(4401, 'Invalid agent token');
+ });
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- // Nothing beyond the beat already in flight when the refusal landed.
- expect(fetchMock.mock.calls).toHaveLength(settled);
+ expect(server.sockets).toHaveLength(1);
+ expect(fetchMock).not.toHaveBeenCalled();
expect(evicted).toHaveLength(1);
expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
expect(evicted[0]?.reason).toContain('credentials');
expect(evicted[0]?.reason).toContain('401');
- expect(log.error).toHaveBeenCalled();
+ expect(evicted[0]?.reason).toContain('Invalid agent token');
+ expect(log.error).toHaveBeenCalledTimes(1);
});
it('ends the stream on a 403 that names no declared room', async () => {
- const { fetchMock, evicted, settled } = await afterRefusal(
- 403,
- 'Agent is not a member of this room'
+ const { evicted } = await afterRefusal((socket) =>
+ socket.refuse(403, 'Agent is not a member of this room')
);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(fetchMock.mock.calls).toHaveLength(settled);
+ expect(server.sockets).toHaveLength(1);
expect(evicted).toHaveLength(1);
expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
expect(evicted[0]?.reason).toContain('credentials');
expect(evicted[0]?.reason).toContain('403');
});
- it('ends the stream and the heartbeat on a 401 heartbeat, and says so once', async () => {
+ it.each([503, 429])('keeps reconnecting after a refusal with status %i', async (status) => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
- : refusal(401, 'Invalid agent token')
+ serve((socket, index) =>
+ index === 0 ? socket.refuse(status, 'try later') : attach(socket, 0)
);
const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onEvicted: (eviction) => evicted.push(eviction),
- });
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS + 1);
- const settled = fetchMock.mock.calls.length;
- await vi.advanceTimersByTimeAsync(5 * 60_000);
- abort.abort();
-
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(fetchMock.mock.calls).toHaveLength(settled);
- expect(evicted).toHaveLength(1);
- expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
- expect(evicted[0]?.reason).toContain('credentials');
- expect(evicted[0]?.reason).toContain('401');
- expect(log.error).toHaveBeenCalledTimes(1);
- });
-
- it('tells a session start to the room itself only where the server says it leaves that to it', async () => {
- for (const [said, expected] of [
- [{ announce_session_starts: true }, true],
- [{}, false],
- ] as const) {
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('connection_state', {
- connection_id: 'conn-1',
- generation: 0,
- ...said,
- }),
- text: async (): Promise<string> => '',
- }
- : new Promise<never>(() => {})
- );
- let connected = false;
- const { stream, abort } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onConnected: () => {
- connected = true;
- },
- });
- expect(stream.announcesSessionStarts).toBe(false);
- for (let i = 0; i < 50 && !connected; i += 1) await flush();
- expect(stream.announcesSessionStarts).toBe(expected);
- abort.abort();
- vi.unstubAllGlobals();
- }
- });
-
- it('tells the owner once when the stream and the heartbeat are refused together', async () => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events')) {
- // Connected once — the heartbeat has nothing to beat for until then —
- // and refused on every reopen after that.
- opens += 1;
- if (opens === 1) {
- return {
- ok: true,
- status: 200,
- body: frameThenClose('connection_state', { connection_id: 'conn-1', generation: 0 }),
- text: async (): Promise<string> => '',
- };
- }
- await new Promise((r) => setTimeout(r, BEAT_INTERVAL_MS));
- return refusal(401, 'Invalid agent token');
- }
- // Slow enough to still be in flight when the reopen is refused: both
- // doors closing at once is the case this test is about.
- await new Promise((r) => setTimeout(r, 2 * BEAT_INTERVAL_MS));
- return refusal(401, 'Invalid agent token');
- });
- const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onEvicted: (eviction) => evicted.push(eviction),
- });
- await vi.advanceTimersByTimeAsync(5 * 60_000);
- abort.abort();
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(evicted).toHaveLength(1);
- expect(log.error).toHaveBeenCalledTimes(1);
- });
-
- it.each([503, 429])('keeps reconnecting after HTTP %i', async (status) => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- opens += 1;
- return opens === 1
- ? { ok: false, status, body: null, text: async (): Promise<string> => 'try later' }
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort } = makeStream({
rooms: [],
onEvicted: (eviction) => evicted.push(eviction),
});
await vi.advanceTimersByTimeAsync(2000);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(server.sockets).toHaveLength(2);
expect(evicted).toEqual([]);
abort.abort();
});
it('keeps reconnecting after a network error', async () => {
vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- opens += 1;
- if (opens === 1) throw new TypeError('fetch failed');
- return { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
+ serve((socket, index) => (index === 0 ? socket.fail() : attach(socket, 0)));
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const disconnected: string[] = [];
+ const { abort } = makeStream({
rooms: [],
onEvicted: (eviction) => evicted.push(eviction),
+ onDisconnected: ({ error }) => disconnected.push(error),
});
await vi.advanceTimersByTimeAsync(2000);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(server.sockets).toHaveLength(2);
expect(evicted).toEqual([]);
+ expect(disconnected).toHaveLength(1);
+ expect(disconnected[0]).toContain('could not connect');
abort.abort();
});
});
-describe('the heartbeat', () => {
- /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */
- async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {
+describe('a server restart', () => {
+ it('comes back within a second of the server, not on the doubling backoff', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- return { ok: false, status, text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
- await vi.advanceTimersByTimeAsync(windowMs);
- const beats = urlsFor(fetchMock, 'connection/beat').length;
- abort.abort();
- return beats;
- }
-
- it('backs off when the connection is rejected, instead of beating at full rate', async () => {
- const windowMs = 20 * BEAT_INTERVAL_MS;
- // At the base cadence this window holds ~20 beats. Doubling from the base
- // gives 4s, 8s, 16s… — four requests in the same window.
- expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);
- expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);
- });
-
- it('keeps backing off the longer the rejection lasts', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(30_000);
- const early = urlsFor(fetchMock, 'connection/beat').length;
- await vi.advanceTimersByTimeAsync(30_000);
- const late = urlsFor(fetchMock, 'connection/beat').length - early;
-
- expect(late).toBeLessThan(early);
- abort.abort();
- });
-
- it('sends nothing until the server has said which incarnation it is', async () => {
- vi.useFakeTimers();
- const server = { announce: (): void => {} };
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- server.announce = () => controller.enqueue(connected(7));
- },
- }),
- text: async (): Promise<string> => '',
- };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
- // An unfenced tick is what a displaced client sends, and the server refuses
- // it. Beating before the first frame would make every healthy connection
- // open with one.
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(0);
-
- server.announce();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
-
- const beats = fetchMock.mock.calls.filter((c) => String(c[0]).includes('connection/beat'));
- expect(beats.length).toBeGreaterThan(0);
- // And it carries the incarnation that frame named, so the server can fence it.
- const [, sent] = beats[0] as unknown as [string, { body: string }];
- expect(JSON.parse(sent.body).generation).toBe(7);
- abort.abort();
- });
-
- it('waits for the new incarnation after its own reconnect, instead of standing down', async () => {
- vi.useFakeTimers();
- // A server that fences: every attach is a new incarnation, and a tick
- // naming an older one is refused the way a displaced client's is.
- const server = { generation: 0, announce: (): void => {}, drop: (): void => {} };
- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
- if (String(url).includes('/events')) {
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- const announce = (): void => controller.enqueue(connected(attached));
- // The first attach is announced at once; the second is held, so
- // the beat falls due inside the window this test is about.
- if (attached === 1) announce();
- else server.announce = announce;
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
+ // Open, then the server restarts: it closes with 1012, refuses three
+ // attempts while it is down, then takes the fifth.
+ serve((socket, index) => {
+ if (index === 0) {
+ attach(socket, 0);
+ queueMicrotask(() => socket.drop(1012, 'service restart'));
+ } else if (index < 4) {
+ socket.fail();
+ } else {
+ attach(socket, 0);
}
- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
- if (sent === server.generation)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
- };
});
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
-
- await vi.advanceTimersByTimeAsync(2 * BEAT_INTERVAL_MS);
- const first = beatGenerations(fetchMock);
- expect(first.length).toBeGreaterThan(0);
- expect(first.every((generation) => generation === 1)).toBe(true);
-
- server.drop();
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
- // Nothing went out while the reopen was in flight: the incarnation we hold
- // is the one before it, and a tick carrying that is a stand-down.
- expect(beatGenerations(fetchMock)).toHaveLength(first.length);
- expect(evicted).toEqual([]);
-
- server.announce();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- const resumed = beatGenerations(fetchMock).slice(first.length);
- expect(resumed.length).toBeGreaterThan(0);
- expect(resumed.every((generation) => generation === 2)).toBe(true);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-
- it('does not stand down over a beat that was already in flight when it reopened', async () => {
- vi.useFakeTimers();
- // The gate stops a beat *starting* inside the reattach window. It cannot
- // recall one that left before the gate shut, and that beat is answered
- // against the incarnation the reopen replaced — a truthful `taken_over`
- // about a takeover this client performed on itself.
- const server = {
- generation: 0,
- announce: (): void => {},
- drop: (): void => {},
- answerBeat: (): void => {},
- beatsHeld: 0,
- };
- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
- if (String(url).includes('/events')) {
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) !== server.generation) {
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
- }
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(attached));
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
- // Held rather than answered: the point of the test is an answer that
- // arrives after the connection has moved on beneath it.
- server.beatsHeld += 1;
- await new Promise<void>((resolve) => {
- server.answerBeat = resolve;
- });
- if (sent === server.generation)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
- };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
+ const { abort } = makeStream({ rooms: [] });
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- expect(server.beatsHeld).toBe(1);
- expect(beatGenerations(fetchMock)).toEqual([1]);
+ await vi.advanceTimersByTimeAsync(4000);
- // Reopen with that beat still outstanding. The bound expires, the open goes
- // ahead, and the server is at 2 by the time the old beat is answered.
- server.drop();
- await vi.advanceTimersByTimeAsync(BEAT_SETTLE_LIMIT_MS + 10 * BEAT_INTERVAL_MS);
- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(1);
-
- server.answerBeat();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
-
- // Undecidable, so inert. Standing down here stops a client that nothing
- // has taken anything from.
- expect(evicted).toEqual([]);
- const resumed = beatGenerations(fetchMock).slice(1);
- expect(resumed.length).toBeGreaterThan(0);
- expect(resumed.every((generation) => generation === 2)).toBe(true);
+ // On the doubling backoff the fifth attempt would come after 1+2+4+8 s.
+ expect(server.sockets).toHaveLength(5);
abort.abort();
});
- it('stands down when its reattach is refused, rather than trying again', async () => {
+ it('keeps the doubling backoff for an ending that is not a restart', async () => {
vi.useFakeTimers();
- // The other half: a client that missed its eviction and comes back. The
- // server refuses the reattach without disturbing the holder, and this is
- // the only thing left that can tell the loser it lost.
- const server = { generation: 0, drop: (): void => {} };
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null) {
- // Someone else attached while we were away, so our claim is stale.
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
+ serve((socket, index) => {
+ if (index === 0) {
+ attach(socket, 0);
+ queueMicrotask(() => socket.drop(1006));
+ } else {
+ socket.fail();
}
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(attached));
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- server.drop();
- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
-
- expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
- // And it stopped asking: retrying is how the pair trade the connection.
- const attempts = urlsFor(fetchMock, 'expected_generation=1').length;
- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(attempts);
- abort.abort();
- });
-
- it('returns to the base cadence once a beat lands', async () => {
- vi.useFakeTimers();
- let reject = true;
- // A socket that ends when the client drops it: each refusal reopens, and
- // the heartbeat is held from the reopen until the new stream is named.
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- };
- if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
});
- const { abort } = makeStream(fetchMock, { rooms: [] });
+ const { abort } = makeStream({ rooms: [] });
- await vi.advanceTimersByTimeAsync(30_000);
- reject = false;
- await vi.advanceTimersByTimeAsync(30_000);
- const recovered = urlsFor(fetchMock, 'connection/beat').length;
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+ await vi.advanceTimersByTimeAsync(4000);
- expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);
- abort.abort();
- });
-});
-
-describe('the heartbeat after a reconnect', () => {
- /**
- * A server that reattaches on every open, can drop the current stream, and
- * records when each stream was opened and each beat arrived. The server's
- * own rule is what these tests hold the client to: a new connection that
- * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.
- */
- const HEARTBEAT_TTL_MS = 6000;
-
- /** Either end may close a stream first; the second close is not an error here. */
- function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {
- try {
- controller.close();
- } catch {
- // Already closed by the other side.
- }
- }
-
- function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {
- const opens: number[] = [];
- const beats: number[] = [];
- let generation = 0;
- let current: ReadableStreamDefaultController<Uint8Array> | null = null;
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('/events')) {
- opens.push(Date.now());
- generation += 1;
- const gen = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- current = controller;
- controller.enqueue(connected(gen));
- init.signal.addEventListener('abort', () => closeQuietly(controller), {
- once: true,
- });
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- beats.push(Date.now());
- const reply = beatReply();
- if (reply === 'fail') throw new TypeError('fetch failed');
- if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- /** The server ends the current stream, as it does a lapsed connection. */
- const drop = (): void => {
- if (current) closeQuietly(current);
- };
- return { fetchMock, opens, beats, drop };
- }
-
- it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {
- // The lockout: beats fail for a while, so the gap between them climbs to
- // its 30-second cap. The stream then reopens. The server starts its
- // 6-second clock on the new connection — and the client used to sit out
- // the rest of the 30 seconds before saying anything, lose the connection
- // for silence, and repeat that on every connection it opened afterwards.
- vi.useFakeTimers();
- let failing = true;
- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- // Long enough of failed beats for the gap to reach its cap.
- await vi.advanceTimersByTimeAsync(70_000);
- failing = false;
- server.drop();
- await vi.advanceTimersByTimeAsync(40_000);
-
- const reopenedAt = server.opens[1];
- expect(reopenedAt).toBeDefined();
- const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);
- expect(firstBeatAfter).toBeDefined();
- expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);
- abort.abort();
- });
-
- it('keeps beating at the base cadence on the new connection once it lands', async () => {
- vi.useFakeTimers();
- let failing = true;
- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(70_000);
- failing = false;
- server.drop();
- await vi.advanceTimersByTimeAsync(5_000);
- const settled = server.beats.length;
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
-
- expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);
- abort.abort();
- });
-
- it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {
- // The brake the old ordering provided, which beating promptly on every
- // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The
- // wait now sits before each reopen and doubles, so a refusing server sees
- // a handful of reopens, not one every couple of seconds per agent.
- vi.useFakeTimers();
- const server = reattachingServer(() => 404);
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(60_000);
-
- // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.
- expect(server.opens.length).toBeLessThanOrEqual(6);
- abort.abort();
- });
-
- it('does not throw away a connection the stream reopened while it waited', async () => {
- // A lapsed connection's socket is closed by the server, and the stream
- // loop reopens it by itself. If that lands during the heartbeat's wait
- // after a refusal, reopening again would discard a healthy connection.
- vi.useFakeTimers();
- let refuse = true;
- const server = reattachingServer(() => (refuse ? 404 : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- expect(server.beats.length).toBe(1);
- refuse = false;
- // The server ends the stream now, inside the heartbeat's wait.
- server.drop();
- await vi.advanceTimersByTimeAsync(20_000);
-
- // One open for the start, one for the stream's own reopen — not a third
- // from the heartbeat discarding it.
- expect(server.opens.length).toBe(2);
+ // Attempts at 1 s and 3 s: the next is not due until 7 s.
+ expect(server.sockets).toHaveLength(3);
abort.abort();
});
});
@@ -865,137 +521,98 @@ describe('reopening while an open is still in flight', () => {
* The next open can be held unanswered, the way a slow connection holds it.
*/
function incarnationServer() {
- const state = {
- generation: 0,
- opens: 0,
- beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',
- subscribeDrops: false,
- holdNext: false,
- release: (): void => {},
- drop: (): void => {},
- };
- const body = (gen: number, signal: AbortSignal) =>
- new ReadableStream<Uint8Array>({
- start(controller) {
- const close = (): void => {
- try {
- controller.close();
- } catch {
- // already closed
- }
- };
- state.drop = close;
- controller.enqueue(connected(gen));
- signal.addEventListener('abort', close, { once: true });
- },
- });
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('connection/subscribe')) {
- if (!state.subscribeDrops)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- state.subscribeDrops = false;
- // The stream ends while the claim is still in flight.
- state.holdNext = true;
- state.drop();
- await new Promise((r) => setTimeout(r, 3_000));
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- if (!String(url).includes('/events')) {
- if (state.beat === 'slow-404-and-drop') {
- // The stream ends while this beat is in flight, and its answer comes
- // back just after the client has started reopening.
- state.beat = 'ok';
- state.holdNext = true;
- state.drop();
- await new Promise((r) => setTimeout(r, 1_200));
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- }
- if (state.beat === 404)
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ const state = { generation: 0, holdNext: false, release: (): void => {} };
+ serve((socket) => {
+ const claimed = socket.params.get('expected_generation');
+ if (claimed !== null && Number(claimed) < state.generation) {
+ socket.refuse(409, { code: 'taken_over', message: 'reattach refused' });
+ return;
}
- state.opens += 1;
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) < state.generation)
- return {
- ok: false,
- status: 409,
- body: null,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
// The server acts on the open the moment it arrives.
state.generation += 1;
- const gen = state.generation;
- const answer = {
- ok: true,
- status: 200,
- body: body(gen, init.signal),
- text: async (): Promise<string> => '',
+ const generation = state.generation;
+ const accept = (): void => {
+ socket.open();
+ socket.announce(generation);
};
- if (!state.holdNext) return answer;
+ if (!state.holdNext) return accept();
state.holdNext = false;
- return new Promise((resolve, reject) => {
- state.release = () => resolve(answer);
- init.signal.addEventListener(
- 'abort',
- () => reject(new DOMException('aborted', 'AbortError')),
- { once: true }
- );
- });
+ state.release = accept;
});
- return { state, fetchMock };
+ return state;
}
it('does not cancel an open the server has already acted on, and so does not stand down', async () => {
// The self-takeover seen on the pilot. The server closes the stream, the
- // client starts reopening, and a beat sent just before comes back 404.
- // Reopening in answer used to cancel the open already in flight — which
- // the server had acted on, moving the connection to a new incarnation.
- // The client never read that answer, reattached claiming the old one, was
- // refused as a takeover, and stood down for good. The "other client" was
- // its own cancelled request.
+ // client starts reopening, and something asks for another reopen while
+ // that open is unanswered. Cancelling the open in flight does not undo it
+ // server-side: the client never reads the incarnation it made, reattaches
+ // claiming the old one, is refused as a takeover, and stands down for
+ // good. The "other client" was its own cancelled request.
vi.useFakeTimers();
- const { state, fetchMock } = incarnationServer();
+ const state = incarnationServer();
const onEvicted = vi.fn();
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });
-
- await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);
+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true, onEvicted });
+ await vi.advanceTimersByTimeAsync(0);
expect(state.generation).toBe(1);
- state.beat = 'slow-404-and-drop';
- // Long enough for the refused beat's wait to end while the open is held.
+ state.holdNext = true;
+ server.sockets[0]!.drop();
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(server.sockets).toHaveLength(2);
+
+ stream.setSpawnCapable(false);
await vi.advanceTimersByTimeAsync(12_000);
+ // Left to be answered rather than cancelled.
+ expect(server.sockets).toHaveLength(2);
+ expect(server.sockets[1]!.closedByClient).toBeNull();
+
state.release();
- await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);
+ await vi.advanceTimersByTimeAsync(1_000);
expect(onEvicted).not.toHaveBeenCalled();
- expect(state.opens).toBe(2);
- expect(beatGenerations(fetchMock).at(-1)).toBe(2);
+ expect(server.sockets).toHaveLength(3);
+ const carried = server.sockets[2]!.params;
+ expect(carried.get('expected_generation')).toBe('2');
+ expect(carried.has('spawn_capable')).toBe(false);
+ expect(state.generation).toBe(3);
abort.abort();
});
it('carries out a repoint asked for mid-open once that open is answered', async () => {
// The other way into the same hole: a room change reopens to include the
- // new room. Mid-open, it must wait for the answer rather than cancel it —
- // and must still happen, since the open in flight was built without it.
+ // new room. Mid-open, it must wait for the answer rather than cancel it,
+ // and must still happen, since the open in flight may have been built
+ // without it.
vi.useFakeTimers();
- const { state, fetchMock } = incarnationServer();
+ const state = incarnationServer();
+ let subscribeDrops = true;
+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) => {
+ if (url.includes('connection/subscribe') && subscribeDrops) {
+ subscribeDrops = false;
+ // The stream ends while the claim is still in flight.
+ state.holdNext = true;
+ server.sockets.at(-1)!.drop();
+ await new Promise((r) => setTimeout(r, 3_000));
+ }
+ return answer();
+ });
const onEvicted = vi.fn();
- const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ const { stream, abort } = makeStream({ rooms: ['room-a'], onEvicted }, fetchMock);
+ await vi.advanceTimersByTimeAsync(0);
- state.subscribeDrops = true;
const repointing = stream.repoint('room-b');
// The claim returns while the stream's reopen is held unanswered.
await vi.advanceTimersByTimeAsync(5_000);
+ expect(server.sockets).toHaveLength(2);
+ expect(server.sockets[1]!.closedByClient).toBeNull();
state.release();
await vi.advanceTimersByTimeAsync(5_000);
await repointing;
expect(onEvicted).not.toHaveBeenCalled();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens.at(-1)).toContain('rooms=room-b');
+ expect(server.sockets.at(-1)!.params.get('rooms')).toBe('room-b');
+ expect(server.sockets.at(-1)!.params.get('expected_generation')).toBe('2');
abort.abort();
});
});
@@ -1006,37 +623,19 @@ describe('a connection another client takes over', () => {
// is too late to protect the winner: by the time it refuses, a displaced
// client has already rewritten the winner's rooms. The claim carries the
// incarnation for that reason, and a refusal ends this client.
- const opens: string[] = [];
const subscribes: (number | null)[] = [];
- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
- if (String(url).includes('/events')) {
- opens.push(String(url));
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- }
- if (String(url).includes('connection/subscribe')) {
- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
- subscribes.push(sent.generation);
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'not your connection' } }),
- };
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
+ if (!url.includes('connection/subscribe')) return answer();
+ subscribes.push((JSON.parse(init?.body ?? '{}') as { generation: number | null }).generation);
+ return takenOver('not your connection');
});
const evicted: Eviction[] = [];
- const { stream, abort } = makeStream(fetchMock, {
- rooms: ['!old'],
- onEvicted: (e) => evicted.push(e),
- });
+ const { stream, abort } = makeStream(
+ { rooms: ['!old'], onEvicted: (e) => evicted.push(e) },
+ fetchMock
+ );
await flush();
- const opensBefore = opens.length;
+ const opensBefore = server.sockets.length;
await stream.repoint('!new');
await flush();
@@ -1045,38 +644,25 @@ describe('a connection another client takes over', () => {
expect(subscribes).toEqual([0]);
expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
// No reopen: coming back would be the takeover the refusal just denied.
- expect(opens.length).toBe(opensBefore);
+ expect(server.sockets).toHaveLength(opensBefore);
abort.abort();
});
it('does not claim a room before the frame that names its incarnation', async () => {
// Claiming nothing is how a client too old to have an incarnation gets
// through, so a claim sent in the window before the first frame would go
- // through the same door — and that window is exactly when this client may
+ // through the same door, and that window is exactly when this client may
// already have been displaced without knowing it.
- let announce: () => void = () => {};
+ serve((socket) => socket.open());
const subscribes: (number | null)[] = [];
- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- // Attached, but silent: the server has not said which incarnation
- // this socket is yet.
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- announce = () => controller.enqueue(connected());
- },
- }),
- text: async (): Promise<string> => '',
- };
- if (String(url).includes('connection/subscribe')) {
- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
- subscribes.push(sent.generation);
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
+ if (url.includes('connection/subscribe'))
+ subscribes.push(
+ (JSON.parse(init?.body ?? '{}') as { generation: number | null }).generation
+ );
+ return answer();
});
- const { stream, abort } = makeStream(fetchMock, { rooms: ['!old'] });
+ const { stream, abort } = makeStream({ rooms: ['!old'] }, fetchMock);
await flush();
const repointed = stream.repoint('!new');
@@ -1084,7 +670,7 @@ describe('a connection another client takes over', () => {
expect(subscribes).toEqual([]);
- announce();
+ server.sockets[0]!.announce();
await repointed;
expect(subscribes).toEqual([0]);
@@ -1093,22 +679,17 @@ describe('a connection another client takes over', () => {
it('halts instead of reopening, because reopening would be a takeover back', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('evicted', {
- code: 'taken_over',
- reason: 'another stream attached to this connection',
- room_id: null,
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
+ serve((socket) => {
+ socket.open();
+ socket.frame('evicted', {
+ code: 'taken_over',
+ reason: 'another stream attached to this connection',
+ room_id: null,
+ });
+ socket.drop();
+ });
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort } = makeStream({
rooms: [],
onEvicted: (eviction) => evicted.push(eviction),
});
@@ -1117,7 +698,7 @@ describe('a connection another client takes over', () => {
// Two clients reopening each other's connection is the loop this prevents:
// whoever reopens wins, so a displaced client that reopens starts it again.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(server.sockets).toHaveLength(1);
expect(evicted).toEqual([
{
code: EVICTION_TAKEN_OVER,
@@ -1128,61 +709,43 @@ describe('a connection another client takes over', () => {
abort.abort();
});
- it('stands down when its heartbeat is refused, rather than taking the connection back', async () => {
+ it('stands down when its reattach is refused, rather than trying again', async () => {
+ // A client that missed its eviction and comes back. The server refuses
+ // the reattach without disturbing the holder, and this is the only thing
+ // left that can tell the loser it lost.
vi.useFakeTimers();
- // The case the SSE frame cannot cover: this client's socket dropped before
- // the eviction reached it, so the refused heartbeat is the only way it ever
- // hears that it lost.
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
- : {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({
- detail: {
- code: 'taken_over',
- message: 'connection conn-1 was reopened since incarnation 0',
- },
- }),
- }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
+ let generation = 0;
+ serve((socket) => {
+ if (socket.params.has('expected_generation')) {
+ // Someone else attached while we were away, so our claim is stale.
+ socket.refuse(409, { code: 'taken_over', message: 'reattach refused' });
+ return;
+ }
+ generation += 1;
+ socket.open();
+ socket.announce(generation);
});
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream({ rooms: [], onEvicted: (e) => evicted.push(e) });
- await vi.advanceTimersByTimeAsync(5 * 60_000);
+ await vi.advanceTimersByTimeAsync(0);
+ server.sockets[0]!.drop();
+ await vi.advanceTimersByTimeAsync(30_000);
- // One beat and no second open: reopening is a takeover, so a loser that
- // reopens pulls the connection straight back off the client that won it.
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
+ expect(server.sockets).toHaveLength(2);
+ expect(server.sockets[1]!.params.get('expected_generation')).toBe('1');
+ // And it stopped asking: retrying is how the pair trade the connection.
+ await vi.advanceTimersByTimeAsync(5 * 60_000);
+ expect(server.sockets).toHaveLength(2);
abort.abort();
});
it('still reopens on a refusal that names no code, the way an old server sends it', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- }
- : {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: 'connection conn-1 has no stream attached' }),
- }
- );
+ serve((socket) => socket.refuse(409, 'connection conn-1 has no stream attached'));
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort } = makeStream({
rooms: [],
onEvicted: (eviction) => evicted.push(eviction),
});
@@ -1191,27 +754,20 @@ describe('a connection another client takes over', () => {
// Only the takeover is terminal. Everything else a 409 can mean is still
// something a reopen fixes, and a server too old to say which is one of them.
- expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);
+ expect(server.sockets.length).toBeGreaterThan(1);
expect(evicted).toEqual([]);
abort.abort();
});
it('reads an old server’s wording as a takeover when it sends no code', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('evicted', {
- reason: 'another stream attached to this connection',
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
+ serve((socket) => {
+ socket.open();
+ socket.frame('evicted', { reason: 'another stream attached to this connection' });
+ socket.drop();
+ });
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort } = makeStream({
rooms: [],
onEvicted: (eviction) => evicted.push(eviction),
});
@@ -1219,200 +775,190 @@ describe('a connection another client takes over', () => {
await vi.advanceTimersByTimeAsync(5 * 60_000);
expect(evicted[0]?.code).toBe(EVICTION_TAKEN_OVER);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(server.sockets).toHaveLength(1);
abort.abort();
});
});
describe('a stream the server closes cleanly', () => {
- /** Opens, closes at once, forever — the shape of a contested connection. */
- function closesAtOnce() {
- return vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- }
+ /** Opens, closes at once, forever: the shape of a contested connection. */
+ const closesAtOnce: Script = (socket) => {
+ socket.open();
+ socket.drop();
+ };
it('waits before reopening rather than reconnecting at full rate', async () => {
vi.useFakeTimers();
- const fetchMock = closesAtOnce();
- const { abort } = makeStream(fetchMock, { rooms: [] });
+ serve(closesAtOnce);
+ const { abort } = makeStream({ rooms: [] });
// A minute of 1s, 2s, 4s… is seven opens. Without pacing a clean close it
// is an unbounded spin, limited only by how fast the server can answer.
await vi.advanceTimersByTimeAsync(60_000);
- expect(urlsFor(fetchMock, '/events').length).toBeLessThanOrEqual(8);
+ expect(server.sockets.length).toBeLessThanOrEqual(8);
abort.abort();
});
it('keeps backing off rather than resetting on every handshake', async () => {
vi.useFakeTimers();
- const fetchMock = closesAtOnce();
- const { abort } = makeStream(fetchMock, { rooms: [] });
+ serve(closesAtOnce);
+ const { abort } = makeStream({ rooms: [] });
await vi.advanceTimersByTimeAsync(60_000);
- const early = urlsFor(fetchMock, '/events').length;
+ const early = server.sockets.length;
await vi.advanceTimersByTimeAsync(60_000);
- const late = urlsFor(fetchMock, '/events').length - early;
+ const late = server.sockets.length - early;
// An open is not evidence of a working stream. Resetting the curve on the
// handshake held two contending clients at a reconnect a second forever.
expect(late).toBeLessThan(early);
abort.abort();
});
-});
-it('does not acknowledge a delivery before the consumer has saved it', async () => {
- let finish: () => void = () => {};
- const saved = new Promise<void>((resolve) => {
- finish = resolve;
- });
- const onEvent = vi.fn(() => saved);
- const fetchMock = vi.fn(async (url: string) => {
- if (!url.includes('/events')) return Response.json({});
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(
- new TextEncoder().encode(
- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"room","sequence":7}\n\n'
- )
- );
- },
- })
- );
- });
- const { stream, abort } = makeStream(fetchMock, { rooms: ['room'], startCursor: 6, onEvent });
- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());
- expect(stream.position).toBe(6);
- finish();
- await vi.waitFor(() => expect(stream.position).toBe(7));
- abort.abort();
-});
+ it('starts the backoff over once a stream has lasted', async () => {
+ vi.useFakeTimers();
+ const openedAt: number[] = [];
+ serve((socket, index) => {
+ openedAt.push(Date.now());
+ // Four that close at once climb the curve; the fifth stays up.
+ if (index < 4) closesAtOnce(socket, index);
+ else attach(socket, index);
+ });
+ const { abort } = makeStream({ rooms: [] });
-it('replays from an explicitly saved zero cursor instead of starting at head', async () => {
- const fetchMock = vi
- .fn()
- .mockImplementation(async (url: string) =>
- url.includes('/events?')
- ? { ok: true, body: openForever() }
- : { ok: true, json: async () => ({ rooms: [] }) }
- );
- const { abort } = makeStream(fetchMock, { rooms: [], startCursor: 0 });
- try {
- await flush();
- expect(new URL(urlsFor(fetchMock, '/events?')[0]).searchParams.get('start_from')).toBe('0');
- } finally {
- abort.abort();
- }
-});
+ // Opens at 0, 1, 3, 7 and 15 seconds.
+ await vi.advanceTimersByTimeAsync(16_000);
+ expect(server.sockets).toHaveLength(5);
-function resetFrames(url: string): Response {
- if (!url.includes('/events')) return Response.json({});
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(
- new TextEncoder().encode(
- 'event: gap\ndata: {"from_sequence":0,"resumed_at":0,"reason":"buffer reset"}\n\n' +
- 'id: 1\nevent: message\ndata: {"type":"message","room_id":"room","sequence":1}\n\n'
- )
- );
- },
- })
- );
-}
+ // Long enough to count as a working stream, then the server ends it.
+ await vi.advanceTimersByTimeAsync(31_000);
+ const closedAt = Date.now();
+ server.sockets[4]!.drop();
+ await vi.advanceTimersByTimeAsync(2_000);
-it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {
- let finish!: () => void;
- const saved = new Promise<void>((resolve) => {
- finish = resolve;
+ // Back at the first step of the curve, not the 16 seconds it had reached.
+ expect(server.sockets).toHaveLength(6);
+ expect(openedAt[5]! - closedAt).toBeLessThanOrEqual(1_000);
+ abort.abort();
});
- const onGap = vi.fn(() => saved);
- const onEvent = vi.fn();
- const { stream, abort } = makeStream(
- vi.fn(async (url: string) => resetFrames(url)),
- { rooms: ['room'], startCursor: 4, onGap, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
- expect(onGap).toHaveBeenCalledWith({
- fromSequence: 0,
- resumedAt: 0,
- cursorReset: true,
- reason: 'buffer reset',
+});
+
+describe('the cursor', () => {
+ it('does not acknowledge a delivery before the consumer has saved it', async () => {
+ let finish: () => void = () => {};
+ const saved = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ const onEvent = vi.fn(() => saved);
+ serve((socket) => {
+ socket.open();
+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 7 }, 7);
});
- expect(stream.position).toBe(4);
- expect(onEvent).not.toHaveBeenCalled();
+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 6, onEvent });
+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());
+ expect(stream.position).toBe(6);
finish();
- await vi.waitFor(() => expect(stream.position).toBe(1));
- expect(onEvent).toHaveBeenCalledOnce();
- } finally {
+ await vi.waitFor(() => expect(stream.position).toBe(7));
abort.abort();
- }
-});
+ });
-it('does not advance or deliver when persisting the reset checkpoint fails', async () => {
- const onGap = vi.fn(async () => {
- throw new Error('journal unavailable');
+ it('replays from an explicitly saved zero cursor instead of starting at head', async () => {
+ const { abort } = makeStream({ rooms: [], startCursor: 0 });
+ try {
+ await flush();
+ expect(server.sockets[0]?.params.get('start_from')).toBe('0');
+ } finally {
+ abort.abort();
+ }
});
- const onEvent = vi.fn();
- const { stream, abort } = makeStream(
- vi.fn(async (url: string) => resetFrames(url)),
- { rooms: ['room'], startCursor: 4, onGap, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
- expect(stream.position).toBe(4);
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
+
+ it('starts at head when it has nothing saved, and resumes from its cursor on reconnect', async () => {
+ vi.useFakeTimers();
+ serve((socket, index) => {
+ attach(socket, index);
+ if (index === 0) {
+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 12 }, 12);
+ socket.drop();
+ }
+ });
+ const { abort } = makeStream({ rooms: ['room'] });
+
+ await vi.advanceTimersByTimeAsync(1_000);
+
+ expect(server.sockets.map((socket) => socket.params.get('start_from'))).toEqual(['head', '12']);
abort.abort();
- }
+ });
+
+ /** A buffer reset, then the first event after it. */
+ const reset: Script = (socket) => {
+ socket.open();
+ socket.frame('gap', { from_sequence: 0, resumed_at: 0, reason: 'buffer reset' });
+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 1 }, 1);
+ };
+
+ it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {
+ let finish!: () => void;
+ const saved = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ const onGap = vi.fn(() => saved);
+ const onEvent = vi.fn();
+ serve(reset);
+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 4, onGap, onEvent });
+ try {
+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
+ expect(onGap).toHaveBeenCalledWith({
+ fromSequence: 0,
+ resumedAt: 0,
+ cursorReset: true,
+ reason: 'buffer reset',
+ });
+ expect(stream.position).toBe(4);
+ expect(onEvent).not.toHaveBeenCalled();
+ finish();
+ await vi.waitFor(() => expect(stream.position).toBe(1));
+ expect(onEvent).toHaveBeenCalledOnce();
+ } finally {
+ abort.abort();
+ }
+ });
+
+ it('does not advance or deliver when persisting the reset checkpoint fails', async () => {
+ const onGap = vi.fn(async () => {
+ throw new Error('journal unavailable');
+ });
+ const onEvent = vi.fn();
+ serve(reset);
+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 4, onGap, onEvent });
+ try {
+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
+ expect(stream.position).toBe(4);
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+ });
});
describe('the permission to start a session', () => {
- function reopenable() {
- return vi.fn(async (url: string, init: { signal: AbortSignal }) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- }
-
it('is redeclared on the wire when it changes under a live connection', async () => {
// Turning automatic sessions off while the controller is connected has to
// reach the server, or it goes on promising a session nothing will start.
- const fetchMock = reopenable();
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });
await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(urlsFor(fetchMock, '/events')[0]).toContain('spawn_capable=true');
+ expect(server.sockets).toHaveLength(1);
+ expect(server.sockets[0]?.params.get('spawn_capable')).toBe('true');
stream.setSpawnCapable(false);
await flush();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(2);
- expect(opens[1]).not.toContain('spawn_capable');
+ expect(server.sockets).toHaveLength(2);
+ expect(server.sockets[0]?.closedByClient).toBe(1000);
+ expect(server.sockets[1]?.params.has('spawn_capable')).toBe(false);
// A reattach, not a takeover: the incarnation goes with it, so the server
// recognises the same client rather than evicting it in favour of itself.
- expect(opens[1]).toContain('expected_generation=0');
+ expect(server.sockets[1]?.params.get('expected_generation')).toBe('0');
abort.abort();
});
@@ -1426,86 +972,59 @@ describe('the permission to start a session', () => {
let generation = 0;
let opens = 0;
const announce: Array<() => void> = [];
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) !== generation)
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: EVICTION_TAKEN_OVER } }),
- };
+ serve((socket) => {
+ const claimed = socket.params.get('expected_generation');
+ if (claimed !== null && Number(claimed) !== generation) {
+ socket.refuse(409, { code: EVICTION_TAKEN_OVER });
+ return;
+ }
opens += 1;
generation += 1;
- const held = opens === holdStateOn;
const named = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- let closed = false;
- const send = () => {
- if (!closed) controller.enqueue(connected(named));
- };
- if (held) announce.push(send);
- else send();
- init.signal.addEventListener(
- 'abort',
- () => {
- closed = true;
- controller.close();
- },
- { once: true }
- );
- },
- }),
- text: async (): Promise<string> => '',
- };
+ socket.open();
+ if (opens === holdStateOn) announce.push(() => socket.announce(named));
+ else socket.announce(named);
});
- return { fetchMock, announce };
+ return { announce };
}
it('holds a second change until the server has answered the first', async () => {
// Redeclaring reattaches, which claims the incarnation this client believes
// it holds. A second change sent before the first open was answered claims
- // the one before it — which the server has moved past, and refuses as a
+ // the one before it, which the server has moved past and refuses as a
// takeover. That would take the agent's only connection off the air over
// two clicks in a row.
const onEvicted = vi.fn();
- const { fetchMock, announce } = fencingServer(2);
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true, onEvicted });
+ const { announce } = fencingServer(2);
+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true, onEvicted });
try {
await flush();
stream.setSpawnCapable(false);
await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(server.sockets).toHaveLength(2);
// The reopen is out but unanswered, so this one has nothing safe to claim.
stream.setSpawnCapable(true);
await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(server.sockets).toHaveLength(2);
- announce[0]();
+ announce[0]!();
await flush();
// Without the hold, this is where the connection has already gone: the
// third open claims the incarnation the second one replaced.
expect(onEvicted).not.toHaveBeenCalled();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(3);
- expect(opens[2]).toContain('spawn_capable=true');
- expect(opens[2]).toContain('expected_generation=2');
+ expect(server.sockets).toHaveLength(3);
+ expect(server.sockets[2]?.params.get('spawn_capable')).toBe('true');
+ expect(server.sockets[2]?.params.get('expected_generation')).toBe('2');
} finally {
abort.abort();
}
});
it('carries nothing when a change made in that window is taken back', async () => {
- const { fetchMock, announce } = fencingServer(2);
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ const { announce } = fencingServer(2);
+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });
try {
await flush();
stream.setSpawnCapable(false);
@@ -1514,262 +1033,278 @@ describe('the permission to start a session', () => {
stream.setSpawnCapable(false);
await flush();
- announce[0]();
+ announce[0]!();
await flush();
// The held open already declares what the setting settled back to.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
+ expect(server.sockets).toHaveLength(2);
} finally {
abort.abort();
}
});
it('does not reopen the socket when it is set to what it already is', async () => {
- const fetchMock = reopenable();
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });
await flush();
stream.setSpawnCapable(true);
await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(server.sockets).toHaveLength(1);
abort.abort();
});
});
-it('routes command wakeups separately from room events and their cursor', async () => {
- const onCommands = vi.fn();
- const onEvent = vi.fn();
- const fetchMock = vi.fn(async (url: string) =>
- url.includes('/events')
- ? new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- controller.enqueue(encodeFrame('session_commands', { session_ids: ['session'] }));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- )
- : Response.json({})
- );
- const { abort } = makeStream(fetchMock, { rooms: [], scope: 'all', onCommands, onEvent });
- try {
- await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
+describe('frames that are not room events', () => {
+ /** An attached connection that then sends these frames. */
+ function streaming(...frames: [string, unknown][]): void {
+ serve((socket) => {
+ attach(socket, 0);
+ for (const [event, data] of frames) socket.frame(event, data);
+ });
}
-});
-function streaming(...frames: Uint8Array[]) {
- return vi.fn(async (url: string) =>
- url.includes('/events')
- ? new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- for (const frame of frames) controller.enqueue(frame);
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- )
- : Response.json({})
- );
-}
+ it('routes command wakeups separately from room events and their cursor', async () => {
+ const onCommands = vi.fn();
+ const onEvent = vi.fn();
+ streaming(['session_commands', { session_ids: ['session'] }]);
+ const { stream, abort } = makeStream({ rooms: [], scope: 'all', onCommands, onEvent });
+ try {
+ await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));
+ expect(onEvent).not.toHaveBeenCalled();
+ expect(stream.position).toBe(0);
+ } finally {
+ abort.abort();
+ }
+ });
-it('hands an approval outcome to its own callback, not the room-event path', async () => {
- const onApprovalOutcome = vi.fn();
- const onEvent = vi.fn();
- const outcome = {
- session_id: 'session',
- request_id: 'permission',
- state: 'answered',
- answer: '0',
- answered_by: '@person:test',
- answered_at: '2026-09-24T12:00:00Z',
- };
- const { abort } = makeStream(streaming(encodeFrame('approval_outcome', outcome)), {
- rooms: [],
- scope: 'all',
- onApprovalOutcome,
- onEvent,
+ it('hands an approval outcome to its own callback, not the room-event path', async () => {
+ const onApprovalOutcome = vi.fn();
+ const onEvent = vi.fn();
+ const outcome = {
+ session_id: 'session',
+ request_id: 'permission',
+ state: 'answered',
+ answer: '0',
+ answered_by: '@person:test',
+ answered_at: '2026-09-24T12:00:00Z',
+ };
+ streaming(['approval_outcome', outcome]);
+ const { abort } = makeStream({ rooms: [], scope: 'all', onApprovalOutcome, onEvent });
+ try {
+ await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
});
- try {
- await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-it('drops a frame it does not know that is not a room event', async () => {
- const onEvent = vi.fn();
- const { abort } = makeStream(
+ it('drops a frame it does not know that is not a room event', async () => {
+ const onEvent = vi.fn();
streaming(
- encodeFrame('something_new', { detail: 'from a later server' }),
- encodeFrame('approval_outcome', { session_id: 'session' }),
- encodeFrame('message', { type: 'message', room_id: 'room', payload: {} })
- ),
- { rooms: [], scope: 'all', onEvent }
- );
- try {
- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));
- expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });
- } finally {
- abort.abort();
- }
-});
+ ['something_new', { detail: 'from a later server' }],
+ ['approval_outcome', { session_id: 'session' }],
+ ['message', { type: 'message', room_id: 'room', payload: {} }]
+ );
+ const { abort } = makeStream({ rooms: [], scope: 'all', onEvent });
+ try {
+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));
+ expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });
+ } finally {
+ abort.abort();
+ }
+ });
-it('hands a relayed session command to its own callback', async () => {
- const onSessionCommand = vi.fn();
- const onEvent = vi.fn();
- const command = {
- contractVersion: 1,
- commandId: 'command-1',
- sessionId: 'session',
- epoch: 'epoch',
- origin: { surface: 'console', actorId: 'owner', roomId: null, threadId: null, messageId: null },
- body: { type: 'turn.interrupt', turnId: 'turn' },
- };
- const { abort } = makeStream(
- streaming(
- encodeFrame('session_command', { commandId: 'no-session' }),
- encodeFrame('session_command', command)
- ),
- { rooms: [], scope: 'all', onSessionCommand, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));
- expect(onSessionCommand).toHaveBeenCalledTimes(1);
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
+ it('hands a relayed session command to its own callback', async () => {
+ const onSessionCommand = vi.fn();
+ const onEvent = vi.fn();
+ const command = {
+ contractVersion: 1,
+ commandId: 'command-1',
+ sessionId: 'session',
+ epoch: 'epoch',
+ origin: {
+ surface: 'console',
+ actorId: 'owner',
+ roomId: null,
+ threadId: null,
+ messageId: null,
+ },
+ body: { type: 'turn.interrupt', turnId: 'turn' },
+ };
+ streaming(['session_command', { commandId: 'no-session' }], ['session_command', command]);
+ const { abort } = makeStream({ rooms: [], scope: 'all', onSessionCommand, onEvent });
+ try {
+ await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));
+ expect(onSessionCommand).toHaveBeenCalledTimes(1);
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+ });
-it('hands a room another connection took over to its own callback', async () => {
- const onRoomReleased = vi.fn();
- const onEvent = vi.fn();
- const { abort } = makeStream(
+ it('hands a room another connection took over to its own callback', async () => {
+ const onRoomReleased = vi.fn();
+ const onEvent = vi.fn();
streaming(
- encodeFrame('room_released', { session_id: 'no-room' }),
- encodeFrame('room_released', { room_id: 'room', session_id: 'session' }),
- encodeFrame('room_released', { room_id: 'other', session_id: null })
- ),
- { rooms: [], scope: 'all', onRoomReleased, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));
- expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });
- expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-it('says when the server has confirmed an open', async () => {
- const onConnected = vi.fn();
- const { abort } = makeStream(streaming(), { rooms: [], scope: 'all', onConnected });
- try {
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
- } finally {
- abort.abort();
- }
+ ['room_released', { session_id: 'no-room' }],
+ ['room_released', { room_id: 'room', session_id: 'session' }],
+ ['room_released', { room_id: 'other', session_id: null }]
+ );
+ const { abort } = makeStream({ rooms: [], scope: 'all', onRoomReleased, onEvent });
+ try {
+ await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));
+ expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });
+ expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
+ }
+ });
});
-it('says when an open stream ended and it is trying again', async () => {
- const onConnected = vi.fn();
- const onDisconnected = vi.fn();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!url.includes('/events')) return Response.json({});
- opens += 1;
- if (opens === 1)
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- controller.close();
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- return new Response(openForever(), { headers: { 'content-type': 'text/event-stream' } });
+describe('the connection’s lifecycle', () => {
+ it('says when the server has confirmed an open', async () => {
+ const onConnected = vi.fn();
+ const { abort } = makeStream({ rooms: [], scope: 'all', onConnected });
+ try {
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
+ } finally {
+ abort.abort();
+ }
});
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onConnected,
- onDisconnected,
+
+ it('says when an open stream ended and it is trying again', async () => {
+ vi.useFakeTimers();
+ serve((socket, index) => {
+ attach(socket, index);
+ if (index === 0) socket.drop();
+ });
+ const onConnected = vi.fn();
+ const onDisconnected = vi.fn();
+ const { abort } = makeStream({ rooms: [], scope: 'all', onConnected, onDisconnected });
+ try {
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onDisconnected).toHaveBeenCalledTimes(1);
+ expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });
+ expect(onConnected).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(onConnected).toHaveBeenCalledTimes(2);
+ expect(onDisconnected).toHaveBeenCalledTimes(1);
+ } finally {
+ abort.abort();
+ }
});
- try {
- await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
- expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });
- expect(onConnected).toHaveBeenCalledTimes(1);
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
- expect(onDisconnected).toHaveBeenCalledTimes(1);
- } finally {
- abort.abort();
- }
});
-it('states placements on the attached incarnation, and raises on a refusal', async () => {
- const bodies: unknown[] = [];
- let refuse = false;
- const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
- if (url.includes('/events'))
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected(4));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- if (url.includes('connection/placements')) {
+describe('placements', () => {
+ it('states placements on the attached incarnation, and raises on a refusal', async () => {
+ serve((socket) => {
+ socket.open();
+ socket.announce(4);
+ });
+ const bodies: unknown[] = [];
+ let refuse = false;
+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
+ if (!url.includes('connection/placements')) return answer();
bodies.push(JSON.parse(init!.body!));
- return refuse ? new Response('not a member', { status: 403 }) : Response.json({ ok: true });
+ return refuse ? answer(403, 'not a member') : answer(200, '{"ok":true}');
+ });
+ const { stream, abort } = makeStream({ rooms: [], scope: 'all' }, fetchMock);
+ try {
+ await stream.replacePlacements({ session: 'room' });
+ expect(bodies).toEqual([
+ { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },
+ ]);
+ refuse = true;
+ await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');
+ } finally {
+ abort.abort();
+ }
+ });
+
+ it('does not give its connection up to itself over a request answered across its own reopen', async () => {
+ // Seen on a real host: placements stated under incarnation 4, the stream
+ // reopened (to 5) before the answer came, and the answer, "taken over:
+ // reopened since 4, now at 5", named this very client as the new holder.
+ // It stood down for good, with nobody else anywhere near the connection.
+ vi.useFakeTimers();
+ serve((socket, index) => {
+ socket.open();
+ socket.announce(4 + index);
+ });
+ let answerPlacements: (response: ReturnType<typeof answer>) => void = () => {};
+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) =>
+ url.includes('connection/placements')
+ ? new Promise<ReturnType<typeof answer>>((resolve) => {
+ answerPlacements = resolve;
+ })
+ : answer()
+ );
+ const evicted: Eviction[] = [];
+ const onConnected = vi.fn();
+ const { stream, abort } = makeStream(
+ {
+ rooms: [],
+ scope: 'all',
+ onConnected,
+ onEvicted: (eviction) => evicted.push(eviction),
+ },
+ fetchMock
+ );
+ try {
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onConnected).toHaveBeenCalledTimes(1);
+ const stating = stream.replacePlacements({ session: 'room' });
+ const settled = expect(stating).rejects.toThrow('409');
+ await vi.advanceTimersByTimeAsync(0);
+ expect(postsTo(fetchMock, 'connection/placements')).toHaveLength(1);
+
+ server.sockets[0]!.drop();
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(onConnected).toHaveBeenCalledTimes(2);
+ answerPlacements(
+ takenOver('connection conn-1 has been reopened since incarnation 4 and is now at 5')
+ );
+
+ await settled;
+ expect(evicted).toEqual([]);
+ } finally {
+ abort.abort();
+ }
+ });
+
+ it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {
+ serve((socket) => {
+ socket.open();
+ socket.announce(4);
+ });
+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) =>
+ url.includes('connection/placements') ? takenOver('another client holds it') : answer()
+ );
+ const evicted: Eviction[] = [];
+ const { stream, abort } = makeStream(
+ { rooms: [], scope: 'all', onEvicted: (eviction) => evicted.push(eviction) },
+ fetchMock
+ );
+ try {
+ await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');
+ expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);
+ } finally {
+ abort.abort();
}
- return Response.json({});
});
- const { stream, abort } = makeStream(fetchMock, { rooms: [], scope: 'all' });
- try {
- await stream.replacePlacements({ session: 'room' });
- expect(bodies).toEqual([
- { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },
- ]);
- refuse = true;
- await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');
- } finally {
- abort.abort();
- }
});
-describe('a hosted worker', () => {
+describe('a hosted worker over the socket', () => {
const worker = { capability: 'cap-1', bootId: 'boot-1', instanceId: 'i-1', stateVersion: 1 };
- function refused(status: number, code: string) {
- return {
- ok: false,
- status,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail: { code, message: code } }),
- };
- }
-
- it('states its capability and host on the open', async () => {
- const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- }));
- const { abort } = makeStream(fetchMock, { rooms: [], worker });
+ it('states its capability and host on the socket it opens', async () => {
+ const { abort } = makeStream({ rooms: [], worker });
await flush();
- const open = fetchMock.mock.calls.find((c) => String(c[0]).includes('/events'));
- expect(open?.[1]?.headers).toMatchObject({
+ expect(server.sockets[0]?.headers).toEqual({
+ Authorization: 'Bearer tok',
'X-Switch-Worker-Capability': 'cap-1',
'X-Switch-Host-Boot-Id': 'boot-1',
'X-Switch-Host-Instance-Id': 'i-1',
@@ -1778,353 +1313,84 @@ describe('a hosted worker', () => {
abort.abort();
});
+ it('sends no worker headers when it is not a worker', async () => {
+ const { abort } = makeStream({ rooms: [] });
+ await flush();
+ expect(server.sockets[0]?.headers).toEqual({ Authorization: 'Bearer tok' });
+ abort.abort();
+ });
+
it.each([
[403, WORKER_CAPABILITY_OBSOLETE],
[403, 'worker_capability_required'],
[426, 'upgrade_required'],
[403, 'hosted_worker_only'],
- ])('stops for good on a %i %s open', async (status, code) => {
+ ])('stops for good on a %i %s refusal, and does not try the stream', async (status, code) => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async () => refused(status, code));
+ serve((socket) => socket.refuse(status, { code, message: code }));
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
+ const { abort, fetchMock } = makeStream({
rooms: [],
worker,
onEvicted: (e) => evicted.push(e),
});
await vi.advanceTimersByTimeAsync(5 * 60_000);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(server.sockets).toHaveLength(1);
+ expect(postsTo(fetchMock, '/events')).toEqual([]);
expect(evicted.map((e) => e.code)).toEqual([code]);
abort.abort();
});
it('backs off and retries while another worker is attached', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async () => refused(409, 'worker_already_attached'));
+ serve((socket) => socket.refuse(409, { code: 'worker_already_attached', message: 'attached' }));
const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- worker,
- onEvicted: (e) => evicted.push(e),
- });
+ const { abort } = makeStream({ rooms: [], worker, onEvicted: (e) => evicted.push(e) });
await vi.advanceTimersByTimeAsync(10_000);
- expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);
- expect(urlsFor(fetchMock, '/events').length).toBeLessThan(6);
+ expect(server.sockets.length).toBeGreaterThan(1);
+ expect(server.sockets.length).toBeLessThan(6);
expect(evicted).toEqual([]);
abort.abort();
});
it('halts on a launch_superseded eviction', async () => {
vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('evicted', { code: 'launch_superseded', reason: 'bumped' }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- worker,
- onEvicted: (e) => evicted.push(e),
+ serve((socket) => {
+ attach(socket, 0);
+ socket.frame('evicted', { code: 'launch_superseded', reason: 'bumped', room_id: null });
+ socket.drop();
});
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream({ rooms: [], worker, onEvicted: (e) => evicted.push(e) });
await vi.advanceTimersByTimeAsync(5 * 60_000);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(server.sockets).toHaveLength(1);
expect(evicted.map((e) => e.code)).toEqual([EVICTION_LAUNCH_SUPERSEDED]);
abort.abort();
});
it('hands worker frames over in order and not as room events', async () => {
- const body = new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(4));
- controller.enqueue(encodeFrame('worker_attached', { launch_revision: 3 }));
- controller.enqueue(encodeFrame('operation', { id: 'op-1' }));
- },
+ serve((socket) => {
+ attach(socket, 0);
+ socket.frame('worker_attached', { launch_revision: 3 });
+ socket.frame('operation', { id: 'op-1' });
});
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
const frames: [string, unknown][] = [];
- const events: unknown[] = [];
- const { abort } = makeStream(fetchMock, {
+ const onEvent = vi.fn();
+ const { abort } = makeStream({
rooms: [],
worker,
- onEvent: (e) => void events.push(e),
+ onEvent,
onWorkerFrame: (name, data) => void frames.push([name, data]),
});
- await flush();
- expect(frames).toEqual([
- ['worker_attached', { launch_revision: 3 }],
- ['operation', { id: 'op-1' }],
- ]);
- expect(events).toEqual([]);
- abort.abort();
- });
-
- it('sends an up-call as its connection and incarnation, and raises a refusal with its code', async () => {
- const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => {
- const u = String(url);
- if (u.includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- if (u.endsWith('/claim')) return refused(409, 'already_claimed');
- return {
- ok: true,
- status: 200,
- text: async (): Promise<string> => '{"queued_operations":[]}',
- };
- });
- const { stream, abort } = makeStream(fetchMock, { rooms: [], worker });
- await expect(
- stream.workerCall('/agents/agent-1/connection/idle', { busy: false })
- ).resolves.toEqual({
- queued_operations: [],
- });
- const idle = fetchMock.mock.calls.find((c) => String(c[0]).endsWith('/connection/idle'));
- expect(String(idle?.[0])).toBe('https://switch.test/agents/agent-1/connection/idle');
- expect(JSON.parse(String(idle?.[1]?.body))).toEqual({
- busy: false,
- connection_id: 'conn-1',
- generation: 0,
- });
- const claim = stream.workerCall('/hosted/operations/op-1/claim', {});
- await expect(claim).rejects.toBeInstanceOf(WorkerCallError);
- await expect(claim).rejects.toMatchObject({ status: 409, code: 'already_claimed' });
- abort.abort();
- });
-});
-
-describe('test_gap_above_cursor_is_not_cursor_reset', () => {
- it('reports a restart gap resuming above the cursor as no cursor reset', async () => {
- const body = new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- controller.enqueue(
- new TextEncoder().encode(
- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"r"}\n\n'
- )
- );
- controller.enqueue(
- encodeFrame('gap', {
- from_sequence: 7,
- resumed_at: 2 ** 32 - 1,
- reason: 'server restarted',
- rooms: [],
- })
- );
- },
- });
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];
- const { stream, abort } = makeStream(fetchMock, { rooms: [], onGap: (g) => void gaps.push(g) });
- await flush();
- expect(gaps).toEqual([expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false })]);
- expect(stream.position).toBe(2 ** 32 - 1);
- abort.abort();
- });
-});
-
-it('does not give its connection up to itself over a request answered across its own reopen', async () => {
- // Seen on a real host: placements stated under incarnation 4, the stream
- // reopened (to 5) before the answer came, and the answer — "taken over:
- // reopened since 4, now at 5" — named this very client as the new holder.
- // It stood down for good, with nobody else anywhere near the connection.
- const opened: ReadableStreamDefaultController<Uint8Array>[] = [];
- let answerPlacements: (response: Response) => void = () => {};
- const fetchMock = vi.fn(async (url: string) => {
- if (url.includes('/events')) {
- const generation = 4 + opened.length;
- return new Response(
- new ReadableStream({
- start(controller) {
- opened.push(controller);
- controller.enqueue(connected(generation));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
+ try {
+ await vi.waitFor(() => expect(frames).toHaveLength(2));
+ expect(frames).toEqual([
+ ['worker_attached', { launch_revision: 3 }],
+ ['operation', { id: 'op-1' }],
+ ]);
+ expect(onEvent).not.toHaveBeenCalled();
+ } finally {
+ abort.abort();
}
- if (url.includes('connection/placements'))
- return new Promise<Response>((resolve) => {
- answerPlacements = resolve;
- });
- return Response.json({});
- });
- const evicted: Eviction[] = [];
- const onConnected = vi.fn();
- const { stream, abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onConnected,
- onEvicted: (eviction) => evicted.push(eviction),
- });
- try {
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
- const stating = stream.replacePlacements({ session: 'room' });
- await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));
-
- opened[0]!.close();
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
- answerPlacements(
- Response.json(
- {
- detail: {
- code: 'taken_over',
- message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',
- },
- },
- { status: 409 }
- )
- );
-
- await expect(stating).rejects.toThrow('409');
- expect(evicted).toEqual([]);
- } finally {
- abort.abort();
- }
-});
-
-it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- if (url.includes('/events'))
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected(4));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- if (url.includes('connection/placements'))
- return Response.json(
- { detail: { code: 'taken_over', message: 'another client holds it' } },
- { status: 409 }
- );
- return Response.json({});
- });
- const evicted: Eviction[] = [];
- const { stream, abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onEvicted: (eviction) => evicted.push(eviction),
- });
- try {
- await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');
- expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);
- } finally {
- abort.abort();
- }
-});
-
-describe('the heartbeat across its own reopen after a refusal', () => {
- /**
- * What took every in-Console watcher off the air when a laptop woke
- * (2026-10-01, four agents within ten seconds). The connection had lapsed
- * while the machine slept, so the first beat after waking was refused with a
- * 404. The heartbeat asked the stream to reopen and went straight back to
- * its gate — which it had found open a moment before the stream loop closed
- * it for the reopen. One beat went out under the lapsed incarnation right
- * behind the open that replaced it. The server, holding the new incarnation,
- * answered that beat `taken_over`, and the client believed it: it stood down
- * over a connection it had just made itself. In the server log the open and
- * the stale beat arrived 11 ms apart, every time.
- */
- function lapsingServer() {
- let generation = 0;
- let lapsed = false;
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {
- if (String(url).includes('/events')) {
- generation += 1;
- lapsed = false;
- const gen = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(gen));
- init.signal.addEventListener(
- 'abort',
- () => {
- try {
- controller.error(new DOMException('The operation was aborted.', 'AbortError'));
- } catch {
- // Already closed.
- }
- },
- { once: true }
- );
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;
- if (claimed !== null && claimed !== generation) {
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({
- detail: {
- code: 'taken_over',
- message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,
- },
- }),
- };
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- /** The server sweeps the connection for silence; its socket stays up
- * client-side, the way a socket does across a sleep. */
- const lapse = (): void => {
- lapsed = true;
- };
- return { fetchMock, lapse };
- }
-
- it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {
- vi.useFakeTimers();
- const server = lapsingServer();
- const evicted: Eviction[] = [];
- const { abort } = makeStream(server.fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);
- server.lapse();
- await vi.advanceTimersByTimeAsync(30_000);
-
- // The beat refused with a 404 is the last one under the lapsed
- // incarnation: what follows it on the wire is the reopen, not another beat
- // that the server could only take for someone else's.
- const sent = server.fetchMock.mock.calls.map((call) =>
- String(call[0]).includes('/events')
- ? 'open'
- : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`
- );
- const reopen = sent.indexOf('open', 1);
- expect(reopen).toBeGreaterThan(0);
- expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);
- expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);
- expect(evicted).toEqual([]);
- abort.abort();
});
});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index e9147bdd3..e7d963c4a 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -1,6 +1,6 @@
import { contractRange } from './artifacts';
import { ReattachFence } from './reattach-fence';
-import { readSse, type SseFrame } from './sse';
+import { readSse } from './sse';
import type { AgentBridgeEvent, SwitchCredentials } from './types';
import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';
@@ -8,44 +8,68 @@ import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';
const AGENT_PROTOCOL = contractRange('agent-protocol', RUNTIME_ARTIFACT);
/**
- * The agent bridge's push transport (CHOO-1857), client side.
+ * The agent bridge's connection (CHOO-1857), client side.
*
- * Replaces the long-poll: one SSE stream carries events, one heartbeat proves
- * we are alive, and both are tied to a connection id we choose. The connection
- * outlives its socket — losing the stream stops delivery but does not end the
- * connection, so reopening within the heartbeat TTL keeps the room slot and the
- * role lease and resumes from the cursor.
+ * One WebSocket per connection, tied to a connection id we choose. It carries
+ * the events down and the heartbeat both ways: the server sends `ping` every
+ * heartbeat interval and we answer `pong` with our cursor, which is what keeps
+ * the connection alive. The token is checked once, when the socket opens. The
+ * connection outlives its socket: losing the socket stops delivery but does not
+ * end the connection, so reconnecting within the heartbeat TTL keeps the room
+ * slot and the role lease and resumes from the cursor.
*
- * Two things this buys that the poll could not:
+ * A server from before the socket (agent-protocol 7 and older) has none, and
+ * is spoken to the way this runtime used to: the same frames as a Server-Sent
+ * Events stream, and a heartbeat we send ourselves, `POST /connection/beat`
+ * every two seconds with the cursor. Which one is decided on each open (see
+ * `openFrames`). That path is kept for a compatibility window and goes when
+ * those servers are gone.
*
- * - **Resume.** Every event carries a sequence number; we reopen with
- * `Last-Event-ID` and get exactly what we missed. The poll had no cursor at
- * all — the server drained the queue on read, so anything delivered while we
- * were away was simply gone.
- * - **One heartbeat instead of three.** `/connection/renew`, `/leases/renew`
- * and `/watch/heartbeat` collapse into `/connection/beat`.
- *
- * A gap is never silent. If the server cannot serve our cursor it says so, and
- * `onGap` fires so the caller can tell the agent to re-read context rather than
- * carry on believing it saw everything.
+ * Every event carries a sequence number, and a reconnect names the last one we
+ * handled, so we get exactly what we missed. A gap is never silent: if the
+ * server cannot serve our cursor it says so, and `onGap` fires so the caller
+ * can tell the agent to re-read context rather than carry on believing it saw
+ * everything.
*/
-/** Cadence of the connection heartbeat. Must stay well inside the server's
- * 6s TTL — the server declares the connection dead without it. */
+/** How long a request on the side (a room claim, placements, a beat) may take. */
+const REQUEST_TIMEOUT_MS = 4000;
+/** Cadence of the heartbeat we send on the event stream. Must stay well inside
+ * the server's 6s TTL: the server declares the connection dead without it. The
+ * socket needs none; the server pings it. */
export const BEAT_INTERVAL_MS = 2000;
-const BEAT_REQUEST_TIMEOUT_MS = 4000;
/**
* How long a reopen waits for a beat already in flight before disowning it.
*
* A beat answered inside this window is still believed, so a takeover that
- * lands while the socket happens to be reopening is acted on. Past it the
+ * lands while the stream happens to be reopening is acted on. Past it the
* answer cannot be told apart from one our own reopen provoked, and the reopen
* matters more: it is what restores delivery, and the server gives the
* connection six seconds without a beat.
*/
export const BEAT_SETTLE_LIMIT_MS = 1000;
+/** The agent-protocol revision that brought the socket. A server that speaks
+ * an older one has no socket, only the event stream. */
+const SOCKET_PROTOCOL_REVISION = 8;
+/** The code a server answers the event stream with once it has removed it. */
+const TRANSPORT_REMOVED = 'transport_removed';
const INITIAL_BACKOFF_MS = 1000;
const MAX_BACKOFF_MS = 30_000;
+/**
+ * The close code a server sends when it is restarting (uvicorn sends it to
+ * every open socket on shutdown).
+ *
+ * A restart is the one ending we know is short, and the doubling backoff fits
+ * it badly: the socket closes as shutdown begins, so the first retries are
+ * refused while the server is down and the next one lands seconds after it is
+ * back. For a while after this code we retry at a short, randomised interval
+ * instead, which brings the agent back within a second of the server and
+ * spreads a fleet of agents over that second rather than one instant.
+ */
+const SERVICE_RESTART = 1012;
+const RESTART_WINDOW_MS = 60_000;
+const RESTART_RETRY_MIN_MS = 250;
+const RESTART_RETRY_SPREAD_MS = 500;
/**
* How long an open socket has to last before it counts as a working stream.
*
@@ -83,7 +107,7 @@ export type DeliveryFilter = 'all' | 'addressed';
* - `taken_over` — another client attached to this connection id. **Terminal.**
* Reopening is itself a takeover, so a displaced client that retries is how
* two clients trade one connection back and forth forever.
- * - `heartbeat_lapsed` — we stopped ticking. Recoverable: reopen and resume.
+ * - `heartbeat_lapsed` — we stopped answering pings. Recoverable: reopen and resume.
* - `closed` — the server ended it for some other reason; `roomId` names the
* room when it was about one.
* - `credentials_rejected` — decided here rather than sent: every reopen would
@@ -163,7 +187,7 @@ function evictionCode(data: Record<string, unknown>): string {
}
/**
- * Read the code off a refusal — a rejected heartbeat or a rejected reattach.
+ * Read the code off a refusal — a rejected open or reattach.
*
* Every refusal shares one status, and they call for opposite responses: being
* superseded is terminal, because attaching is itself a takeover, while the
@@ -325,7 +349,7 @@ export interface SwitchEventStreamDeps {
cursorReset?: boolean;
}): void | Promise<void>;
/** Fired when another stream took this connection over, or it was closed.
- * A `taken_over` eviction has already halted both loops before this runs:
+ * A `taken_over` eviction has already halted the connection before this runs:
* there is nothing left to reconnect, only something to report. */
onEvicted(eviction: Eviction): void;
/**
@@ -335,21 +359,58 @@ export interface SwitchEventStreamDeps {
worker: WorkerIdentity | null;
onWorkerFrame?: (frame: WorkerFrameName, data: Record<string, unknown>) => Promise<void> | void;
log: EventStreamLogger;
- /** Aborts the stream and the heartbeat together. */
+ /** Aborts the connection. */
signal: AbortSignal;
}
+/**
+ * The server refused to open the connection: a `refused` frame, or a close
+ * code of 4000 plus an HTTP status. `body` is shaped like the HTTP error body
+ * the same refusal would have had (`{"detail": ...}`), so the remedies that
+ * read it do not care which transport it came over.
+ */
+export class OpenRefused extends Error {
+ constructor(
+ readonly status: number,
+ readonly body: string
+ ) {
+ super(`HTTP ${status}: ${body}`);
+ this.name = 'OpenRefused';
+ }
+}
+
+/** One frame off the socket: `{event, data, id}`, `id` being its sequence number. */
+export type SocketFrame = { event: string; id?: string; data: Record<string, unknown> };
+
+/** What a connection's frames travel over: the socket, or the event stream an
+ * old server offers instead. */
+type Transport = 'socket' | 'stream';
+
+/** The socket closed before it opened: the server is unreachable, or has no
+ * socket to offer. Only a request to the event stream tells the two apart. */
+class SocketNeverOpened extends Error {
+ constructor(code: number | null) {
+ super(
+ `could not connect to Switch (socket closed with ${code ?? 'no code'}, and the event stream did not answer either): the server is unreachable`
+ );
+ this.name = 'SocketNeverOpened';
+ }
+}
+
+/** Node's WebSocket, which, unlike a browser's, takes request headers. */
+type HeaderedWebSocket = new (url: string, init: { headers: Record<string, string> }) => WebSocket;
+
export class SwitchEventStream {
private readonly deps: SwitchEventStreamDeps;
- /** Highest sequence number received. Sent on every beat and used as
- * `Last-Event-ID` when reopening. Seeded from `startCursor` when the caller
+ /** Highest sequence number received. Sent with every pong and as
+ * `start_from` when reopening. Seeded from `startCursor` when the caller
* needs to begin behind head rather than at it. */
private cursor: number;
/** Aborts only the current socket, so a reconnect can replace it without
* tearing down the connection. */
private socketAbort: AbortController | null = null;
/** Aborts the connection for good. Distinct from the caller's signal: some
- * refusals can never be retried into a success, and both loops have to end. */
+ * refusals can never be retried into a success, and the loop has to end. */
private readonly halt = new AbortController();
private rooms: string[];
/**
@@ -364,52 +425,64 @@ export class SwitchEventStream {
/**
* Which incarnation of the connection id the server last told us we are.
*
- * Sent on every beat so the server can refuse a tick from a client that has
- * been displaced: sharing the id is what makes takeover work, and it is also
- * what makes the loser's beat indistinguishable from the winner's.
+ * Sent as `expected_generation` on every reopen, so the server refuses a
+ * reattach from a client that has been displaced: sharing the id is what
+ * makes takeover work, and it is also what makes the loser's reopen
+ * indistinguishable from the winner's.
*
* Kept across a dropped socket rather than cleared. The connection outlives
* its socket, so a client whose socket merely dropped is still the holder,
- * and nulling this would have it beat unfenced — or, worse, stop beating and
- * lose the connection it still owns. Null only before the first
- * `connection_state`, and against a server too old to send one.
+ * and nulling this would have it reopen unfenced and take the connection
+ * back off whoever holds it now. Null only before the first
+ * `connection_state`.
*/
private generation: number | null = null;
/** The server leaves telling a room a session is starting to this client (`connection_state`). */
private announces = false;
/**
- * The barrier between the heartbeat and the socket it beats for.
+ * Closed from the moment an open starts until the frame naming the
+ * incarnation that open made arrives.
*
- * The heartbeat waits on it before every tick, so no beat is sent between an
- * open and the frame naming the incarnation that open made: there is nothing
- * current to fence such a tick with, and a stale incarnation is worse than
- * none — the server reads it as a takeover and we would stand down over our
- * own reconnect. Nothing is lost by waiting, since a beat with no stream
- * attached is refused anyway, and a server too old to carry an incarnation
- * still sends the frame.
- *
- * It also disowns a beat that was already in flight when the open began, for
- * the same reason from the other end: that answer was decided about the
- * incarnation we have just replaced.
+ * A reopen asked for in between waits on it (see `reopen`): there is nothing
+ * current to claim, and a stale incarnation is worse than none, since the
+ * server reads it as a takeover and we would stand down over our own
+ * reconnect.
*/
private readonly fence = new ReattachFence();
+ /**
+ * Reopens asked for, and how many of those the open now in progress was
+ * built after — so a reopen asked for mid-open is carried out once that open
+ * is answered, never by cancelling it. See `reopen`.
+ */
+ private reopensWanted = 0;
+ /** Until when the server is taken to be restarting (see `SERVICE_RESTART`). */
+ private restartingUntil = 0;
+ private openCarries = 0;
+ /**
+ * Whether to open the event stream rather than try the socket first.
+ *
+ * Set when a server we reached over the stream says it speaks a revision
+ * from before the socket, so each reopen does not first knock on a door that
+ * is not there. Cleared as soon as the server says otherwise: a
+ * `connection_state` naming a later revision, or the stream refused as
+ * removed. A socket that merely fails to open never sets it; only an answer
+ * from the server does, so a network failure cannot leave us on the stream.
+ */
+ private preferStream = false;
+ /** What the open in progress travels over, and what the attached one does. */
+ private openingOver: Transport = 'socket';
+ private attachedOver: Transport | null = null;
/**
* How many times the server has named an incarnation for this client, and
* how to cut the heartbeat's current wait short when it does.
*
* A fresh attach starts the server's 6-second clock on a connection that has
- * never beaten, so the heartbeat must go out promptly — whatever back-off an
- * earlier, failing connection had built up. See `beatLoop`.
+ * never beaten, so on the event stream the heartbeat must go out promptly,
+ * whatever back-off an earlier, failing connection had built up. See
+ * `beatLoop`.
*/
private attaches = 0;
private wakeBeat: () => void = () => {};
- /**
- * Reopens asked for, and how many of those the open now in progress was
- * built after — so a reopen asked for mid-open is carried out once that open
- * is answered, never by cancelling it. See `reopen`.
- */
- private reopensWanted = 0;
- private openCarries = 0;
constructor(deps: SwitchEventStreamDeps) {
this.deps = deps;
@@ -475,11 +548,11 @@ export class SwitchEventStream {
* quick succession would permanently stand down the agent's only connection.
* The confirmation carries whatever the declaration has settled on by then
* instead, and a change that cancels itself out carries nothing. This is the
- * same hazard the heartbeat's fence exists for, from the other side.
+ * same hazard the fence exists for.
*
- * Nothing to fence before the first `connection_state`, or against a server
- * too old to send one: with no incarnation to claim the reopen is a plain
- * attach, which cannot be refused for being stale.
+ * Nothing to fence before the first `connection_state`: with no incarnation
+ * to claim the reopen is a plain attach, which cannot be refused for being
+ * stale.
*/
private redeclare(): void {
if (this.declaredSpawnCapable === this.spawnCapable) return;
@@ -508,19 +581,16 @@ export class SwitchEventStream {
this.reopensWanted += 1;
if (!this.fence.admitting) return;
// Held from this instant, not from when the stream loop gets round to its
- // own `detaching`. Whoever asked for the reopen — the heartbeat, after a
- // refusal — goes straight back to the gate, and could otherwise find it
- // still open and send one more beat under the incarnation this reopen is
- // about to replace. Arriving just behind the open, that beat is refused as
- // a takeover and the client stands down over a connection it made itself.
+ // own `detaching`, so a second reopen asked for straight after this one
+ // waits for the incarnation it makes rather than claiming the old one.
this.fence.closeAdmission();
this.socketAbort?.abort();
}
/**
- * Reopen only to get attached again — the heartbeat's reason. An open
- * already in flight does exactly that, so there is nothing to add to it; and
- * one scheduled after a dropped socket carries the request anyway.
+ * Reopen only to get attached again, the heartbeat's reason. An open already
+ * in flight does exactly that, so there is nothing to add to it; and one
+ * scheduled after a dropped stream carries the request anyway.
*/
private reattach(): void {
if (this.fence.admitting) this.reopen();
@@ -598,7 +668,7 @@ export class SwitchEventStream {
}
/** A rejected credential is not an outage: every reopen would carry the same
- * token, so end both loops and tell the owner once. */
+ * token, so end the connection and tell the owner once. */
private rejectCredentials(status: number, body: string): void {
if (this.halt.signal.aborted) return;
if (this.refuseWorker(status, body)) return;
@@ -622,14 +692,12 @@ export class SwitchEventStream {
*
* A request carries the incarnation it was sent under, and the server
* refuses it as taken over once the connection has moved past that. But
- * this client reopens its own stream — after an eviction, a refused beat, a
+ * this client reopens its own socket — after an eviction, a dropped socket, a
* repoint — and a request in flight across that reopen is refused the same
* way, naming as the new holder the incarnation this client now is. Standing
- * down on that gives the connection up to ourselves, for good. Heartbeats
- * already discard such answers through the fence; this is the same test for
- * the requests that do not go through it. A real takeover after our reopen
- * is still caught: the next beat, under the incarnation we now hold, is
- * refused, and stands down.
+ * down on that gives the connection up to ourselves, for good. A real
+ * takeover after our reopen is still caught: the server tells the socket we
+ * now hold with an `evicted` frame.
*/
private supersededOurselves(sent: number | null): boolean {
return sent !== null && this.generation !== null && this.generation !== sent;
@@ -638,14 +706,14 @@ export class SwitchEventStream {
/**
* Give the connection up to the client that now holds it.
*
- * Ends both loops and reports through the same callback an `evicted` frame
+ * Ends the connection and reports through the same callback an `evicted` frame
* does, with the same code, so whatever records the stand-down does not have
* to care which door the news came through.
*/
private standDown(): void {
if (this.halt.signal.aborted) return;
this.deps.log.warn('SwitchEventStream: the connection was taken over — standing down', {
- event: 'switch_beat_superseded',
+ event: 'switch_stream_superseded',
connectionId: this.deps.connectionId,
});
this.halt.abort();
@@ -764,7 +832,7 @@ export class SwitchEventStream {
return text ? JSON.parse(text) : null;
}
- private post(path: string, body: unknown, timeoutMs = BEAT_REQUEST_TIMEOUT_MS) {
+ private post(path: string, body: unknown, timeoutMs = REQUEST_TIMEOUT_MS) {
const { creds } = this.deps;
return fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/${path}`, {
method: 'POST',
@@ -787,7 +855,7 @@ export class SwitchEventStream {
* the same line every thirty seconds until the app is quit, which is how a
* real failure elsewhere gets lost. The first failure is reported, then
* powers of two, and recovery is stated so the outage has an end in the log
- * as well as a beginning — the same discipline the heartbeat already applies.
+ * as well as a beginning.
*/
private async streamLoop(): Promise<void> {
const { creds, connectionId, scope, filter, log, signal } = this.deps;
@@ -803,6 +871,11 @@ export class SwitchEventStream {
* server is ending streams on purpose.
*/
const pace = async (): Promise<void> => {
+ if (Date.now() < this.restartingUntil) {
+ const wait = RESTART_RETRY_MIN_MS + Math.random() * RESTART_RETRY_SPREAD_MS;
+ await new Promise((r) => setTimeout(r, wait));
+ return;
+ }
await new Promise((r) => setTimeout(r, backoff));
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
};
@@ -811,9 +884,10 @@ export class SwitchEventStream {
const socketAbort = new AbortController();
this.socketAbort = socketAbort;
// Closed before the open rather than after it lands: it is this open
- // that makes the new incarnation, so the heartbeat has to be held from
- // here until the frame naming it arrives. A beat already in flight is
- // given a moment to come back and be believed, and disowned after that.
+ // that makes the new incarnation, so a reopen and the event stream's
+ // heartbeat have to wait from here until the frame naming it arrives. A
+ // beat already in flight is given a moment to come back and be
+ // believed, and disowned after that. The socket has none in flight.
await this.fence.detaching(BEAT_SETTLE_LIMIT_MS);
if (signal.aborted || this.halt.signal.aborted) return;
let openedAt = 0;
@@ -841,66 +915,49 @@ export class SwitchEventStream {
// Reattaching, so say which incarnation we believe we still are and
// let the server refuse us if we are wrong. An attach is a takeover,
// and a client that missed its own eviction — a partition, a dropped
- // socket, a beat whose refusal never arrived — would otherwise take
+ // socket, an eviction frame that never arrived — would otherwise take
// the connection straight back off whoever legitimately holds it. The
// first open of this object's life sends nothing, which is how a
// deliberate takeover still works: it has no incarnation to claim.
if (this.generation !== null) params.set('expected_generation', String(this.generation));
- const worker = this.deps.worker;
- const resp = await fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/events?${params}`, {
- headers: {
- Authorization: `Bearer ${creds.token}`,
- Accept: 'text/event-stream',
- ...(this.cursor > 0 ? { 'Last-Event-ID': String(this.cursor) } : {}),
- ...(worker
- ? {
- 'X-Switch-Worker-Capability': worker.capability,
- 'X-Switch-Host-Boot-Id': worker.bootId,
- 'X-Switch-Host-Instance-Id': worker.instanceId,
- 'X-Switch-Worker-State-Version': String(worker.stateVersion),
- }
- : {}),
- },
- signal: AbortSignal.any([socketAbort.signal, signal, this.halt.signal]),
- });
-
- if (!resp.ok || !resp.body) {
- const body = await resp.text();
- if (this.refuseWorker(resp.status, body)) return;
+ for await (const frame of this.openFrames(params, socketAbort.signal, () => {
+ openedAt = Date.now();
+ this.restartingUntil = 0;
+ log.debug('SwitchEventStream: connection open', {
+ event: 'switch_stream_open',
+ connectionId,
+ cursor: this.cursor,
+ rooms: this.rooms,
+ });
+ })) {
+ await this.handleFrame(frame);
+ if (frame.id) this.cursor = Math.max(this.cursor, Number(frame.id) || 0);
+ }
+ } catch (error) {
+ if (error instanceof OpenRefused) {
+ const { status, body } = error;
+ if (this.refuseWorker(status, body)) return;
// Reopening without the refused room is a different request from the
// one that just failed, and the declared set strictly shrinks, so
// this cannot spin: retry now rather than serving the backoff a
// transport failure earned.
- if (this.dropRefusedRooms(resp.status, body)) continue;
- if (resp.status === 401 || resp.status === 403) {
- this.rejectCredentials(resp.status, body);
+ if (this.dropRefusedRooms(status, body)) continue;
+ if (status === 401 || status === 403) {
+ this.rejectCredentials(status, body);
return;
}
- if (resp.status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
+ if (status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
// The reattach was refused because someone else holds the
- // connection now, and nothing was disturbed in refusing it. This
- // is the only place the loser can be told once its socket and its
- // heartbeat have both stopped being able to reach it.
+ // connection now, and nothing was disturbed in refusing it.
this.standDown();
return;
}
- throw new Error(`HTTP ${resp.status}: ${body}`);
+ // The event stream we went to straight away is gone, so the server
+ // has the socket now (`openFrames` has stopped preferring the
+ // stream). A different request from the one refused, so retry at once.
+ if (status === 410 && refusalCode(body) === TRANSPORT_REMOVED) continue;
}
-
- openedAt = Date.now();
- log.debug('SwitchEventStream: stream open', {
- event: 'switch_stream_open',
- connectionId,
- cursor: this.cursor,
- rooms: this.rooms,
- });
-
- for await (const frame of readSse(resp.body, socketAbort.signal)) {
- await this.handleFrame(frame);
- if (frame.id) this.cursor = Math.max(this.cursor, Number(frame.id) || 0);
- }
- } catch (error) {
failure = error;
}
@@ -939,16 +996,208 @@ export class SwitchEventStream {
}
}
- private async handleFrame(frame: SseFrame): Promise<void> {
+ /**
+ * Open the connection and yield its frames, over the socket if the server
+ * has one and over the event stream if it does not.
+ *
+ * Node's WebSocket does not say why a handshake failed, so a server with no
+ * socket (it answers the upgrade with 404) looks exactly like one we cannot
+ * reach. The event stream is what tells them apart: a socket that never
+ * opened is followed at once by a request for the stream, and a server that
+ * answers it is one we can talk to that way. Decided on each open rather
+ * than once for good, so a network failure costs one request and decides
+ * nothing, and a server upgraded under us is moved back onto the socket.
+ */
+ private async *openFrames(
+ params: URLSearchParams,
+ abort: AbortSignal,
+ onOpen: () => void
+ ): AsyncGenerator<SocketFrame> {
+ const { creds } = this.deps;
+ if (!this.preferStream) {
+ this.openingOver = 'socket';
+ const url = `${creds.apiEndpoint.replace(/^http/, 'ws')}/agents/${creds.agentId}/connection/ws?${params}`;
+ try {
+ yield* this.readSocket(url, abort, onOpen);
+ return;
+ } catch (error) {
+ if (!(error instanceof SocketNeverOpened) || abort.aborted) throw error;
+ this.openingOver = 'stream';
+ yield* this.readEventStream(params, abort, onOpen, error);
+ return;
+ }
+ }
+ this.openingOver = 'stream';
+ yield* this.readEventStream(params, abort, onOpen, null);
+ }
+
+ /** Who a hosted worker is, as the headers every open of its connection carries. */
+ private workerHeaders(): Record<string, string> {
+ const worker = this.deps.worker;
+ if (!worker) return {};
+ return {
+ 'X-Switch-Worker-Capability': worker.capability,
+ 'X-Switch-Host-Boot-Id': worker.bootId,
+ 'X-Switch-Host-Instance-Id': worker.instanceId,
+ 'X-Switch-Worker-State-Version': String(worker.stateVersion),
+ };
+ }
+
+ /**
+ * Open the connection as a Server-Sent Events stream, the way a server from
+ * before the socket serves it, and yield its frames until it ends.
+ *
+ * `probing` is the socket's failure when this request is the one that tells
+ * an old server from an unreachable one. Then anything short of a stream
+ * means unreachable, and that failure is what is reported. A refusal is
+ * still the server's answer, and handled as one.
+ */
+ private async *readEventStream(
+ params: URLSearchParams,
+ abort: AbortSignal,
+ onOpen: () => void,
+ probing: SocketNeverOpened | null
+ ): AsyncGenerator<SocketFrame> {
+ const { creds, signal } = this.deps;
+ let resp: Response;
+ try {
+ resp = await fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/events?${params}`, {
+ headers: {
+ Authorization: `Bearer ${creds.token}`,
+ Accept: 'text/event-stream',
+ ...(this.cursor > 0 ? { 'Last-Event-ID': String(this.cursor) } : {}),
+ ...this.workerHeaders(),
+ },
+ signal: AbortSignal.any([abort, signal, this.halt.signal]),
+ });
+ } catch (error) {
+ throw probing ?? error;
+ }
+ if (!resp.ok || !resp.body) {
+ const body = await resp.text();
+ if (resp.status === 410 && refusalCode(body) === TRANSPORT_REMOVED) {
+ // A server that has removed the stream has the socket.
+ this.preferStream = false;
+ throw probing ?? new OpenRefused(resp.status, body);
+ }
+ if (probing && resp.ok) throw probing;
+ throw new OpenRefused(resp.status, body);
+ }
+ onOpen();
+ yield* readSse(resp.body, abort);
+ }
+
+ /**
+ * Open the connection's socket and yield its frames until it closes.
+ *
+ * The heartbeat is answered here, as each `ping` arrives, not queued behind
+ * the frames: handling an event can take as long as the agent likes, and
+ * the server must not decide we are gone while it does. The pong carries
+ * the cursor as it stands, which is the last event fully handled.
+ */
+ private async *readSocket(
+ url: string,
+ abort: AbortSignal,
+ onOpen: () => void
+ ): AsyncGenerator<SocketFrame> {
+ const Socket = (globalThis as { WebSocket?: HeaderedWebSocket }).WebSocket;
+ if (!Socket) {
+ throw new Error(
+ `this runtime needs Node 22 or newer to connect to Switch (it has ${process.version}, with no WebSocket)`
+ );
+ }
+ // A hosted worker proves who it is on the socket exactly as on the stream.
+ const socket = new Socket(url, {
+ headers: { Authorization: `Bearer ${this.deps.creds.token}`, ...this.workerHeaders() },
+ });
+ const pending: SocketFrame[] = [];
+ let refused: unknown = null;
+ let closed: { code: number; reason: string } | null = null;
+ let opened = false;
+ let wake: (() => void) | null = null;
+ const notify = (): void => {
+ const resolve = wake;
+ wake = null;
+ resolve?.();
+ };
+ socket.addEventListener('open', () => {
+ opened = true;
+ onOpen();
+ });
+ socket.addEventListener('message', (message: MessageEvent) => {
+ let parsed: { event?: unknown; data?: unknown; id?: unknown };
+ try {
+ parsed = JSON.parse(String(message.data)) as typeof parsed;
+ } catch {
+ return;
+ }
+ if (parsed.event === 'ping') {
+ socket.send(JSON.stringify({ type: 'pong', cursor: this.cursor }));
+ return;
+ }
+ if (parsed.event === 'refused') {
+ refused = (parsed.data as { detail?: unknown } | undefined)?.detail ?? null;
+ return;
+ }
+ if (typeof parsed.event !== 'string') return;
+ pending.push({
+ event: parsed.event,
+ data: (parsed.data as Record<string, unknown> | undefined) ?? {},
+ ...(parsed.id !== undefined ? { id: String(parsed.id) } : {}),
+ });
+ notify();
+ });
+ socket.addEventListener('close', (event: CloseEvent) => {
+ closed = { code: event.code, reason: event.reason };
+ if (event.code === SERVICE_RESTART) this.restartingUntil = Date.now() + RESTART_WINDOW_MS;
+ notify();
+ });
+ socket.addEventListener('error', () => notify());
+ const stop = (): void => socket.close(1000);
+ abort.addEventListener('abort', stop, { once: true });
+ try {
+ for (;;) {
+ while (pending.length > 0) yield pending.shift() as SocketFrame;
+ if (closed !== null || abort.aborted) break;
+ await new Promise<void>((resolve) => {
+ wake = resolve;
+ });
+ }
+ const ending = closed as { code: number; reason: string } | null;
+ if (ending !== null && ending.code >= 4000 && ending.code < 5000) {
+ throw new OpenRefused(
+ ending.code - 4000,
+ JSON.stringify({ detail: refused ?? ending.reason })
+ );
+ }
+ if (!opened && !abort.aborted) throw new SocketNeverOpened(ending?.code ?? null);
+ } finally {
+ abort.removeEventListener('abort', stop);
+ if (socket.readyState === socket.CONNECTING || socket.readyState === socket.OPEN) {
+ socket.close(1000);
+ }
+ }
+ }
+
+ private async handleFrame(frame: SocketFrame): Promise<void> {
const { log, onGap, onEvicted, onEvent } = this.deps;
switch (frame.event) {
case 'connection_state':
if (typeof frame.data.generation === 'number') this.generation = frame.data.generation;
this.announces = frame.data.announce_session_starts === true;
+ this.attachedOver = this.openingOver;
+ // Reached over the stream: whether the server has a socket for next
+ // time. One that names no revision is older than the declaration, and
+ // so older than the socket.
+ if (this.openingOver === 'stream') {
+ const speaks = frame.data.protocol;
+ this.preferStream = !(typeof speaks === 'number' && speaks >= SOCKET_PROTOCOL_REVISION);
+ }
log.debug('SwitchEventStream: connection established', {
event: 'switch_stream_connected',
rooms: frame.data.rooms,
generation: this.generation,
+ transport: this.attachedOver,
// What the server says it is (CHOO-1865). Recorded, not acted on —
// logging it is what makes "which versions are actually talking to
// each other" answerable from a bug report rather than a guess.
@@ -958,7 +1207,7 @@ export class SwitchEventStream {
// A reopen asked for while this open was in flight, for something this
// open was built too early to carry. Now that we know the incarnation
// to claim, it is safe to replace the socket — and done before the
- // gate opens, so no beat goes out on a socket about to be dropped.
+ // gate opens, so nothing else is sent on a socket about to be dropped.
if (this.reopensWanted > this.openCarries) {
this.socketAbort?.abort();
return;
@@ -1030,8 +1279,8 @@ export class SwitchEventStream {
roomId: frame.data.room_id ?? null,
});
// Halt before reporting: a takeover is the one ending that reopening
- // cannot recover, because reopening is itself a takeover. Both loops
- // end here, and nothing the callback does can restart them.
+ // cannot recover, because reopening is itself a takeover. The
+ // connection ends here, and nothing the callback does can restart them.
if (code === EVICTION_TAKEN_OVER || code === EVICTION_LAUNCH_SUPERSEDED) this.halt.abort();
onEvicted({
code,
@@ -1083,9 +1332,13 @@ export class SwitchEventStream {
}
/**
- * The single heartbeat. Proves we are alive and reports the cursor.
+ * The event stream's heartbeat. Proves we are alive and reports the cursor.
*
- * A 404 or 409 means we are not receiving — the connection expired, or it has
+ * Only while the connection is attached over the event stream: the socket is
+ * pinged by the server and answers from `readSocket`, and a beat beside it
+ * would be a request every two seconds for nothing.
+ *
+ * A 404 or 409 means we are not receiving: the connection expired, or it has
* no stream attached. Both are recovered by reopening, which resumes from the
* cursor, and both count as a beat that did not land. The exception is the
* 409 that says another client has taken the connection over, which no
@@ -1096,11 +1349,11 @@ export class SwitchEventStream {
* client that has stopped receiving while believing it is connected is
* exactly the bug this transport exists to remove.
*
- * While beats succeed the cadence is fixed and short — the server declares the
+ * While beats succeed the cadence is fixed and short: the server declares the
* connection dead without them. While they fail it backs off, because a beat
* that has already missed the TTL cannot save the connection: reopening the
* stream is what re-establishes it, and that loop is doing its own retrying.
- * Hammering a dead endpoint every two seconds forever is what this avoids —
+ * Hammering a dead endpoint every two seconds forever is what this avoids:
* an endpoint that is simply gone (a managed server's port after the stack was
* destroyed) should cost a trickle of requests and a handful of log lines, not
* a permanent stream of both.
@@ -1117,7 +1370,7 @@ export class SwitchEventStream {
/**
* Wait `ms`, but stop waiting the moment the server names a new
- * incarnation — and do not start waiting if it already has.
+ * incarnation, and do not start waiting if it already has.
*
* The back-off used to be served in full across a reattach. A beat that
* had been failing left the loop sleeping for up to 30 seconds; the stream
@@ -1163,7 +1416,7 @@ export class SwitchEventStream {
* One beat, as a value rather than a throw.
*
* A beat the fence disowns must have no outcome at all, and an exception
- * is an outcome — it would slow the loop down over a request the reattach
+ * is an outcome: it would slow the loop down over a request the reattach
* itself invalidated. So the whole exchange, body included, happens inside
* the flight the fence is timing, and comes back as something to ignore or
* act on once it is known which.
@@ -1196,6 +1449,13 @@ export class SwitchEventStream {
// promise we were handed is the old one. Checked here, with nothing
// awaited before the tick, so no beat leaves while admission is shut.
if (!this.fence.admitting) continue;
+ if (this.attachedOver !== 'stream') {
+ // Attached over the socket, which carries its own heartbeat. Sleep
+ // until the next attach, which may be over the stream.
+ seenAttach = this.attaches;
+ await pause(MAX_BACKOFF_MS);
+ continue;
+ }
// A connection the server has just attached has never beaten, and has
// 6 seconds to. Whatever an earlier connection's failures built up says
// nothing about this one, so it starts at the base cadence.
@@ -1211,7 +1471,7 @@ export class SwitchEventStream {
// an incarnation we are no longer on and cannot be told apart from
// one that arrived late about our own reopen. Inert: no stand-down,
// no reopen, no cursor, and no mark against the beat rate either.
- log.debug('SwitchEventStream: heartbeat answer discarded — reattached in flight', {
+ log.debug('SwitchEventStream: heartbeat answer discarded: reattached in flight', {
event: 'switch_beat_stale',
connectionId,
});
@@ -1232,15 +1492,15 @@ export class SwitchEventStream {
return;
}
if (status === 404 || status === 409) {
- // The server answered, so this is not an outage — but it is not a
+ // The server answered, so this is not an outage, but it is not a
// beat that landed either: we are not attached, and only a reopen
// fixes that. A reopen that keeps being refused is a client that
// cannot currently succeed, and it must not spend the endpoint at
// full rate while it fails.
//
// So the wait goes *before* the reopen, and grows with each refusal
- // in a row. It used to go after — reopen at once, then sleep the
- // back-off before beating — which starved the very connection the
+ // in a row. It used to go after (reopen at once, then sleep the
+ // back-off before beating), which starved the very connection the
// reopen had just made. Pacing the reopen instead keeps the same
// brake on a refusing server while letting each new connection beat
// as soon as it is attached.
@@ -1248,7 +1508,7 @@ export class SwitchEventStream {
rejections += 1;
const waitMs = Math.min(BEAT_INTERVAL_MS * 2 ** rejections, MAX_BACKOFF_MS);
if ((failures & (failures - 1)) === 0) {
- log.warn('SwitchEventStream: heartbeat rejected — reopening', {
+ log.warn('SwitchEventStream: heartbeat rejected: reopening', {
event: 'switch_beat_rejected',
status,
connectionId,
@@ -1258,7 +1518,7 @@ export class SwitchEventStream {
}
await pause(waitMs);
if (signal.aborted || this.halt.signal.aborted) return;
- // The stream may have reattached on its own while we waited — the
+ // The stream may have reattached on its own while we waited: the
// server closes a lapsed connection's socket, and the stream loop
// reopens it. Reopening again would throw that healthy connection
// away; beat on it instead.
diff --git a/console/packages/switch-agent-runtime/src/index.ts b/console/packages/switch-agent-runtime/src/index.ts
index ce12f58ca..5ee1940f4 100644
--- a/console/packages/switch-agent-runtime/src/index.ts
+++ b/console/packages/switch-agent-runtime/src/index.ts
@@ -26,7 +26,7 @@ export {
type ContractRange,
} from './artifacts';
export {
- BEAT_INTERVAL_MS,
+ OpenRefused,
EVICTION_CLOSED,
EVICTION_CREDENTIALS_REJECTED,
EVICTION_HEARTBEAT_LAPSED,
@@ -55,7 +55,7 @@ export {
type RoomDelivery,
type RoomReservation,
} from './room-admission';
-export { readSse, type SseFrame } from './sse';
+export type { SocketFrame } from './event-stream';
export type {
AgentBridgeEvent,
AttachmentRef,
diff --git a/console/packages/switch-agent-runtime/src/sse.test.ts b/console/packages/switch-agent-runtime/src/sse.test.ts
index 7235ab9f9..da53e9838 100644
--- a/console/packages/switch-agent-runtime/src/sse.test.ts
+++ b/console/packages/switch-agent-runtime/src/sse.test.ts
@@ -6,7 +6,7 @@ import { readSse, type SseFrame } from './sse';
*
* These moved here with the parser. They used to sit beside a drift check that
* compared two copies of this file; the copies are gone, so the check is too,
- * but the behaviour it protected still needs testing — and now testing it once
+ * but the behaviour it protected still needs testing, and now testing it once
* genuinely covers every client, which was the point of the move.
*/
@@ -40,7 +40,7 @@ describe('SSE framing', () => {
});
it('leaves id undefined when the server sends none', async () => {
- // Control frames carry no sequence — treating a missing id as 0 would drag
+ // Control frames carry no sequence, treating a missing id as 0 would drag
// the cursor backwards and replay the whole buffer on reconnect.
const frames = await parse(['event: connection_state\ndata: {"rooms":[]}\n\n']);
expect(frames[0].id).toBeUndefined();
diff --git a/console/packages/switch-agent-runtime/src/sse.ts b/console/packages/switch-agent-runtime/src/sse.ts
index 667967ea2..13b8f2204 100644
--- a/console/packages/switch-agent-runtime/src/sse.ts
+++ b/console/packages/switch-agent-runtime/src/sse.ts
@@ -2,8 +2,8 @@
* The agent bridge's SSE framing.
*
* One implementation, imported by everything that speaks the protocol: the
- * local MCP runtime next to the agent, and Switch Console. It used to exist twice —
- * separate deployables, no shared package — and the copies drifted within a day
+ * local MCP runtime next to the agent, and Switch Console. It used to exist twice
+ * (separate deployables, no shared package) and the copies drifted within a day
* of being made. This package is what removed the excuse.
*
* Deliberately free of imports and of anything specific to a host: it is the
diff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.py
index d52e3e94b..07eb88b9a 100644
--- a/core/switch_core/artifacts.py
+++ b/core/switch_core/artifacts.py
@@ -33,8 +33,8 @@ ARTIFACT_VERSIONS: Final[dict[str, str]] = {
CONTRACTS: Final[dict[str, dict[str, ContractRange]]] = {
# The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
"agent-protocol": {
- "switch-core": ContractRange(speaks=7, accepts=1),
- "agent-runtime": ContractRange(speaks=7, accepts=1),
+ "switch-core": ContractRange(speaks=8, accepts=1),
+ "agent-runtime": ContractRange(speaks=8, accepts=1),
},
# The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
"gateway-api": {
diff --git a/core/switch_core/bridges/agent/api/handlers.py b/core/switch_core/bridges/agent/api/handlers.py
index 4a5c9f74d..1b29b2049 100644
--- a/core/switch_core/bridges/agent/api/handlers.py
+++ b/core/switch_core/bridges/agent/api/handlers.py
@@ -1,7 +1,9 @@
from __future__ import annotations
+import asyncio
import hashlib
import logging
+from collections.abc import AsyncGenerator
from contextvars import ContextVar
from datetime import UTC, datetime
from typing import Annotated, Any, cast
@@ -14,6 +16,8 @@ from fastapi import (
HTTPException,
Query,
UploadFile,
+ WebSocket,
+ WebSocketDisconnect,
)
from pydantic import ValidationError
from sqlalchemy.ext.asyncio import AsyncSession
@@ -86,6 +90,7 @@ from switch_core.bridges.agent.dependencies import (
)
from switch_core.bridges.agent.hosted_mailbox import deliver_on_attach
from switch_core.bridges.agent.protocol.agent_connections import (
+ HEARTBEAT_INTERVAL_SECONDS,
TAKEN_OVER,
AgentConnection,
ClientDeclaration,
@@ -99,6 +104,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (
SupersededConnectionError,
SupersededControlError,
SupersededReattachError,
+ Transport,
UnfencedBeatError,
UnfencedControlError,
UnknownConnectionError,
@@ -107,7 +113,13 @@ from switch_core.bridges.agent.protocol.agent_connections import (
from switch_core.bridges.agent.protocol.agent_core import AgentCore, AgentExistsError
from switch_core.bridges.agent.protocol.event_buffer import Reader
from switch_core.bridges.agent.protocol.hosted_workers import hosted_launch_of
-from switch_core.bridges.agent.protocol.stream import event_stream
+from switch_core.bridges.agent.protocol.stream import (
+ KEEPALIVE,
+ Frame,
+ encode_ws,
+ event_frames,
+ sse_stream,
+)
from switch_core.bridges.agent.registration_bootstrap import (
BOOTSTRAP_KEY_TYPE,
REGISTRATION_KEY_TYPES,
@@ -747,16 +759,16 @@ async def poll_events(
) -> EventResponse | Response:
"""Deliver the agent's events, as a push stream or a long poll.
- `Accept: text/event-stream` opens a connection and streams: catch-up from
- the client's cursor, then live delivery. Anything else falls back to the
- long poll, which is served from the same buffer so the two cannot diverge
- while both exist.
+ The live connection is the WebSocket at `/connection/ws`. `Accept:
+ text/event-stream` opens the same connection as a Server-Sent Events
+ stream, with its heartbeat on `POST /connection/beat`: that is how an agent
+ runtime built before the WebSocket connects (agent-protocol revision 7 and
+ older), and it is kept for those clients for a compatibility window. It
+ goes once no client still connects over it. Anything else falls back to
+ the long poll, served from the same buffer.
- The four declaration parameters are all optional and all default to None,
- meaning *unknown* (CHOO-1865). `protocol` previously defaulted to the
- server's own value, so a client that said nothing was read as having
- agreed — and since no shipped client sent it, the check had never once
- fired. Absent now records as unknown, and still connects.
+ The declaration parameters are all optional and all default to None,
+ meaning *unknown* (CHOO-1865): a client that says nothing still connects.
"""
if accept and "text/event-stream" in accept:
return await _open_event_stream(
@@ -795,12 +807,12 @@ def _resolve_start_cursor(
protocol: AgentCore,
agent_id: str,
start_from: str,
- last_event_id: str | None,
+ last_event_id: str | None = None,
) -> int:
- """Where a stream begins.
+ """Where a connection's events begin: the head, or a sequence number.
- `Last-Event-ID` wins when present: a reconnecting SSE client sends it
- automatically and it is the most accurate statement of what it processed.
+ `Last-Event-ID` wins when present: an event-stream client that reconnects
+ sends it, and it is the most accurate statement of what it processed.
"""
raw = last_event_id or start_from
if raw in ("", "head"):
@@ -814,7 +826,7 @@ def _resolve_start_cursor(
) from exc
-def _open_connection(
+def _register_connection(
*,
protocol: AgentCore,
agent: Agent,
@@ -825,6 +837,7 @@ def _open_connection(
cursor: int,
declaration: ClientDeclaration,
expected_generation: int | None,
+ transport: Transport,
) -> AgentConnection:
try:
conn = protocol.connections.open(
@@ -836,6 +849,7 @@ def _open_connection(
cursor=cursor,
declaration=declaration,
expected_generation=expected_generation,
+ transport=transport,
)
except SupersededReattachError as exc:
# Structured, like the refused heartbeat: this is the same ending, and
@@ -869,7 +883,7 @@ def _open_connection(
return conn
-async def _open_event_stream(
+async def _open_connection(
*,
agent: Agent,
protocol: AgentCore,
@@ -881,13 +895,20 @@ async def _open_event_stream(
spawn_capable: bool,
declaration: ClientDeclaration,
rooms: str | None,
- last_event_id: str | None,
expected_generation: int | None,
- worker_capability: str | None,
- host_boot_id: str | None,
- host_instance_id: str | None,
- worker_state_version: int | None,
-) -> StreamingResponse:
+ worker_capability: str | None = None,
+ host_boot_id: str | None = None,
+ host_instance_id: str | None = None,
+ worker_state_version: int | None = None,
+ last_event_id: str | None = None,
+ transport: Transport = "websocket",
+) -> tuple[AgentConnection, AsyncGenerator[Frame]]:
+ """Open or reattach the connection and claim its declared rooms.
+
+ Shared by every transport, so the socket and the event stream fence,
+ admit a hosted worker and claim rooms in exactly the same way. Returns
+ the connection and its frames, for the transport to encode.
+ """
if not connection_id:
raise HTTPException(
status_code=400,
@@ -907,8 +928,8 @@ async def _open_event_stream(
cursor = _resolve_start_cursor(protocol, agent.id, start_from, last_event_id)
- def open_connection() -> AgentConnection:
- return _open_connection(
+ def register() -> AgentConnection:
+ return _register_connection(
protocol=protocol,
agent=agent,
connection_id=connection_id,
@@ -918,11 +939,12 @@ async def _open_event_stream(
cursor=cursor,
declaration=declaration,
expected_generation=expected_generation,
+ transport=transport,
)
launch_id = hosted_launch_of(agent.metadata_)
if launch_id is None:
- conn = open_connection()
+ conn = register()
else:
# Admission, the open and the binding all happen under the launch
# lock, so no revision bump lands between the check and the bind.
@@ -940,7 +962,7 @@ async def _open_event_stream(
instance_id=host_instance_id,
state_version=worker_state_version,
)
- conn = open_connection()
+ conn = register()
if attach.takes_over is not None and attach.takes_over.id != conn.id:
protocol.connections.close(attach.takes_over.id, TAKEN_OVER)
protocol.connections.bind_worker(conn, attach.binding, attach.attached)
@@ -951,7 +973,7 @@ async def _open_event_stream(
# Built before anything below can yield, so it holds the generation this
# open produced; a reconnect during the bookkeeping supersedes it rather
# than being detached by it.
- stream = event_stream(
+ frames = event_frames(
conn=conn,
registry=protocol.connections,
buffer=protocol.event_buffer,
@@ -1013,9 +1035,55 @@ async def _open_event_stream(
# session rather than starting another, and an open that failed after
# registering its connection, retried on the same id, is still counted.
await protocol.sessions.started(agent, conn)
+ return conn, frames
+
+
+async def _open_event_stream(
+ *,
+ agent: Agent,
+ protocol: AgentCore,
+ config: SwitchConfig,
+ connection_id: str | None,
+ scope: str,
+ event_filter: str,
+ start_from: str,
+ spawn_capable: bool,
+ declaration: ClientDeclaration,
+ rooms: str | None,
+ last_event_id: str | None,
+ expected_generation: int | None,
+ worker_capability: str | None,
+ host_boot_id: str | None,
+ host_instance_id: str | None,
+ worker_state_version: int | None,
+) -> StreamingResponse:
+ """The connection as a Server-Sent Events stream, for an old client.
+ Opens exactly as the socket does and streams the same frames from the same
+ loop, so the two cannot drift while both exist. A refusal is the HTTP
+ error the socket would have sent as its `refused` frame.
+ """
+ _conn, frames = await _open_connection(
+ agent=agent,
+ protocol=protocol,
+ config=config,
+ connection_id=connection_id,
+ scope=scope,
+ event_filter=event_filter,
+ start_from=start_from,
+ spawn_capable=spawn_capable,
+ declaration=declaration,
+ rooms=rooms,
+ expected_generation=expected_generation,
+ worker_capability=worker_capability,
+ host_boot_id=host_boot_id,
+ host_instance_id=host_instance_id,
+ worker_state_version=worker_state_version,
+ last_event_id=last_event_id,
+ transport="sse",
+ )
return StreamingResponse(
- stream,
+ sse_stream(frames),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
@@ -1026,6 +1094,32 @@ async def _open_event_stream(
)
+def _record_beat(
+ protocol: AgentCore,
+ agent_id: str,
+ connection_id: str,
+ cursor: int,
+ generation: int | None,
+) -> AgentConnection:
+ """Count a client's beat, and adopt its cursor: a pong on the socket, or a
+ `POST /connection/beat` from an event-stream client.
+
+ The cursor is clamped to the buffer head first. The buffer is in memory,
+ so a restart resets the sequence while a client keeps reporting the
+ number it had reached, and neither `beat()` nor `confirm()` will move a
+ cursor backwards: adopting the stale number would skip every event up to
+ it, silently, and mark them consumed on the way past.
+
+ Raises the registry's refusal when the connection is no longer this
+ client's to beat for: another socket took it over (`taken_over`), or it
+ has no stream attached (`no_stream`), or it is gone.
+ """
+ cursor = min(cursor, protocol.event_buffer.head(agent_id))
+ conn = protocol.connections.beat(agent_id, connection_id, cursor, generation)
+ protocol.event_buffer.confirm(agent_id, conn.id, cursor)
+ return conn
+
+
@router.post("/{agent_id}/connection/beat")
async def connection_beat(
agent_id: str,
@@ -1033,49 +1127,233 @@ async def connection_beat(
agent: Annotated[Agent, Depends(get_agent_from_scope)],
protocol: Annotated[AgentCore, Depends(get_protocol)],
) -> dict[str, Any]:
- """The single per-connection heartbeat.
-
- Proves the client is alive and reports its cursor. Rejected when the
- connection is unknown, dead, has no stream attached, or belongs to another
- incarnation — an agent that can still make calls but is receiving nothing
- must be told, not left believing it is connected. A refusal carries a code
- beside its prose, because the remedies differ: `taken_over` is terminal for
- the client that receives it, and the rest are recovered by reopening.
+ """The heartbeat of a connection held over the event stream.
+
+ The socket carries its own heartbeat; this is for a client built before it,
+ kept with the event stream for the same compatibility window. Proves the
+ client is alive and reports its cursor. Rejected when the connection is
+ unknown, dead, has no stream attached, or belongs to another incarnation.
+ A refusal carries a code beside its prose, because the remedies differ:
+ `taken_over` is terminal for the client that receives it, and the rest are
+ recovered by reopening.
"""
- # A cursor above the buffer's head belongs to a previous life of this
- # process: the buffer is in memory, so a restart resets the sequence while
- # the client keeps beating the number it had reached. Both consumers below
- # only ever move a cursor forward, so adopting it undoes the rewind the
- # stream performs on resume — the connection then skips every event up to
- # the stale value and confirms events it was never delivered. Clamp it here,
- # where the untrusted value enters, rather than in either consumer.
- head = protocol.event_buffer.head(agent.id)
- cursor = min(req.cursor, head)
-
try:
- conn = protocol.connections.beat(
- agent.id, req.connection_id, cursor, req.generation
+ conn = _record_beat(
+ protocol, agent.id, req.connection_id, req.cursor, req.generation
)
except (
NoStreamAttachedError,
SupersededConnectionError,
UnfencedBeatError,
) as exc:
- # All three refuse the tick, and one of them means something the others
- # do not: a superseded tick is terminal for the client that sent it,
- # because reopening is itself a takeover and would pull the connection
- # back off the client that now holds it. Prose alone could not tell
- # them apart, so every refusal was answered with a reopen.
raise HTTPException(
status_code=409, detail={"code": exc.code, "message": str(exc)}
) from exc
except UnknownConnectionError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
-
- protocol.event_buffer.confirm(agent.id, conn.id, cursor)
return {"ok": True, "rooms": sorted(conn.rooms), "cursor": conn.cursor}
+# A WebSocket close code for each refusal that would otherwise be an HTTP
+# status: 4000 plus the status, so a client maps them onto one set of remedies.
+def _ws_close_code(status: int) -> int:
+ return 4000 + status
+
+
[email protected]("/{agent_id}/connection/ws")
+async def connection_socket(
+ websocket: WebSocket,
+ agent_id: str,
+ agent: Annotated[Agent, Depends(get_agent_from_scope)],
+ protocol: Annotated[AgentCore, Depends(get_protocol)],
+ config: Annotated[SwitchConfig, Depends(get_config)],
+ connection_id: Annotated[str | None, Query()] = None,
+ scope: Annotated[str, Query()] = "single",
+ event_filter: Annotated[str, Query(alias="filter")] = "all",
+ start_from: Annotated[str, Query()] = "head",
+ spawn_capable: Annotated[bool, Query()] = False,
+ protocol_version: Annotated[int | None, Query(alias="protocol")] = None,
+ protocol_accepts: Annotated[int | None, Query()] = None,
+ expected_generation: Annotated[int | None, Query()] = None,
+ client: Annotated[str | None, Query()] = None,
+ client_version: Annotated[str | None, Query()] = None,
+ rooms: Annotated[str | None, Query()] = None,
+ worker_capability: Annotated[
+ str | None, Header(alias="x-switch-worker-capability")
+ ] = None,
+ host_boot_id: Annotated[str | None, Header(alias="x-switch-host-boot-id")] = None,
+ host_instance_id: Annotated[
+ str | None, Header(alias="x-switch-host-instance-id")
+ ] = None,
+ worker_state_version: Annotated[
+ int | None, Header(alias="x-switch-worker-state-version")
+ ] = None,
+) -> None:
+ """The agent's connection over one WebSocket: events down, heartbeats up.
+
+ Opens exactly as the event stream does (same parameters, same fencing,
+ same room claims) and carries the same frames, as JSON
+ `{"event", "data", "id"}`. Authenticated once, when the socket opens.
+
+ The heartbeat runs the other way round from `/connection/beat`: the server
+ sends a `ping` frame every heartbeat interval and the client answers
+ `{"type": "pong", "cursor": n}`, which counts as a beat. A client that
+ stops answering lapses like one that stops beating. No request per beat,
+ and no token lookup.
+
+ A refusal on opening is sent as a `refused` frame carrying the status and
+ detail an HTTP request would have been answered with, then the socket closes
+ with code 4000 plus that status.
+
+ A hosted agent's worker opens it as it opens the stream, with its
+ capability and host identity in the `X-Switch-Worker-*` and
+ `X-Switch-Host-*` headers, and is admitted and refused by the same rules
+ (agent-protocol 7). Its frames come down the socket; its up-calls stay
+ HTTP requests.
+ """
+ await websocket.accept()
+ try:
+ conn, frames = await _open_connection(
+ agent=agent,
+ protocol=protocol,
+ config=config,
+ connection_id=connection_id,
+ scope=scope,
+ event_filter=event_filter,
+ start_from=start_from,
+ spawn_capable=spawn_capable,
+ declaration=ClientDeclaration(
+ speaks=protocol_version,
+ accepts=protocol_accepts,
+ artifact=client,
+ version=client_version,
+ ),
+ rooms=rooms,
+ expected_generation=expected_generation,
+ worker_capability=worker_capability,
+ host_boot_id=host_boot_id,
+ host_instance_id=host_instance_id,
+ worker_state_version=worker_state_version,
+ )
+ except HTTPException as exc:
+ await websocket.send_json(
+ {
+ "event": "refused",
+ "data": {"status": exc.status_code, "detail": exc.detail},
+ }
+ )
+ await websocket.close(code=_ws_close_code(exc.status_code))
+ return
+
+ # Bounded, so a client that reads slowly holds the stream back rather
+ # than growing this queue.
+ outbox: asyncio.Queue[Frame] = asyncio.Queue(maxsize=_SOCKET_OUTBOX)
+ pump = asyncio.create_task(_pump_frames(frames, outbox))
+ pongs = asyncio.create_task(
+ _receive_pongs(websocket, protocol, agent.id, conn.id, conn.stream_generation)
+ )
+ loop = asyncio.get_running_loop()
+ next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS
+ try:
+ # One writer: a WebSocket must not be sent to from two tasks at once,
+ # so frames and pings both go out from this loop.
+ while not pump.done() or not outbox.empty():
+ if pongs.done():
+ refusal = pongs.result()
+ if refusal is not None:
+ await websocket.send_json(
+ {
+ "event": "evicted",
+ "data": {
+ "code": getattr(refusal, "code", "closed"),
+ "reason": str(refusal),
+ "room_id": None,
+ },
+ }
+ )
+ return
+ try:
+ frame = await asyncio.wait_for(
+ outbox.get(), timeout=max(0.0, next_ping - loop.time())
+ )
+ except TimeoutError:
+ frame = None
+ if frame is not None and frame is not KEEPALIVE:
+ await websocket.send_json(encode_ws(frame))
+ if loop.time() >= next_ping:
+ await websocket.send_json({"event": "ping", "data": {}})
+ next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS
+ except WebSocketDisconnect:
+ return
+ except RuntimeError:
+ # The server closed the socket under us (uvicorn does on shutdown), so
+ # a send raises this rather than WebSocketDisconnect. Same ending.
+ return
+ finally:
+ # The pump owns the stream: cancelling it runs the stream's own
+ # cleanup (detaching it from the connection) inside that task.
+ pump.cancel()
+ pongs.cancel()
+ failure = pump.exception() if pump.done() and not pump.cancelled() else None
+ if failure is not None:
+ logger.error(
+ "[STREAM] agent=%s connection=%s delivery failed, closing its socket",
+ agent.id,
+ conn.id,
+ exc_info=failure,
+ )
+ try:
+ await websocket.close(code=1011 if failure is not None else 1000)
+ except RuntimeError:
+ pass # the client closed it first
+
+
+# Frames buffered between the stream and the socket.
+_SOCKET_OUTBOX = 64
+
+
+async def _pump_frames(
+ frames: AsyncGenerator[Frame], outbox: asyncio.Queue[Frame]
+) -> None:
+ # Closed here: cancelled while waiting on the outbox, the stream would
+ # otherwise detach only when the generator is garbage collected.
+ try:
+ async for frame in frames:
+ await outbox.put(frame)
+ finally:
+ await frames.aclose()
+
+
+async def _receive_pongs(
+ websocket: WebSocket,
+ protocol: AgentCore,
+ agent_id: str,
+ connection_id: str,
+ generation: int,
+) -> ConnectionError_ | None:
+ """Turn each pong into a beat, until the client goes or a beat is refused.
+
+ Returns the refusal, so the socket can say which one before it closes:
+ `taken_over` is final for this client, the rest are recovered by
+ reconnecting. Returns None when the client simply went away.
+ """
+ while True:
+ try:
+ message = await websocket.receive_json()
+ except (WebSocketDisconnect, ValueError, RuntimeError):
+ # Gone, or sent something that is not JSON: either way no more
+ # beats are coming on this socket.
+ return None
+ if not isinstance(message, dict) or message.get("type") != "pong":
+ continue
+ raw = message.get("cursor", 0)
+ cursor = raw if isinstance(raw, int) and raw >= 0 else 0
+ try:
+ _record_beat(protocol, agent_id, connection_id, cursor, generation)
+ except ConnectionError_ as refusal:
+ return refusal
+
+
def _current_connection(
protocol: AgentCore,
agent: Agent,
diff --git a/core/switch_core/bridges/agent/api/schemas.py b/core/switch_core/bridges/agent/api/schemas.py
index 1b1573453..3040f1815 100644
--- a/core/switch_core/bridges/agent/api/schemas.py
+++ b/core/switch_core/bridges/agent/api/schemas.py
@@ -139,11 +139,12 @@ class ConnectionPlacementsRequest(BaseModel):
class ConnectionBeatRequest(BaseModel):
- """The single client tick that keeps a connection alive (CHOO-1857).
+ """The heartbeat of a connection held over the event stream (CHOO-1857).
- Replaces /connection/renew, /watch/heartbeat and /leases/renew: it proves
- the client is alive *and* consuming, and reports how far it has read so the
- event buffer knows what has been seen.
+ Kept for clients built before the WebSocket, whose socket carries the
+ heartbeat itself, for a compatibility window. Proves the client is alive
+ *and* consuming, and reports how far it has read so the event buffer knows
+ what has been seen.
"""
connection_id: str
@@ -153,7 +154,7 @@ class ConnectionBeatRequest(BaseModel):
#: been displaced still holds the id and the token, and is otherwise
#: indistinguishable from the one that replaced it. Null is accepted only
#: while the connection's holder is a client built before the fence existed
- #: — unknown, not current; from a holder that declares the revision which
+ #: (unknown, not current); from a holder that declares the revision which
#: carries it, a tick without one is refused.
generation: int | None = None
diff --git a/core/switch_core/bridges/agent/auth.py b/core/switch_core/bridges/agent/auth.py
index f670b0d8f..edf161bbe 100644
--- a/core/switch_core/bridges/agent/auth.py
+++ b/core/switch_core/bridges/agent/auth.py
@@ -1,15 +1,17 @@
from __future__ import annotations
import hashlib
+import json
import logging
import re
from dataclasses import dataclass
from typing import Protocol
import jwt
-from fastapi import HTTPException, Request
+from fastapi import HTTPException
from jwt import PyJWKClient
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from starlette.requests import HTTPConnection
from starlette.responses import JSONResponse, Response
from starlette.types import ASGIApp, Receive, Scope, Send
@@ -248,10 +250,9 @@ class BearerAuthMiddleware:
return
if not auth_header.startswith("Bearer "):
- response = Response(
- "Missing or invalid Authorization header", status_code=401
+ await _unauthorized(
+ scope, receive, send, "Missing or invalid Authorization header"
)
- await response(scope, receive, send)
return
token = auth_header[7:]
@@ -325,8 +326,7 @@ class BearerAuthMiddleware:
await self.app(scope, receive, send)
return
- response = Response("Invalid credentials", status_code=401)
- await response(scope, receive, send)
+ await _unauthorized(scope, receive, send, "Invalid credentials")
async def _serve_controller(
self,
@@ -531,12 +531,36 @@ class BearerAuthMiddleware:
return await self._agent_store.get_by_oauth_client_id(session, client_id)
-def _controller_refusal(code: str, message: str, status_code: int) -> Response:
- return JSONResponse(
+def _controller_refusal(code: str, message: str, status_code: int) -> ASGIApp:
+ """A controller refusal, in the contract's error envelope.
+
+ On a WebSocket, which cannot read a refused handshake (see
+ `_unauthorized`), the same refusal the agent connection sends: a `refused`
+ frame carrying the code, then a close with 4000 plus the status.
+ """
+ response = JSONResponse(
{"error": {"code": code, "message": message, "retryable": False}},
status_code=status_code,
)
+ async def refuse(scope: Scope, receive: Receive, send: Send) -> None:
+ if scope["type"] != "websocket":
+ await response(scope, receive, send)
+ return
+ await receive() # the client's websocket.connect
+ await send({"type": "websocket.accept"})
+ frame = {
+ "event": "refused",
+ "data": {
+ "status": status_code,
+ "detail": {"code": code, "message": message},
+ },
+ }
+ await send({"type": "websocket.send", "text": json.dumps(frame)})
+ await send({"type": "websocket.close", "code": 4000 + status_code})
+
+ return refuse
+
def _is_agent_route(path: str) -> bool:
if path.startswith(_AGENT_SESSIONS_PREFIX):
@@ -553,13 +577,35 @@ def _path_agent(path: str) -> tuple[str | None, str]:
return match["segment"], match["rest"] or ""
+async def _unauthorized(
+ scope: Scope, receive: Receive, send: Send, reason: str
+) -> None:
+ """Refuse an unauthenticated request in a form its client can read.
+
+ An HTTP 401. A WebSocket client cannot read the status of a handshake that
+ was refused (the browser-style API hides it), so it would see only a
+ failed connection and retry for ever. A WebSocket is accepted and closed at
+ once with 4401 instead: 4000 plus the status, the same mapping the agent
+ connection uses for every refusal.
+ """
+ if scope["type"] == "websocket":
+ await receive() # the client's websocket.connect
+ await send({"type": "websocket.accept"})
+ await send({"type": "websocket.close", "code": 4401, "reason": reason})
+ return
+ await Response(reason, status_code=401)(scope, receive, send)
+
+
def _is_public_path(path: str) -> bool:
return any(path == p or path.startswith(p + "/") for p in PUBLIC_PATH_PREFIXES)
-def get_agent_from_scope(request: Request) -> Agent:
- """Get authenticated agent from request scope (set by middleware)."""
- agent: object = request.scope.get("agent")
+def get_agent_from_scope(connection: HTTPConnection) -> Agent:
+ """Get authenticated agent from the connection's scope (set by middleware).
+
+ Any HTTP connection, so a WebSocket route can use it too.
+ """
+ agent: object = connection.scope.get("agent")
if not isinstance(agent, Agent):
raise HTTPException(status_code=401, detail="Not authenticated")
return agent
diff --git a/core/switch_core/bridges/agent/protocol/agent_connections.py b/core/switch_core/bridges/agent/protocol/agent_connections.py
index e11ce9d4b..182535d25 100644
--- a/core/switch_core/bridges/agent/protocol/agent_connections.py
+++ b/core/switch_core/bridges/agent/protocol/agent_connections.py
@@ -62,6 +62,10 @@ logger = logging.getLogger(__name__)
Scope = Literal["single", "all"]
DeliveryFilter = Literal["all", "addressed"]
+# How a connection's stream reaches its client. `sse` is the event stream an
+# agent runtime built before the WebSocket still asks for, served for a
+# compatibility window and told apart so we can see those clients drain.
+Transport = Literal["websocket", "sse"]
# Refuse a client that cannot meet this server's protocol rather than degrading
# in ways neither side can see. The runtime lives on the user's machine and
@@ -405,6 +409,8 @@ class AgentConnection:
# agent belongs to that no sibling has claimed.
rooms: set[str] = field(default_factory=set)
stream_attached: bool = False
+ # What the attached stream travels over, as of the last attach.
+ stream_transport: Transport = "websocket"
# How many heartbeats this connection has received. Diagnostic: it is the
# difference between a client that never started beating and one that beat
# and then stopped, which the timestamp alone cannot tell you.
@@ -505,6 +511,7 @@ class AgentConnectionRegistry:
cursor: int,
declaration: ClientDeclaration,
expected_generation: int | None,
+ transport: Transport = "websocket",
) -> AgentConnection:
"""Open a connection, or reattach to one the client already owns.
@@ -563,18 +570,24 @@ class AgentConnectionRegistry:
existing.last_beat = time.monotonic()
existing.closure = None
existing.stream_attached = True
+ existing.stream_transport = transport
existing.stream_generation = self._new_incarnation()
+ # Wake the stream this one replaces, so its client hears it was
+ # taken over now rather than on the stream's next idle tick.
+ existing.wake.set()
# A reattach can come from an upgraded client, so the declaration
# is replaced rather than kept. The connection outlives the socket;
# what is on the other end of it need not.
existing.declaration = declaration
self._reset_worker(existing)
logger.info(
- "[CONN] reattached agent=%s connection=%s scope=%s generation=%s",
+ "[CONN] reattached agent=%s connection=%s scope=%s generation=%s "
+ "transport=%s",
agent_id,
connection_id,
scope,
existing.stream_generation,
+ transport,
)
return existing
@@ -593,6 +606,7 @@ class AgentConnectionRegistry:
last_beat=now,
opened_at=now,
stream_attached=True,
+ stream_transport=transport,
stream_generation=self._new_incarnation(),
declaration=declaration,
)
@@ -600,7 +614,7 @@ class AgentConnectionRegistry:
owned.add(connection_id)
logger.info(
"[CONN] opened agent=%s connection=%s scope=%s filter=%s spawn=%s "
- "client=%s version=%s protocol=%s",
+ "client=%s version=%s protocol=%s transport=%s",
agent_id,
connection_id,
scope,
@@ -611,6 +625,7 @@ class AgentConnectionRegistry:
f"{floor}-{declaration.speaks}"
if declaration.declares_protocol
else "unknown",
+ transport,
)
return conn
diff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.py
index 75b68192c..a872e768c 100644
--- a/core/switch_core/bridges/agent/protocol/stream.py
+++ b/core/switch_core/bridges/agent/protocol/stream.py
@@ -1,13 +1,15 @@
-"""Server-sent event stream for agent connections (CHOO-1857).
+"""The frames of an agent connection (CHOO-1857).
-Turns a connection plus the event buffer into a `text/event-stream`: catch-up
+Turns a connection plus the event buffer into a sequence of frames: catch-up
from the client's cursor, then live delivery as events are appended. The client
-never asks again — it opens once and reads.
-
-Every event carries its sequence number as the SSE `id`, so a client that
-reconnects sends `Last-Event-ID` and resumes exactly where it stopped. Gaps are
-reported as their own event rather than skipped: a client that has missed
-events must never see a stream that looks complete.
+never asks again; it opens once and reads. `api/handlers.py` sends them over
+the WebSocket, or, for a client built before it, as a Server-Sent Events
+stream: both encode the same frames from the same loop, so they cannot drift.
+
+Every event carries its sequence number, so a client that reconnects names the
+last one it processed and resumes exactly where it stopped. Gaps are reported
+as their own event rather than skipped: a client that has missed events must
+never see a stream that looks complete.
"""
from __future__ import annotations
@@ -16,7 +18,8 @@ import asyncio
import json
import logging
import time
-from collections.abc import AsyncIterator
+from collections.abc import AsyncGenerator, AsyncIterator
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from switch_core.bridges.agent.protocol.agent_connections import (
@@ -42,9 +45,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-# How long to wait for an event before writing a keepalive comment. This exists
-# only to stop proxies dropping an idle connection — liveness comes from the
-# client's heartbeat, never from this.
+# How long the loop waits for work before yielding an idle tick, so whoever
+# reads it gets a chance to do its own periodic work.
KEEPALIVE_INTERVAL_SECONDS = 15.0
# Cap on how many buffered events are written in one batch, so a large catch-up
@@ -52,15 +54,67 @@ KEEPALIVE_INTERVAL_SECONDS = 15.0
CATCH_UP_BATCH = 200
-def _frame(event: str, data: dict[str, Any], *, seq: int | None = None) -> bytes:
+@dataclass(frozen=True)
+class Frame:
+ """One message on an agent's event stream, before it is encoded.
+
+ The loop decides what to send; the transport decides how it is encoded.
+ """
+
+ event: str
+ data: dict[str, Any]
+ seq: int | None = None
+
+
+# Not a message: the loop's signal that it waited a keepalive interval with
+# nothing to send. The WebSocket has its own ping and drops it; the event
+# stream writes a comment, which only stops proxies dropping an idle
+# connection. Liveness comes from the client's beat, never from this.
+KEEPALIVE = Frame("keepalive", {})
+
+
+def _frame(event: str, data: dict[str, Any], *, seq: int | None = None) -> Frame:
+ return Frame(event, data, seq)
+
+
+def encode_ws(frame: Frame) -> dict[str, Any]:
+ """A frame as a WebSocket message. `id` is the event's sequence number,
+ which a reconnecting client passes back as `start_from`."""
+ message: dict[str, Any] = {"event": frame.event, "data": frame.data}
+ if frame.seq is not None:
+ message["id"] = frame.seq
+ return message
+
+
+def encode_sse(frame: Frame) -> bytes:
+ """A frame as a Server-Sent Event. `id` is the event's sequence number,
+ which a reconnecting client sends back as `Last-Event-ID`."""
+ if frame is KEEPALIVE:
+ return b": keepalive\n\n"
lines = []
- if seq is not None:
- lines.append(f"id: {seq}")
- lines.append(f"event: {event}")
- lines.append(f"data: {json.dumps(data, separators=(',', ':'))}")
+ if frame.seq is not None:
+ lines.append(f"id: {frame.seq}")
+ lines.append(f"event: {frame.event}")
+ lines.append(f"data: {json.dumps(frame.data, separators=(',', ':'))}")
return ("\n".join(lines) + "\n\n").encode()
+async def sse_stream(frames: AsyncGenerator[Frame]) -> AsyncIterator[bytes]:
+ """The connection's frames as a `text/event-stream` body.
+
+ Kept for clients built before the WebSocket (agent-protocol revision 7 and
+ older), for a compatibility window: it goes once no client still connects
+ over it.
+ """
+ try:
+ async for frame in frames:
+ yield encode_sse(frame)
+ finally:
+ # Closed here, so the stream detaches from its connection when the
+ # client goes rather than when the generator is garbage collected.
+ await frames.aclose()
+
+
def _connection_state(conn: AgentConnection) -> dict[str, Any]:
"""The first frame of every stream, and where the server declares itself.
@@ -104,13 +158,14 @@ def _eviction(closure: Closure) -> dict[str, Any]:
}
-def event_stream(
+def event_frames(
*,
conn: AgentConnection,
registry: AgentConnectionRegistry,
buffer: EventBuffer,
approvals: ApprovalOutcomes | None,
-) -> AsyncIterator[bytes]:
+) -> AsyncGenerator[Frame]:
+ """The connection's frames, for a transport to encode."""
return _event_stream(
conn=conn,
registry=registry,
@@ -127,8 +182,8 @@ async def _event_stream(
buffer: EventBuffer,
approvals: ApprovalOutcomes | None,
generation: int,
-) -> AsyncIterator[bytes]:
- """Yield SSE frames for a connection until its stream is superseded or it dies."""
+) -> AsyncGenerator[Frame]:
+ """Yield frames for a connection until its stream is superseded or it dies."""
if conn.stream_generation != generation:
return
agent_id = conn.agent_id
@@ -348,7 +403,7 @@ async def _event_stream(
# cursor still where it started.
conn.wake.clear()
if not conn.rooms and not await _wait_for_wake(conn):
- yield b": keepalive\n\n"
+ yield KEEPALIVE
continue
# Where counting starts for a room nothing is counting yet. It is
@@ -454,7 +509,7 @@ async def _event_stream(
continue
if not await _wait_for_work(bell, conn):
- yield b": keepalive\n\n"
+ yield KEEPALIVE
finally:
if unsubscribe_outcomes is not None:
unsubscribe_outcomes()
diff --git a/core/tests/benchmarks/client.py b/core/tests/benchmarks/client.py
index a0593b9c3..2d90fa251 100644
--- a/core/tests/benchmarks/client.py
+++ b/core/tests/benchmarks/client.py
@@ -7,8 +7,8 @@ is proving the harness itself works — that the server serves, that a stream
delivers, and that the instrumentation points fire — without needing the whole
Node side up first.
-It speaks only the part of the protocol those checks need: open the stream,
-heartbeat, read frames.
+It speaks only the part of the protocol those checks need: open the
+connection's WebSocket, answer the server's pings, read frames.
"""
from __future__ import annotations
@@ -17,12 +17,9 @@ import asyncio
import json
from dataclasses import dataclass
from typing import Any
+from urllib.parse import urlencode
-import httpx
-
-from switch_core.bridges.agent.protocol.agent_connections import (
- HEARTBEAT_INTERVAL_SECONDS,
-)
+import websockets
@dataclass(frozen=True, slots=True)
@@ -33,7 +30,7 @@ class StreamFrame:
class AgentConnection:
- """One SSE connection, with the heartbeat the server requires to keep it."""
+ """One agent connection over its WebSocket, answering the server's pings."""
def __init__(
self,
@@ -55,9 +52,7 @@ class AgentConnection:
self._spawn_capable = spawn_capable
self._frames: asyncio.Queue[StreamFrame] = asyncio.Queue()
self._cursor = 0
- self._client: httpx.AsyncClient | None = None
self._reader: asyncio.Task[None] | None = None
- self._beater: asyncio.Task[None] | None = None
self._opened = asyncio.Event()
self._failure: BaseException | None = None
@@ -66,11 +61,6 @@ class AgentConnection:
return self._cursor
async def open(self, timeout: float) -> None:
- self._client = httpx.AsyncClient(
- base_url=self._base_url,
- headers={"Authorization": f"Bearer {self._api_key}"},
- timeout=httpx.Timeout(None),
- )
self._reader = asyncio.create_task(self._read())
try:
await asyncio.wait_for(self._opened.wait(), timeout)
@@ -79,23 +69,17 @@ class AgentConnection:
raise
if self._failure is not None:
raise self._failure
- self._beater = asyncio.create_task(self._beat())
async def close(self) -> None:
- for task in (self._beater, self._reader):
- if task is not None:
- task.cancel()
- for task in (self._beater, self._reader):
- if task is not None:
- try:
- await task
- except asyncio.CancelledError:
- pass
- if self._client is not None:
- await self._client.aclose()
+ if self._reader is not None:
+ self._reader.cancel()
+ try:
+ await self._reader
+ except asyncio.CancelledError:
+ pass
async def next_frame(self, event: str, timeout: float) -> StreamFrame:
- """The next frame of a given type, discarding keepalives and others.
+ """The next frame of a given type, discarding pings and others.
Raises on timeout rather than returning None: every caller here is
asserting that something was delivered, and a None would turn a
@@ -114,38 +98,44 @@ class AgentConnection:
return frame
async def _read(self) -> None:
- assert self._client is not None
- params = {
- "connection_id": self._connection_id,
- "scope": self._scope,
- "filter": self._filter,
- "spawn_capable": str(self._spawn_capable).lower(),
- }
+ query = urlencode(
+ {
+ "connection_id": self._connection_id,
+ "scope": self._scope,
+ "filter": self._filter,
+ "spawn_capable": str(self._spawn_capable).lower(),
+ }
+ )
+ url = (
+ self._base_url.replace("http", "ws", 1)
+ + f"/agents/{self._agent_id}/connection/ws?{query}"
+ )
try:
- async with self._client.stream(
- "GET",
- f"/agents/{self._agent_id}/events",
- params=params,
- headers={"Accept": "text/event-stream"},
- ) as response:
- if response.status_code != 200:
- body = await response.aread()
- self._failure = RuntimeError(
- f"event stream refused with {response.status_code}: "
- f"{body.decode(errors='replace')}"
- )
+ async with websockets.connect(
+ url, additional_headers={"Authorization": f"Bearer {self._api_key}"}
+ ) as socket:
+ async for raw in socket:
+ message = json.loads(raw)
+ event = message.get("event")
+ if event == "refused":
+ self._failure = RuntimeError(
+ f"connection refused: {message.get('data')}"
+ )
+ self._opened.set()
+ return
self._opened.set()
- return
- self._opened.set()
- block: list[str] = []
- async for line in response.aiter_lines():
- if line:
- block.append(line)
- continue
- frame = _parse(block)
- block = []
- if frame is None:
+ if event == "ping":
+ # The heartbeat: the answer is what keeps the
+ # connection alive.
+ await socket.send(
+ json.dumps({"type": "pong", "cursor": self._cursor})
+ )
continue
+ frame = StreamFrame(
+ event=event,
+ sequence=message.get("id"),
+ data=message.get("data") or {},
+ )
if frame.sequence is not None:
self._cursor = frame.sequence
await self._frames.put(frame)
@@ -154,35 +144,3 @@ class AgentConnection:
except Exception as exc: # noqa: BLE001 - surfaced through open()/next_frame
self._failure = exc
self._opened.set()
-
- async def _beat(self) -> None:
- assert self._client is not None
- while True:
- await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS)
- response = await self._client.post(
- f"/agents/{self._agent_id}/connection/beat",
- json={"connection_id": self._connection_id, "cursor": self._cursor},
- )
- if response.status_code != 200:
- raise RuntimeError(
- f"heartbeat for {self._connection_id} rejected with "
- f"{response.status_code}: {response.text}"
- )
-
-
-def _parse(block: list[str]) -> StreamFrame | None:
- event = None
- data = None
- sequence = None
- for line in block:
- if line.startswith(":"):
- continue
- if line.startswith("id: "):
- sequence = int(line[4:])
- elif line.startswith("event: "):
- event = line[7:]
- elif line.startswith("data: "):
- data = json.loads(line[6:])
- if event is None or not isinstance(data, dict):
- return None
- return StreamFrame(event=event, sequence=sequence, data=data)
diff --git a/core/tests/benchmarks/test_baseline.py b/core/tests/benchmarks/test_baseline.py
index bd4ef1b0d..05d43493c 100644
--- a/core/tests/benchmarks/test_baseline.py
+++ b/core/tests/benchmarks/test_baseline.py
@@ -380,7 +380,9 @@ async def test_baseline_idles_at_session_count(
)
lines.append(
f"{rooms} idle session(s) over {watched:.1f}s: "
- f"watcher heartbeats {requests['agents/connection/beat']}, "
+ # Heartbeats ride the connection's WebSocket now, so they are no
+ # longer requests; anything left on this route is an old client.
+ f"heartbeat requests {requests.get('agents/connection/beat', 0)}, "
f"session host requests {reports}, "
f"all requests {sum(requests.values())} ({routes}); "
f"{statements} database statements; CPU driver "
diff --git a/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py b/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py
index 501f01d20..bb5194604 100644
--- a/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py
+++ b/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py
@@ -1,10 +1,9 @@
-"""A heartbeat cursor is clamped to the buffer head before anything adopts it.
+"""A heartbeat's cursor is clamped to the buffer head before anything adopts it.
The event buffer is in memory, so restarting switch-core resets the sequence to
zero while clients keep beating the number they had reached. Nothing downstream
can undo that: `beat()` and `confirm()` both refuse to move a cursor backwards,
-and the stream's own rewind on resume is re-poisoned by the next heartbeat two
-seconds later.
+and the stream's own rewind on resume is re-poisoned by the next heartbeat.
The result is silent. The connection is claimed, heartbeats succeed and posting
works, so the session looks healthy while every event up to the stale cursor is
@@ -13,11 +12,9 @@ skipped — and `confirm()` marks those events consumed on the way past.
from __future__ import annotations
-from types import SimpleNamespace
from typing import Any
-from switch_core.bridges.agent.api.handlers import connection_beat
-from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
+from switch_core.bridges.agent.api.handlers import _record_beat
from switch_core.bridges.agent.protocol.agent_connections import (
PROTOCOL_VERSION,
AgentConnectionRegistry,
@@ -67,17 +64,11 @@ def _connect(protocol: _Protocol, cursor: int = 0) -> Any:
return conn
-async def _beat(protocol: _Protocol, cursor: int) -> Any:
- # Fenced with the only incarnation these tests ever open. A client that
- # declares the current revision and then ticks without one is refused, and
- # nothing here is about that.
+def _beat(protocol: _Protocol, cursor: int) -> Any:
+ # Fenced with the only incarnation these tests ever open, as the socket
+ # does with the incarnation it opened.
current = protocol.connections.require(AGENT_ID, CONN_ID).stream_generation
- return await connection_beat(
- AGENT_ID,
- ConnectionBeatRequest(connection_id=CONN_ID, cursor=cursor, generation=current),
- SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]
- protocol, # type: ignore[arg-type]
- )
+ return _record_beat(protocol, AGENT_ID, CONN_ID, cursor, current) # type: ignore[arg-type]
async def test_a_cursor_past_the_head_is_pulled_back_to_it() -> None:
@@ -86,11 +77,11 @@ async def test_a_cursor_past_the_head_is_pulled_back_to_it() -> None:
protocol = _Protocol()
conn = _connect(protocol)
- result = await _beat(protocol, cursor=9)
+ beaten = _beat(protocol, cursor=9)
assert protocol.event_buffer.head(AGENT_ID) == 0
assert conn.cursor == 0
- assert result["cursor"] == 0
+ assert beaten.cursor == 0
async def test_the_first_event_after_a_restart_is_still_delivered() -> None:
@@ -101,7 +92,7 @@ async def test_the_first_event_after_a_restart_is_still_delivered() -> None:
conn = _connect(protocol)
protocol.connections.claim_room(conn, ROOM_ID, takeover=True)
- await _beat(protocol, cursor=9)
+ _beat(protocol, cursor=9)
protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())
delivered = protocol.event_buffer.read_from(AGENT_ID, conn.cursor, rooms={ROOM_ID})
@@ -114,7 +105,7 @@ async def test_a_cursor_within_the_buffer_is_left_alone() -> None:
for _ in range(3):
protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())
- await _beat(protocol, cursor=2)
+ _beat(protocol, cursor=2)
assert conn.cursor == 2
@@ -129,7 +120,7 @@ async def test_the_clamped_value_is_what_gets_confirmed() -> None:
lambda agent_id, connection_id, cursor: confirmed.append(cursor)
)
- await _beat(protocol, cursor=9)
+ _beat(protocol, cursor=9)
assert confirmed == [0]
@@ -139,6 +130,6 @@ async def test_a_heartbeat_from_the_old_boot_cannot_undo_the_gap_position():
protocol.event_buffer = EventBuffer(sequence_base=2 << 32)
sequence = protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())
conn = _connect(protocol, cursor=2 << 32)
- await _beat(protocol, cursor=(1 << 32) + 2)
+ _beat(protocol, cursor=(1 << 32) + 2)
assert conn.cursor == 2 << 32
assert protocol.event_buffer.read_from(AGENT_ID, conn.cursor)[0].seq == sequence
diff --git a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
index 48e98e3e1..ddf493796 100644
--- a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
+++ b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
@@ -17,12 +17,14 @@ from typing import Any
import pytest
from fastapi import HTTPException
-from switch_core.bridges.agent.api.handlers import connection_beat
+from switch_core.bridges.agent.api.handlers import _record_beat, connection_beat
from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
from switch_core.bridges.agent.protocol.agent_connections import (
PROTOCOL_VERSION,
AgentConnectionRegistry,
ClientDeclaration,
+ NoStreamAttachedError,
+ SupersededConnectionError,
)
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
@@ -49,7 +51,47 @@ def _open(protocol: _Protocol, *, speaks: int | None = PROTOCOL_VERSION) -> Any:
)
-async def _beat(protocol: _Protocol, generation: int | None) -> Any:
+def _beat(protocol: _Protocol, generation: int | None) -> Any:
+ return _record_beat(protocol, AGENT_ID, CONN_ID, 0, generation) # type: ignore[arg-type]
+
+
+def test_a_superseded_tick_is_refused_as_a_takeover() -> None:
+ protocol = _Protocol()
+ displaced = _open(protocol).stream_generation
+ _open(protocol) # the winner attaches; the incarnation bumps
+
+ with pytest.raises(SupersededConnectionError) as caught:
+ _beat(protocol, displaced)
+
+ # The same code the displaced stream is evicted with: one ending reaching
+ # the client by whichever door is still open to it.
+ assert caught.value.code == "taken_over"
+ # The prose still names both incarnations for whoever reads it.
+ assert str(displaced) in str(caught.value)
+ assert str(displaced + 1) in str(caught.value)
+
+
+def test_a_tick_with_no_stream_is_refused_as_something_to_reopen() -> None:
+ protocol = _Protocol()
+ conn = _open(protocol)
+ protocol.connections.detach_stream(conn, conn.stream_generation)
+
+ with pytest.raises(NoStreamAttachedError) as caught:
+ _beat(protocol, conn.stream_generation)
+
+ # Recoverable, and distinct from the takeover: this client still holds the
+ # connection and only has to reconnect.
+ assert caught.value.code == "no_stream"
+
+
+# ── Over HTTP, for a client on the event stream ─────────────────────────────
+#
+# A client built before the WebSocket beats with `POST /connection/beat`, kept
+# with the event stream for a compatibility window. Its refusals are the same
+# ones, answered as HTTP with the code beside the prose.
+
+
+async def _post_beat(protocol: _Protocol, generation: int | None) -> Any:
return await connection_beat(
AGENT_ID,
ConnectionBeatRequest(connection_id=CONN_ID, cursor=0, generation=generation),
@@ -58,52 +100,56 @@ async def _beat(protocol: _Protocol, generation: int | None) -> Any:
)
-async def test_a_superseded_tick_is_refused_as_a_takeover() -> None:
+async def test_a_superseded_beat_is_answered_409_taken_over() -> None:
protocol = _Protocol()
displaced = _open(protocol).stream_generation
- _open(protocol) # the winner attaches; the incarnation bumps
+ _open(protocol)
with pytest.raises(HTTPException) as caught:
- await _beat(protocol, displaced)
+ await _post_beat(protocol, displaced)
- # The same code the displaced stream is evicted with: one ending reaching
- # the client by whichever door is still open to it.
assert caught.value.status_code == 409
assert caught.value.detail["code"] == "taken_over" # type: ignore[index]
- # The prose stays, and still names both incarnations for whoever reads it.
message = caught.value.detail["message"] # type: ignore[index]
assert str(displaced) in message and str(displaced + 1) in message
-async def test_a_tick_with_no_stream_is_refused_as_something_to_reopen() -> None:
+async def test_a_beat_with_no_stream_is_answered_409_no_stream() -> None:
protocol = _Protocol()
conn = _open(protocol)
protocol.connections.detach_stream(conn, conn.stream_generation)
with pytest.raises(HTTPException) as caught:
- await _beat(protocol, conn.stream_generation)
+ await _post_beat(protocol, conn.stream_generation)
- # Recoverable, and distinct from the takeover: this client still holds the
- # connection and only has to reopen the stream.
assert caught.value.status_code == 409
assert caught.value.detail["code"] == "no_stream" # type: ignore[index]
-async def test_an_unfenced_tick_from_a_fenced_holder_is_refused_on_its_own_code() -> (
+async def test_an_unfenced_beat_from_a_fenced_holder_is_refused_on_its_own_code() -> (
None
):
protocol = _Protocol()
_open(protocol)
with pytest.raises(HTTPException) as caught:
- await _beat(protocol, None)
+ await _post_beat(protocol, None)
assert caught.value.status_code == 409
assert caught.value.detail["code"] == "unfenced" # type: ignore[index]
+async def test_a_beat_for_an_unknown_connection_is_answered_404() -> None:
+ protocol = _Protocol()
+
+ with pytest.raises(HTTPException) as caught:
+ await _post_beat(protocol, None)
+
+ assert caught.value.status_code == 404
+
+
async def test_a_client_that_cannot_be_fenced_still_beats() -> None:
protocol = _Protocol()
_open(protocol, speaks=None)
- assert (await _beat(protocol, None))["ok"] is True
+ assert (await _post_beat(protocol, None))["ok"] is True
diff --git a/core/tests/switch_core/bridges/agent/api/test_connection_placements.py b/core/tests/switch_core/bridges/agent/api/test_connection_placements.py
index 9ca4eade0..246c9d8b8 100644
--- a/core/tests/switch_core/bridges/agent/api/test_connection_placements.py
+++ b/core/tests/switch_core/bridges/agent/api/test_connection_placements.py
@@ -10,7 +10,7 @@ from fastapi import HTTPException
from switch_core.bridges.agent.api.handlers import connection_placements
from switch_core.bridges.agent.api.schemas import ConnectionPlacementsRequest
from switch_core.bridges.agent.protocol.agent_connections import (
- ROOM_RELEASED_PROTOCOL_REVISION,
+ PROTOCOL_VERSION,
AgentConnection,
AgentConnectionRegistry,
ClientDeclaration,
@@ -50,7 +50,7 @@ def _open(
delivery_filter="all",
spawn_capable=True,
cursor=0,
- declaration=ClientDeclaration(speaks=ROOM_RELEASED_PROTOCOL_REVISION),
+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
expected_generation=None,
)
diff --git a/core/tests/switch_core/bridges/agent/api/test_connection_socket.py b/core/tests/switch_core/bridges/agent/api/test_connection_socket.py
new file mode 100644
index 000000000..d15ff1f09
--- /dev/null
+++ b/core/tests/switch_core/bridges/agent/api/test_connection_socket.py
@@ -0,0 +1,260 @@
+"""The agent connection over one WebSocket: same frames as the event stream,
+authenticated once, and a heartbeat the server drives.
+
+Opening, fencing and room claims are shared with the SSE stream and tested
+there; these cover what is particular to the socket.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncGenerator, Iterator
+from typing import Any
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from starlette.types import ASGIApp, Receive, Scope, Send
+from starlette.websockets import WebSocket, WebSocketDisconnect
+
+from switch_core.bridges.agent.api import handlers
+from switch_core.bridges.agent.api.session_reporter import SessionReporter
+from switch_core.bridges.agent.dependencies import get_config, get_protocol
+from switch_core.bridges.agent.protocol.agent_connections import (
+ PROTOCOL_VERSION,
+ AgentConnectionRegistry,
+ ClientDeclaration,
+)
+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
+from switch_core.db.models import Agent
+
+AGENT_ID = "agent-1"
+ROOM = "room-a"
+
+
+class _Protocol:
+ def __init__(self) -> None:
+ self.event_buffer = EventBuffer(sequence_base=0)
+ self.connections = AgentConnectionRegistry()
+ self.approval_outcomes = None
+ self.sessions = SessionReporter(None)
+
+ async def record_client_declaration(
+ self, agent_id: str, connection_id: str, declaration: ClientDeclaration
+ ) -> None:
+ return None
+
+ async def require_room_member(self, agent_id: str, room_id: str) -> None:
+ return None
+
+
+class _AuthenticatedAs:
+ """What the bearer-auth middleware leaves in scope for a valid token."""
+
+ def __init__(self, app: ASGIApp, agent: Agent) -> None:
+ self.app = app
+ self.agent = agent
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ scope["agent"] = self.agent
+ await self.app(scope, receive, send)
+
+
+def protocol() -> _Protocol:
+ return _Protocol()
+
+
+def client(
+ protocol: _Protocol, monkeypatch: pytest.MonkeyPatch
+) -> Iterator[TestClient]:
+ # Pings every 50 ms rather than every heartbeat interval.
+ monkeypatch.setattr(handlers, "HEARTBEAT_INTERVAL_SECONDS", 0.05)
+ app = FastAPI()
+ app.include_router(handlers.router, prefix="/agents")
+ app.dependency_overrides[get_protocol] = lambda: protocol
+ app.dependency_overrides[get_config] = lambda: None
+ agent = Agent(
+ id=AGENT_ID, name="agent", metadata_={"known_agent_type": "claude-code"}
+ )
+ with TestClient(_AuthenticatedAs(app, agent)) as client: # type: ignore[arg-type]
+ yield client
+
+
+def _url(**params: Any) -> str:
+ query = {"connection_id": "c1", "protocol": PROTOCOL_VERSION, **params}
+ return f"/agents/{AGENT_ID}/connection/ws?" + "&".join(
+ f"{k}={v}" for k, v in query.items()
+ )
+
+
+def _next(ws: Any, event: str) -> dict[str, Any]:
+ """The next message with this event name, skipping pings."""
+ for _ in range(50):
+ message = ws.receive_json()
+ if message["event"] == event:
+ return message
+ raise AssertionError(f"no {event} frame")
+
+
+def test_it_opens_like_the_event_stream(
+ client: TestClient, protocol: _Protocol
+) -> None:
+ with client.websocket_connect(_url(scope="all")) as ws:
+ state = ws.receive_json()
+
+ assert state["event"] == "connection_state"
+ assert state["data"]["connection_id"] == "c1"
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ assert conn.stream_attached
+ assert state["data"]["generation"] == conn.stream_generation
+
+
+def test_events_arrive_with_their_sequence_number(
+ client: TestClient, protocol: _Protocol
+) -> None:
+ with client.websocket_connect(_url(rooms=ROOM)) as ws:
+ ws.receive_json()
+ seq = protocol.event_buffer.enqueue(
+ AGENT_ID,
+ ROOM,
+ AgentEvent(
+ type="message",
+ room_id=ROOM,
+ payload=MessagePayload(
+ addressed=True,
+ sender="@u:s",
+ sender_name="u",
+ message_id="$evt-1",
+ body="hello",
+ timestamp=0,
+ ),
+ ),
+ )
+
+ message = _next(ws, "message")
+
+ assert message["id"] == seq
+ assert message["data"]["payload"]["body"] == "hello"
+
+
+def test_the_server_pings_and_a_pong_counts_as_a_beat(
+ client: TestClient, protocol: _Protocol
+) -> None:
+ with client.websocket_connect(_url()) as ws:
+ ws.receive_json()
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ beats_before = conn.beats
+
+ _next(ws, "ping")
+ ws.send_json({"type": "pong", "cursor": 0})
+ _next(ws, "ping") # the pong has been handled by the next ping
+
+ assert conn.beats > beats_before
+
+
+def test_a_refusal_is_sent_then_the_socket_closes_with_its_status(
+ client: TestClient,
+) -> None:
+ with client.websocket_connect(_url(scope="everywhere")) as ws:
+ refused = ws.receive_json()
+
+ assert refused["event"] == "refused"
+ assert refused["data"]["status"] == 400
+ with pytest.raises(WebSocketDisconnect) as closed:
+ ws.receive_json()
+ assert closed.value.code == 4400
+
+
+def test_closing_the_socket_detaches_the_stream(
+ client: TestClient, protocol: _Protocol
+) -> None:
+ with client.websocket_connect(_url()) as ws:
+ ws.receive_json()
+
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ assert not conn.stream_attached
+
+
+def test_a_socket_whose_connection_is_taken_over_is_told_so(
+ client: TestClient, protocol: _Protocol
+) -> None:
+ """Final for the client that lost it: reconnecting would take the
+ connection straight back off the winner."""
+ with client.websocket_connect(_url()) as loser:
+ loser.receive_json()
+ with client.websocket_connect(_url()) as winner:
+ winner.receive_json()
+
+ evicted = _next(loser, "evicted")
+
+ assert evicted["data"]["code"] == "taken_over"
+
+
+def test_a_socket_the_server_closed_under_it_ends_cleanly(
+ client: TestClient, protocol: _Protocol, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Uvicorn closes sockets itself on shutdown, and the next send raises
+ RuntimeError rather than WebSocketDisconnect."""
+ send_json = WebSocket.send_json
+
+ async def closed_on_ping(self: WebSocket, data: Any, mode: str = "text") -> None:
+ if data.get("event") == "ping":
+ raise RuntimeError("Unexpected ASGI message 'websocket.send'")
+ await send_json(self, data, mode) # type: ignore[arg-type]
+
+ monkeypatch.setattr(WebSocket, "send_json", closed_on_ping)
+ with client.websocket_connect(_url()) as ws:
+ ws.receive_json()
+ with pytest.raises(WebSocketDisconnect):
+ ws.receive_json()
+
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ assert not conn.stream_attached
+
+
+def test_a_stream_that_fails_closes_the_socket_as_an_error(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch, caplog: Any
+) -> None:
+ async def failing(**kwargs: Any) -> AsyncGenerator[Any]:
+ raise RuntimeError("approval resync failed")
+ yield
+
+ monkeypatch.setattr(handlers, "event_frames", failing)
+ with client.websocket_connect(_url()) as ws:
+ with pytest.raises(WebSocketDisconnect) as closed:
+ _next(ws, "connection_state")
+
+ assert closed.value.code == 1011
+ assert "delivery failed" in caplog.text
+
+
+async def test_a_cancelled_pump_closes_its_stream() -> None:
+ """Cancelled while the outbox is full, the pump is not inside the stream,
+ so only closing it runs the stream's cleanup there and then."""
+ closed = asyncio.Event()
+
+ async def frames() -> AsyncGenerator[Any]:
+ try:
+ while True:
+ yield "frame"
+ finally:
+ closed.set()
+
+ stream = frames()
+ outbox: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)
+ pump = asyncio.create_task(handlers._pump_frames(stream, outbox))
+ while not outbox.full():
+ await asyncio.sleep(0)
+
+ pump.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await pump
+
+ assert closed.is_set()
diff --git a/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py b/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py
new file mode 100644
index 000000000..8e7ed15e1
--- /dev/null
+++ b/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py
@@ -0,0 +1,256 @@
+"""An old client keeps working against this server.
+
+Agent runtimes up to 0.8.x connect over the Server-Sent Events stream and beat
+with `POST /connection/beat`, declaring agent-protocol 6 (0.7.x) or 7 (0.8.0,
+which added the hosted worker).
+The WebSocket replaces both, and both are kept beside it for a compatibility
+window (the expand half of expand/contract), so these drive the server the way
+such a client does: open the stream, read events, beat, and lapse when it stops
+beating. The stream encodes the same frames the socket sends, from the same
+loop, so the rules themselves are tested once, on the frames.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections.abc import AsyncIterator
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from fastapi import HTTPException
+from starlette.responses import StreamingResponse
+
+from switch_core.artifacts import contract_range
+from switch_core.bridges.agent.api.handlers import connection_beat, poll_events
+from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
+from switch_core.bridges.agent.api.session_reporter import SessionReporter
+from switch_core.bridges.agent.protocol.agent_connections import (
+ HEARTBEAT_TTL_SECONDS,
+ AgentConnectionRegistry,
+ ClientDeclaration,
+)
+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
+
+AGENT_ID = "agent-1"
+ROOM = "room-a"
+# What agent-runtime 0.7.x declares: agent-protocol 6, accepting 1.
+OLD_RUNTIME = {
+ "protocol_version": 6,
+ "protocol_accepts": 1,
+ "client": "agent-runtime",
+ "client_version": "0.7.2",
+}
+
+
+class _Protocol:
+ def __init__(self) -> None:
+ self.event_buffer = EventBuffer(sequence_base=0)
+ self.connections = AgentConnectionRegistry()
+ self.approval_outcomes = None
+ self.sessions = SessionReporter(None)
+
+ async def record_client_declaration(
+ self, agent_id: str, connection_id: str, declaration: ClientDeclaration
+ ) -> None:
+ return None
+
+ async def require_room_member(self, agent_id: str, room_id: str) -> None:
+ return None
+
+
+def _agent() -> Any:
+ return SimpleNamespace(id=AGENT_ID, metadata_={"known_agent_type": "claude-code"})
+
+
+def _message(body: str) -> AgentEvent:
+ return AgentEvent(
+ type="message",
+ room_id=ROOM,
+ payload=MessagePayload(
+ addressed=True,
+ sender="@u:s",
+ sender_name="u",
+ message_id=f"${body}",
+ body=body,
+ timestamp=0,
+ ),
+ )
+
+
+async def _open(protocol: _Protocol, **kw: Any) -> StreamingResponse:
+ params: dict[str, Any] = {
+ "accept": "text/event-stream",
+ "connection_id": "c1",
+ "rooms": ROOM,
+ **OLD_RUNTIME,
+ **kw,
+ }
+ resp = await poll_events(
+ agent_id=AGENT_ID,
+ agent=_agent(),
+ protocol=protocol, # type: ignore[arg-type]
+ config=None, # type: ignore[arg-type]
+ **params,
+ )
+ assert isinstance(resp, StreamingResponse)
+ return resp
+
+
+class _Reader:
+ """Reads a response body the way the old runtime's `readSse` did."""
+
+ def __init__(self, body: AsyncIterator[Any]) -> None:
+ self.body = body
+ self.buffered = ""
+
+ async def frame(self, timeout: float = 2.0) -> tuple[str, str | None, dict]:
+ while "\n\n" not in self.buffered:
+ chunk = await asyncio.wait_for(anext(self.body), timeout)
+ self.buffered += chunk.decode() if isinstance(chunk, bytes) else chunk
+ raw, self.buffered = self.buffered.split("\n\n", 1)
+ event, seq, data = "message", None, None
+ for line in raw.split("\n"):
+ if line.startswith(":"):
+ continue
+ if line.startswith("event: "):
+ event = line[len("event: ") :]
+ elif line.startswith("id: "):
+ seq = line[len("id: ") :]
+ elif line.startswith("data: "):
+ data = line[len("data: ") :]
+ if data is None:
+ return await self.frame(timeout)
+ return event, seq, json.loads(data)
+
+ async def close(self) -> None:
+ await self.body.aclose() # type: ignore[attr-defined]
+
+
+async def _beat(protocol: _Protocol, cursor: int, generation: int | None) -> Any:
+ return await connection_beat(
+ AGENT_ID,
+ ConnectionBeatRequest(connection_id="c1", cursor=cursor, generation=generation),
+ SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]
+ protocol, # type: ignore[arg-type]
+ )
+
+
+async def test_an_old_client_connects_receives_beats_and_lapses() -> None:
+ protocol = _Protocol()
+ stream = _Reader((await _open(protocol)).body_iterator)
+
+ event, _, state = await stream.frame()
+ assert event == "connection_state"
+ assert state["rooms"] == [ROOM]
+ generation = state["generation"]
+
+ seq = protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("hello"))
+ event, sent_seq, data = await stream.frame()
+ assert event == "message"
+ assert sent_seq == str(seq)
+ assert data["payload"]["body"] == "hello"
+
+ beaten = await _beat(protocol, seq, generation)
+ assert beaten == {"ok": True, "rooms": [ROOM], "cursor": seq}
+
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ conn.last_beat = time.monotonic() - (HEARTBEAT_TTL_SECONDS + 1)
+ protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("not after a lapse"))
+
+ event, _, data = await stream.frame()
+ assert event == "evicted"
+ assert data["code"] == "heartbeat_lapsed"
+ await stream.close()
+
+ with pytest.raises(HTTPException) as caught:
+ await _beat(protocol, seq, generation)
+ assert caught.value.status_code == 404
+
+
+async def test_an_old_client_resumes_from_last_event_id() -> None:
+ protocol = _Protocol()
+ first = protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("seen"))
+ protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("missed"))
+
+ # The old runtime sent the cursor both ways; the header wins.
+ stream = _Reader(
+ (
+ await _open(protocol, start_from="head", last_event_id=str(first))
+ ).body_iterator
+ )
+ await stream.frame()
+ event, _, data = await stream.frame()
+ await stream.close()
+
+ assert event == "message"
+ assert data["payload"]["body"] == "missed"
+
+
+async def test_closing_the_stream_detaches_it_and_keeps_the_connection() -> None:
+ protocol = _Protocol()
+ stream = _Reader((await _open(protocol)).body_iterator)
+ await stream.frame()
+
+ await stream.close()
+
+ conn = protocol.connections.get("c1")
+ assert conn is not None
+ assert not conn.stream_attached
+
+
+# ── The declared ranges still meet ──────────────────────────────────────────
+
+
+def _overlap(a: tuple[int, int], b: tuple[int, int]) -> bool:
+ """Whether two (speaks, accepts) ranges share a revision."""
+ return a[1] <= b[0] and b[1] <= a[0]
+
+
+# What the releases before the socket declare, on either side: agent-protocol
+# 6 (agent-runtime 0.7.x, and switch-core before the hosted worker) and 7
+# (agent-runtime 0.8.0, and switch-core with it).
+BEFORE_THE_SOCKET = [(6, 1), (7, 1)]
+
+
[email protected]("old", BEFORE_THE_SOCKET)
+def test_an_old_runtime_and_this_server_share_a_revision(old: tuple[int, int]) -> None:
+ server = contract_range("agent-protocol", "switch-core")
+
+ assert _overlap(old, (server.speaks, server.accepts))
+
+
[email protected]("old", BEFORE_THE_SOCKET)
+def test_this_runtime_and_an_old_server_share_a_revision(old: tuple[int, int]) -> None:
+ runtime = contract_range("agent-protocol", "agent-runtime")
+
+ assert _overlap((runtime.speaks, runtime.accepts), old)
+
+
+def test_this_runtime_and_this_server_share_the_newest_revision() -> None:
+ server = contract_range("agent-protocol", "switch-core")
+ runtime = contract_range("agent-protocol", "agent-runtime")
+
+ assert runtime.speaks == server.speaks
+
+
[email protected]("speaks", [6, 7])
+def test_this_server_admits_an_old_runtime(speaks: int) -> None:
+ registry = AgentConnectionRegistry()
+ conn = registry.open(
+ agent_id=AGENT_ID,
+ connection_id="c1",
+ scope="single",
+ delivery_filter="all",
+ spawn_capable=False,
+ cursor=0,
+ declaration=ClientDeclaration(speaks=speaks, accepts=1),
+ expected_generation=None,
+ transport="sse",
+ )
+
+ assert conn.stream_transport == "sse"
diff --git a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
index 4744a0332..dfedf3c7b 100644
--- a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
+++ b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
@@ -1,8 +1,9 @@
-"""The /events endpoint serves both transports off one path (CHOO-1857).
+"""Opening an agent connection, and the /events endpoint.
-`Accept: text/event-stream` opens a connection and streams; anything else gets
-the long poll. Both read the same buffer, so the two cannot diverge while they
-coexist — but the dispatch itself is easy to break silently, hence these.
+The WebSocket and the event stream both open through `_open_connection`,
+which these drive directly: validation, protocol ranges, declarations, room
+claims and session reporting. `/events` with `Accept: text/event-stream` opens
+the event stream an old client still asks for; anything else long polls.
"""
from __future__ import annotations
@@ -16,7 +17,11 @@ from fastapi import HTTPException
from starlette.responses import StreamingResponse
from switch_core.bridges.agent.api import session_reporter
-from switch_core.bridges.agent.api.handlers import _resolve_start_cursor, poll_events
+from switch_core.bridges.agent.api.handlers import (
+ _open_connection,
+ _resolve_start_cursor,
+ poll_events,
+)
from switch_core.bridges.agent.api.session_reporter import SessionReporter
from switch_core.bridges.agent.protocol.agent_connections import (
HEARTBEAT_LAPSED,
@@ -90,10 +95,31 @@ async def _call(protocol: _Protocol, **kw: Any) -> Any:
"client": None,
"client_version": None,
"rooms": None,
- "last_event_id": None,
}
params.update(kw)
- return await poll_events(**params)
+ if params.pop("accept") != "text/event-stream":
+ return await poll_events(**params, accept=None)
+ # What used to open the stream now opens the connection the WebSocket
+ # carries; the rules are the same ones.
+ conn, _frames = await _open_connection(
+ config=None, # type: ignore[arg-type]
+ agent=params["agent"],
+ protocol=params["protocol"],
+ connection_id=params["connection_id"],
+ scope=params["scope"],
+ event_filter=params["event_filter"],
+ start_from=params["start_from"],
+ spawn_capable=params["spawn_capable"],
+ declaration=ClientDeclaration(
+ speaks=params["protocol_version"],
+ accepts=params["protocol_accepts"],
+ artifact=params["client"],
+ version=params["client_version"],
+ ),
+ rooms=params["rooms"],
+ expected_generation=params.get("expected_generation"),
+ )
+ return conn
async def test_without_the_sse_accept_header_it_long_polls() -> None:
@@ -104,10 +130,18 @@ async def test_without_the_sse_accept_header_it_long_polls() -> None:
assert resp.status_code == 204
-async def test_with_the_sse_accept_header_it_opens_a_connection() -> None:
+async def test_with_the_sse_accept_header_it_opens_the_event_stream() -> None:
+ """How a runtime built before the WebSocket connects, kept for a
+ compatibility window."""
protocol = _Protocol()
- resp = await _call(
- protocol, accept="text/event-stream", connection_id="c1", scope="all"
+ resp = await poll_events(
+ agent_id=AGENT_ID,
+ agent=_agent(),
+ protocol=protocol, # type: ignore[arg-type]
+ config=None, # type: ignore[arg-type]
+ accept="text/event-stream",
+ connection_id="c1",
+ scope="all",
)
assert isinstance(resp, StreamingResponse)
@@ -120,6 +154,19 @@ async def test_with_the_sse_accept_header_it_opens_a_connection() -> None:
conn = protocol.connections.get("c1")
assert conn is not None
assert conn.scope == "all"
+ assert conn.stream_transport == "sse"
+
+
+async def test_opening_registers_the_connection() -> None:
+ protocol = _Protocol()
+ conn = await _call(
+ protocol, accept="text/event-stream", connection_id="c1", scope="all"
+ )
+
+ assert protocol.connections.get("c1") is conn
+ assert conn.scope == "all"
+ assert conn.stream_attached
+ assert conn.stream_transport == "websocket"
async def test_streaming_without_a_connection_id_is_refused() -> None:
@@ -278,7 +325,7 @@ async def test_an_older_client_inside_the_server_range_is_accepted() -> None:
def test_head_means_only_what_happens_next() -> None:
protocol = _Protocol()
protocol.event_buffer.enqueue(AGENT_ID, "room", _event())
- assert _resolve_start_cursor(protocol, AGENT_ID, "head", None) == 1
+ assert _resolve_start_cursor(protocol, AGENT_ID, "head") == 1
def test_last_event_id_wins_over_start_from() -> None:
@@ -288,13 +335,13 @@ def test_last_event_id_wins_over_start_from() -> None:
def test_explicit_start_from_is_honoured() -> None:
protocol = _Protocol()
- assert _resolve_start_cursor(protocol, AGENT_ID, "17", None) == 17
+ assert _resolve_start_cursor(protocol, AGENT_ID, "17") == 17
def test_a_nonsense_cursor_is_refused_rather_than_guessed() -> None:
protocol = _Protocol()
with pytest.raises(HTTPException) as excinfo:
- _resolve_start_cursor(protocol, AGENT_ID, "banana", None)
+ _resolve_start_cursor(protocol, AGENT_ID, "banana")
assert excinfo.value.status_code == 400
@@ -341,14 +388,13 @@ class TestDeclaringARoomAtOpenTakesOver:
)
protocol.connections.claim_room(incumbent, "room-1")
- resp = await _call(
+ await _call(
protocol,
accept="text/event-stream",
connection_id="supervisor",
rooms="room-1",
)
- assert isinstance(resp, StreamingResponse)
claimant = protocol.connections.claimant_of(AGENT_ID, "room-1")
assert claimant is not None
assert claimant.id == "supervisor"
@@ -367,12 +413,20 @@ async def test_reconnect_during_bookkeeping_cannot_detach_the_new_stream(
entered.set()
await release.wait()
+ async def open_stream() -> Any:
+ return await poll_events(
+ agent_id=AGENT_ID,
+ agent=_agent(),
+ protocol=protocol, # type: ignore[arg-type]
+ config=None, # type: ignore[arg-type]
+ accept="text/event-stream",
+ connection_id="c1",
+ )
+
monkeypatch.setattr(protocol, "record_client_declaration", record)
- first = asyncio.create_task(
- _call(protocol, accept="text/event-stream", connection_id="c1")
- )
+ first = asyncio.create_task(open_stream())
await entered.wait()
- newer = await _call(protocol, accept="text/event-stream", connection_id="c1")
+ newer = await open_stream()
await anext(newer.body_iterator)
release.set()
older = await first
diff --git a/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py b/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py
index 8759b71d7..356332f42 100644
--- a/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py
+++ b/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py
@@ -17,7 +17,7 @@ from typing import Any
import pytest
from fastapi import HTTPException
-from switch_core.bridges.agent.api.handlers import poll_events
+from switch_core.bridges.agent.api.handlers import _open_connection
from switch_core.bridges.agent.api.session_reporter import SessionReporter
from switch_core.bridges.agent.protocol.agent_connections import (
PROTOCOL_VERSION,
@@ -62,16 +62,20 @@ def _attach(protocol: _Protocol) -> Any:
async def _reopen(protocol: _Protocol, expected_generation: int | None) -> Any:
- return await poll_events(
- AGENT_ID,
- SimpleNamespace(id=AGENT_ID, metadata_={}), # type: ignore[arg-type]
- protocol, # type: ignore[arg-type]
- None, # type: ignore[arg-type]
- accept="text/event-stream",
+ conn, _frames = await _open_connection(
+ config=None, # type: ignore[arg-type]
+ agent=SimpleNamespace(id=AGENT_ID, metadata_={}), # type: ignore[arg-type]
+ protocol=protocol, # type: ignore[arg-type]
connection_id=CONN_ID,
- protocol_version=PROTOCOL_VERSION,
+ scope="single",
+ event_filter="all",
+ start_from="head",
+ spawn_capable=False,
+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
+ rooms=None,
expected_generation=expected_generation,
)
+ return conn
async def test_a_reattach_claiming_a_superseded_incarnation_is_refused() -> None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py b/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
index 18a2d749b..3c98bd96d 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import asyncio
-import json
from typing import Any
import pytest
@@ -11,6 +10,7 @@ import pytest
from switch_core.bridges.agent.protocol.agent_connections import (
HEARTBEAT_LAPSED,
LAUNCH_SUPERSEDED,
+ PROTOCOL_VERSION,
ROOM_RELEASED_PROTOCOL_REVISION,
AgentConnection,
AgentConnectionRegistry,
@@ -20,7 +20,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
from switch_core.bridges.agent.protocol.hosted_workers import WorkerBinding
from switch_core.bridges.agent.protocol.presence import agents_present_in
-from switch_core.bridges.agent.protocol.stream import event_stream
+from switch_core.bridges.agent.protocol.stream import KEEPALIVE, event_frames
AGENT = "agent-1"
ROOM_A = "room-a"
@@ -54,15 +54,9 @@ async def _frames_until_quiet(
async def pump() -> None:
async for frame in stream:
- if frame.startswith(b":"):
+ if frame is KEEPALIVE:
continue
- name, data = "", "{}"
- for line in frame.decode().strip().splitlines():
- if line.startswith("event: "):
- name = line[len("event: ") :]
- elif line.startswith("data: "):
- data = line[len("data: ") :]
- out.append((name, json.loads(data)))
+ out.append((frame.event, frame.data))
try:
await asyncio.wait_for(pump(), timeout=timeout)
@@ -73,12 +67,8 @@ async def _frames_until_quiet(
def test_a_claim_taken_over_is_released_to_its_holder_with_its_session() -> None:
registry = AgentConnectionRegistry()
- loser = _open(
- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
- winner = _open(
- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")
+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.claim_room(loser, ROOM_A)
registry.place_session(AGENT, "session-1", ROOM_A, loser.id)
@@ -91,12 +81,8 @@ def test_a_claim_taken_over_is_released_to_its_holder_with_its_session() -> None
def test_connect_to_room_placement_releases_the_other_connections_session() -> None:
"""`connect_to_room` places before it claims; the frame names the session once."""
registry = AgentConnectionRegistry()
- loser = _open(
- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
- winner = _open(
- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")
+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.claim_room(loser, ROOM_A)
registry.place_session(AGENT, "session-old", ROOM_A, loser.id)
@@ -108,9 +94,7 @@ def test_connect_to_room_placement_releases_the_other_connections_session() -> N
def test_displacing_a_sibling_on_the_same_connection_releases_nothing() -> None:
registry = AgentConnectionRegistry()
- conn = _open(
- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")
registry.place_session(AGENT, "session-old", ROOM_A, conn.id)
registry.place_session(AGENT, "session-new", ROOM_A, conn.id)
@@ -120,11 +104,11 @@ def test_displacing_a_sibling_on_the_same_connection_releases_nothing() -> None:
@pytest.mark.parametrize("speaks", [ROOM_RELEASED_PROTOCOL_REVISION - 1, None])
def test_a_client_that_cannot_take_the_frame_is_not_sent_it(speaks: int | None) -> None:
+ """A client from before the frame, or one that declared no revision, gets
+ only what every client takes."""
registry = AgentConnectionRegistry()
loser = _open(registry, "loser", speaks=speaks, scope="single")
- winner = _open(
- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.claim_room(loser, ROOM_A)
registry.claim_room(winner, ROOM_A, takeover=True)
@@ -135,12 +119,8 @@ def test_a_client_that_cannot_take_the_frame_is_not_sent_it(speaks: int | None)
def test_taking_the_room_back_withdraws_the_unsent_release() -> None:
registry = AgentConnectionRegistry()
- first = _open(
- registry, "first", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
- second = _open(
- registry, "second", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ first = _open(registry, "first", speaks=PROTOCOL_VERSION, scope="all")
+ second = _open(registry, "second", speaks=PROTOCOL_VERSION, scope="all")
registry.claim_room(first, ROOM_A)
registry.claim_room(second, ROOM_A, takeover=True)
@@ -152,9 +132,7 @@ def test_taking_the_room_back_withdraws_the_unsent_release() -> None:
def test_replacing_placements_is_a_full_replacement() -> None:
registry = AgentConnectionRegistry()
- conn = _open(
- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(conn, {"session-1": ROOM_A, "session-2": ROOM_B})
released = registry.replace_placements(conn, {"session-2": ROOM_C})
@@ -167,10 +145,8 @@ def test_replacing_placements_is_a_full_replacement() -> None:
def test_replacing_placements_leaves_other_connections_alone() -> None:
registry = AgentConnectionRegistry()
- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")
- theirs = _open(
- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")
+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(theirs, {"session-t": ROOM_B})
registry.replace_placements(mine, {"session-m": ROOM_A})
@@ -184,10 +160,8 @@ def test_replacing_placements_leaves_other_connections_alone() -> None:
def test_replacing_placements_takes_a_room_over_and_releases_it() -> None:
registry = AgentConnectionRegistry()
- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")
- theirs = _open(
- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")
+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(theirs, {"session-t": ROOM_A, "session-u": ROOM_B})
released = registry.replace_placements(mine, {"session-m": ROOM_A})
@@ -203,10 +177,8 @@ def test_replacing_placements_takes_a_room_over_and_releases_it() -> None:
def test_a_room_claimed_without_a_placement_is_released_with_no_session() -> None:
registry = AgentConnectionRegistry()
- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")
- theirs = _open(
- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="single"
- )
+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")
+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="single")
registry.claim_room(theirs, ROOM_A)
released = registry.replace_placements(mine, {"session-m": ROOM_A})
@@ -219,9 +191,7 @@ def test_a_room_claimed_without_a_placement_is_released_with_no_session() -> Non
def test_two_sessions_in_one_room_are_refused_without_change() -> None:
registry = AgentConnectionRegistry()
- conn = _open(
- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(conn, {"session-1": ROOM_B})
with pytest.raises(ValueError, match=ROOM_A):
@@ -234,14 +204,10 @@ def test_two_sessions_in_one_room_are_refused_without_change() -> None:
async def test_the_displaced_stream_is_sent_room_released() -> None:
registry = AgentConnectionRegistry()
buffer = EventBuffer(sequence_base=0)
- loser = _open(
- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
- winner = _open(
- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")
+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(loser, {"session-1": ROOM_A})
- stream = event_stream(conn=loser, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=loser, registry=registry, buffer=buffer, approvals=None)
try:
await anext(stream) # connection_state
waiting = asyncio.ensure_future(_frames_until_quiet(stream))
@@ -255,17 +221,13 @@ async def test_the_displaced_stream_is_sent_room_released() -> None:
assert [name for name, _ in frames].count("room_released") == 1
-async def test_an_older_client_stream_is_not_sent_room_released() -> None:
+async def test_a_client_that_declared_nothing_is_not_sent_room_released() -> None:
registry = AgentConnectionRegistry()
buffer = EventBuffer(sequence_base=0)
- loser = _open(
- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION - 1, scope="all"
- )
- winner = _open(
- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"
- )
+ loser = _open(registry, "loser", speaks=None, scope="all")
+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(loser, {"session-1": ROOM_A})
- stream = event_stream(conn=loser, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=loser, registry=registry, buffer=buffer, approvals=None)
try:
await anext(stream) # connection_state
waiting = asyncio.ensure_future(_frames_until_quiet(stream))
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_stream.py b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
index 03b2a3422..79a4eb78e 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_stream.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import asyncio
-import json
from typing import Any
import pytest
@@ -20,7 +19,14 @@ from switch_core.bridges.agent.protocol.agent_connections import (
Closure,
)
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
-from switch_core.bridges.agent.protocol.stream import event_stream
+from switch_core.bridges.agent.protocol.stream import (
+ KEEPALIVE,
+ Frame,
+ encode_sse,
+ encode_ws,
+ event_frames,
+ sse_stream,
+)
from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
from switch_core.observability.metrics import MetricsRegistry, install, uninstall
@@ -44,17 +50,9 @@ def _message(body: str, *, addressed: bool = False, room: str = ROOM_A) -> Agent
)
-def _parse(frame: bytes) -> tuple[str, dict[str, Any]]:
- """Split one SSE frame into (event name, data)."""
- text = frame.decode()
- name = ""
- data = "{}"
- for line in text.strip().splitlines():
- if line.startswith("event: "):
- name = line[len("event: ") :]
- elif line.startswith("data: "):
- data = line[len("data: ") :]
- return name, json.loads(data)
+def _parse(frame: Frame) -> tuple[str, dict[str, Any]]:
+ """A frame as (event name, data)."""
+ return frame.event, frame.data
async def _take(stream, count: int, timeout: float = 2.0) -> list[tuple[str, dict]]:
@@ -63,7 +61,7 @@ async def _take(stream, count: int, timeout: float = 2.0) -> list[tuple[str, dic
async def pump() -> None:
async for frame in stream:
- if frame.startswith(b":"):
+ if frame is KEEPALIVE:
continue
out.append(_parse(frame))
if len(out) >= count:
@@ -94,7 +92,7 @@ async def test_first_frame_is_the_connection_state() -> None:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
((name, data),) = await _take(stream, 1)
assert name == "connection_state"
@@ -118,7 +116,7 @@ async def test_the_first_frame_declares_the_server(monkeypatch) -> None:
buffer = EventBuffer(sequence_base=0)
conn = _open(registry)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
((_, data),) = await _take(stream, 1)
assert data["server"] == {
@@ -141,7 +139,7 @@ async def test_an_unreadable_server_version_is_null_not_a_placeholder(
buffer = EventBuffer(sequence_base=0)
conn = _open(registry)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
((_, data),) = await _take(stream, 1)
assert data["server"]["version"] is None
@@ -165,7 +163,7 @@ async def test_the_first_frame_echoes_what_the_client_declared() -> None:
),
)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
((_, data),) = await _take(stream, 1)
assert data["client"] == {
@@ -181,7 +179,7 @@ async def test_an_undeclared_client_is_echoed_as_all_null() -> None:
buffer = EventBuffer(sequence_base=0)
conn = _open(registry, declaration=ClientDeclaration())
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
((_, data),) = await _take(stream, 1)
assert data["client"] == {
@@ -201,7 +199,7 @@ async def test_catch_up_then_live_delivery() -> None:
# Queued before the stream opened: must be caught up, not skipped.
buffer.enqueue(AGENT, ROOM_A, _message("before"))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "message"
assert frames[1][1]["payload"]["body"] == "before"
@@ -225,7 +223,7 @@ async def test_sequence_number_is_carried_for_resume() -> None:
registry.claim_room(conn, ROOM_A)
seq = buffer.enqueue(AGENT, ROOM_A, _message("one"))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][1]["sequence"] == seq
@@ -241,7 +239,7 @@ async def test_resuming_from_a_cursor_skips_what_was_already_seen() -> None:
conn = _open(registry, cursor=first)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][1]["payload"]["body"] == "missed"
@@ -256,7 +254,7 @@ async def test_expired_cursor_produces_a_gap_event_rather_than_silence() -> None
conn = _open(registry, cursor=0)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "gap"
@@ -272,7 +270,7 @@ async def test_events_for_rooms_the_connection_does_not_cover_are_skipped() -> N
buffer.enqueue(AGENT, ROOM_B, _message("elsewhere", room=ROOM_B))
buffer.enqueue(AGENT, ROOM_A, _message("mine"))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][1]["payload"]["body"] == "mine"
@@ -286,7 +284,7 @@ async def test_addressed_filter_drops_ambient_chatter() -> None:
buffer.enqueue(AGENT, ROOM_A, _message("chatter"))
buffer.enqueue(AGENT, ROOM_B, _message("wanted", addressed=True, room=ROOM_B))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][1]["payload"]["body"] == "wanted"
@@ -298,7 +296,7 @@ async def test_a_superseded_stream_is_told_it_was_evicted() -> None:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await _take(stream, 1)
# A second stream attaches to the same connection id.
@@ -318,7 +316,7 @@ async def test_closing_the_connection_ends_the_stream_with_a_reason() -> None:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await _take(stream, 1)
registry.close(conn.id, HEARTBEAT_LAPSED)
@@ -337,7 +335,7 @@ async def test_a_close_about_a_room_names_the_room_it_was_about() -> None:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await _take(stream, 1)
registry.close(
@@ -357,7 +355,7 @@ async def test_subscription_change_is_announced() -> None:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await _take(stream, 1)
registry.claim_room(conn, ROOM_B)
@@ -378,7 +376,7 @@ async def test_stream_detaches_on_exit_without_killing_the_connection() -> None:
buffer = EventBuffer(sequence_base=0)
conn = _open(registry)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await _take(stream, 1)
await stream.aclose()
@@ -419,8 +417,8 @@ async def test_two_connections_receive_the_same_event(scope: str) -> None:
buffer.enqueue(AGENT, ROOM_A, _message("shared"))
buffer.enqueue("agent-2", ROOM_A, _message("shared"))
- a = event_stream(conn=session, registry=registry, buffer=buffer, approvals=None)
- b = event_stream(conn=other_agent, registry=registry, buffer=buffer, approvals=None)
+ a = event_frames(conn=session, registry=registry, buffer=buffer, approvals=None)
+ b = event_frames(conn=other_agent, registry=registry, buffer=buffer, approvals=None)
assert (await _take(a, 2))[1][1]["payload"]["body"] == "shared"
assert (await _take(b, 2))[1][1]["payload"]["body"] == "shared"
@@ -444,7 +442,7 @@ async def test_resume_replays_buffered_events_for_a_room_claimed_at_open() -> No
# Claimed before the stream is created, as the endpoint does.
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "message"
@@ -461,7 +459,7 @@ async def test_events_for_uncovered_rooms_do_not_block_the_cursor() -> None:
buffer.enqueue(AGENT, ROOM_B, _message("someone else's room", room=ROOM_B))
last = buffer.enqueue(AGENT, ROOM_A, _message("mine"))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][1]["payload"]["body"] == "mine"
@@ -482,7 +480,7 @@ async def test_a_cursor_from_before_a_restart_is_reported_not_ignored() -> None:
conn = _open(registry, cursor=4812) # from a previous life
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "gap"
@@ -501,7 +499,7 @@ async def test_stream_reports_restart_below_floor() -> None:
conn = _open(registry, cursor=old_cursor)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 3)
kind, gap = frames[1]
@@ -535,7 +533,7 @@ class TestALapsedHeartbeatStopsDelivery:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
# Consume the opening frame while still healthy.
@@ -559,7 +557,7 @@ class TestALapsedHeartbeatStopsDelivery:
conn = _open(registry)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 1)
@@ -600,7 +598,7 @@ class TestFilteredEventsDoNotSpinTheLoop:
# milliseconds. A spinning generator never yields one at any interval.
monkeypatch.setattr(stream_module, "KEEPALIVE_INTERVAL_SECONDS", 0.05)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 1) # connection_state
@@ -621,7 +619,7 @@ class TestFilteredEventsDoNotSpinTheLoop:
buffer.enqueue(AGENT, ROOM_A, _message("chatter", addressed=False))
buffer.enqueue(AGENT, ROOM_A, _message("for-you", addressed=True))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -656,7 +654,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:
buffer.enqueue(AGENT, ROOM_A, _message("the trigger", addressed=True))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 1) # connection_state
@@ -676,7 +674,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:
buffer.enqueue(AGENT, ROOM_A, _message("the trigger", addressed=True))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 1)
@@ -699,7 +697,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:
buffer.enqueue(AGENT, ROOM_A, _message("hello", addressed=True))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -712,7 +710,7 @@ async def test_replaced_stream_cannot_capture_its_successors_generation() -> Non
registry = AgentConnectionRegistry()
buffer = EventBuffer(sequence_base=0)
conn = _open(registry)
- old = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ old = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
_open(registry)
assert await _take(old, 1) == []
assert registry.beat(AGENT, conn.id, 0, conn.stream_generation).stream_attached
@@ -722,7 +720,7 @@ async def test_closed_stream_cannot_detach_a_recreated_connection() -> None:
registry = AgentConnectionRegistry()
buffer = EventBuffer(sequence_base=0)
conn = _open(registry)
- old = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ old = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
await anext(old)
registry.close(conn.id, HEARTBEAT_LAPSED)
replacement = _open(registry)
@@ -761,7 +759,7 @@ async def test_a_resumed_stream_does_not_replay_a_room_the_agent_was_removed_fro
# The supervisor comes back and resumes from before both events.
conn = _open(registry, connection_id="c-resumed", scope="all", cursor=0)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "message"
@@ -792,7 +790,7 @@ async def test_a_last_event_id_reconnect_does_not_replay_a_removed_room() -> Non
# while it was still in room A, resolved to exactly this by
# `_resolve_start_cursor`.
conn = _open(registry, connection_id="c-reconnect", scope="all", cursor=seen)
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
frames = await _take(stream, 2)
assert frames[1][0] == "message"
@@ -821,7 +819,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:
buffer.enqueue(AGENT, ROOM_A, _message("for you", addressed=True))
buffer.enqueue(AGENT, ROOM_B, _message("also you", addressed=True, room=ROOM_B))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 3)
@@ -842,7 +840,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:
registry.claim_room(conn, ROOM_A)
chatter = buffer.enqueue(AGENT, ROOM_A, _message("chatter"))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 1)
@@ -868,7 +866,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:
buffer.enqueue(AGENT, ROOM_B, _message("before the claim", room=ROOM_B))
buffer.enqueue(AGENT, ROOM_A, _message("for you", addressed=True))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 2) # connection_state, then the message in room A
@@ -887,7 +885,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:
conn = _open(registry, cursor=4812) # from a previous life
registry.claim_room(conn, ROOM_A)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
assert (await _take(stream, 2))[1][0] == "gap"
@@ -915,7 +913,7 @@ class TestAGapNamesTheRoomsThatLostEvents:
registry.claim_room(conn, ROOM_A)
registry.claim_room(conn, ROOM_B)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -932,7 +930,7 @@ class TestAGapNamesTheRoomsThatLostEvents:
conn = _open(registry, cursor=4812)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -954,7 +952,7 @@ class TestAGapNamesTheRoomsThatLostEvents:
buffer.enqueue(AGENT, ROOM_A, _message("after the restart"))
conn = _open(registry, scope="all", cursor=4812)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -981,7 +979,7 @@ class TestAGapNamesTheRoomsThatLostEvents:
conn = _open(registry, cursor=0)
registry.claim_room(conn, ROOM_A)
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
frames = await _take(stream, 2)
@@ -999,7 +997,7 @@ async def test_a_delivered_event_names_no_session() -> None:
registry.place_session(AGENT, "session-1", ROOM_A, conn.id)
buffer.enqueue(AGENT, ROOM_A, _message("hello"))
- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
(_, (name, data)) = await _take(stream, 2)
assert name == "message"
@@ -1013,7 +1011,7 @@ async def test_new_boot_reports_gap_and_keeps_messages_when_old_cursor_is_below_
conn = _open(registry, cursor=(1 << 32) + 2)
registry.claim_room(conn, ROOM_A)
frames = await _take(
- event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None), 5
+ event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None), 5
)
assert frames[1][0] == "gap"
assert frames[1][1]["resumed_at"] == 2 << 32
@@ -1032,7 +1030,7 @@ async def test_each_delivered_event_counts_as_an_agent_bridge_event_out() -> Non
buffer.enqueue(AGENT, ROOM_A, _message("one"))
buffer.enqueue(AGENT, ROOM_B, _message("two", room=ROOM_B))
- stream = event_stream(
+ stream = event_frames(
conn=conn, registry=registry, buffer=buffer, approvals=None
)
await _take(stream, 3)
@@ -1048,3 +1046,43 @@ async def test_each_delivered_event_counts_as_an_agent_bridge_event_out() -> Non
}
finally:
uninstall()
+
+
+class TestBothTransportsEncodeTheSameFrames:
+ """The socket and the event stream an old client still reads carry the
+ same frames from the same loop; only the encoding differs."""
+
+ def test_an_event_carries_its_sequence_as_the_sse_id(self) -> None:
+ frame = Frame("message", {"payload": {"body": "hi"}}, seq=7)
+
+ assert encode_sse(frame) == (
+ b'id: 7\nevent: message\ndata: {"payload":{"body":"hi"}}\n\n'
+ )
+ assert encode_ws(frame) == {
+ "event": "message",
+ "data": {"payload": {"body": "hi"}},
+ "id": 7,
+ }
+
+ def test_a_frame_without_a_sequence_has_no_id(self) -> None:
+ assert encode_sse(Frame("gap", {"reason": "x"})) == (
+ b'event: gap\ndata: {"reason":"x"}\n\n'
+ )
+
+ def test_the_keepalive_is_a_comment_on_the_event_stream(self) -> None:
+ assert encode_sse(KEEPALIVE) == b": keepalive\n\n"
+
+ async def test_closing_the_event_stream_detaches_its_connection(self) -> None:
+ registry = AgentConnectionRegistry()
+ buffer = EventBuffer(sequence_base=0)
+ conn = _open(registry)
+
+ body = sse_stream(
+ event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
+ )
+ first = await anext(body)
+ await body.aclose()
+
+ assert first.startswith(b"event: connection_state\n")
+ assert not conn.stream_attached
+ assert registry.get(conn.id) is conn
diff --git a/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.py b/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.py
new file mode 100644
index 000000000..552059179
--- /dev/null
+++ b/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.py
@@ -0,0 +1,148 @@
+"""BearerAuthMiddleware refusing a WebSocket.
+
+A WebSocket client cannot read the status of a refused handshake, so the
+refusal is a close code it can read: 4000 plus the status, 4401 for a missing
+or bad token.
+"""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from typing import Any
+
+from switch_core.bridges.agent.api_key_cache import ApiKeyCache
+from switch_core.bridges.agent.auth import BearerAuthMiddleware, ControllerPrincipal
+from switch_core.db.stores.agent_store import AgentStore
+from switch_core.db.stores.api_key_store import ApiKeyStore
+
+
+def _middleware() -> tuple[BearerAuthMiddleware, list[Any]]:
+ reached: list[Any] = []
+
+ async def _app(scope: Any, receive: Any, send: Any) -> None:
+ reached.append(scope)
+
+ mw = BearerAuthMiddleware(
+ _app,
+ agent_store=AgentStore(),
+ api_key_store=ApiKeyStore(),
+ api_key_cache=ApiKeyCache(ttl_seconds=5, max_entries=8),
+ session_factory=None, # type: ignore[arg-type]
+ )
+ return mw, reached
+
+
+async def _call(mw: BearerAuthMiddleware, scope: dict[str, Any]) -> list[dict]:
+ sent: list[dict] = []
+ inbox = [{"type": "websocket.connect"}]
+
+ async def receive() -> dict:
+ return inbox.pop(0)
+
+ async def send(message: dict) -> None:
+ sent.append(message)
+
+ await mw(scope, receive, send)
+ return sent
+
+
+async def test_a_socket_without_a_token_is_accepted_and_closed_with_4401() -> None:
+ mw, reached = _middleware()
+
+ sent = await _call(
+ mw,
+ {
+ "type": "websocket",
+ "path": "/agents/a1/connection/ws",
+ "headers": [],
+ "query_string": b"",
+ },
+ )
+
+ assert [m["type"] for m in sent] == ["websocket.accept", "websocket.close"]
+ assert sent[1]["code"] == 4401
+ assert reached == []
+
+
+async def test_a_request_without_a_token_is_still_an_http_401() -> None:
+ mw, reached = _middleware()
+ sent: list[dict] = []
+
+ async def receive() -> dict:
+ return {"type": "http.request", "body": b"", "more_body": False}
+
+ async def send(message: dict) -> None:
+ sent.append(message)
+
+ await mw(
+ {
+ "type": "http",
+ "method": "GET",
+ "path": "/agents/a1/events",
+ "headers": [],
+ "query_string": b"",
+ },
+ receive,
+ send,
+ )
+
+ assert sent[0]["status"] == 401
+ assert reached == []
+
+
+class _NotAssigned:
+ """A controller whose token is good and which runs no agent at all."""
+
+ presence = SimpleNamespace(binding=lambda agent_id: None)
+ auth_cache = None
+
+ def handles(self, path: str) -> bool:
+ return False
+
+ def is_public(self, path: str) -> bool:
+ return False
+
+ def is_controller_token(self, token: str) -> bool:
+ return True
+
+ async def authenticate(self, token: str) -> ControllerPrincipal:
+ return ControllerPrincipal(controller_id="c1", owner_id="u1", tenant_id="t1")
+
+
+async def test_a_controller_refused_on_the_socket_is_told_why_in_a_close() -> None:
+ reached: list[Any] = []
+
+ async def _app(scope: Any, receive: Any, send: Any) -> None:
+ reached.append(scope)
+
+ mw = BearerAuthMiddleware(
+ _app,
+ agent_store=AgentStore(),
+ api_key_store=ApiKeyStore(),
+ api_key_cache=ApiKeyCache(ttl_seconds=5, max_entries=8),
+ session_factory=None, # type: ignore[arg-type]
+ controller_auth=_NotAssigned(), # type: ignore[arg-type]
+ )
+
+ sent = await _call(
+ mw,
+ {
+ "type": "websocket",
+ "path": "/agents/a1/connection/ws",
+ "headers": [(b"authorization", b"Bearer controller-token")],
+ "query_string": b"",
+ },
+ )
+
+ assert [m["type"] for m in sent] == [
+ "websocket.accept",
+ "websocket.send",
+ "websocket.close",
+ ]
+ frame = json.loads(sent[1]["text"])
+ assert frame["event"] == "refused"
+ assert frame["data"]["status"] == 403
+ assert frame["data"]["detail"]["code"] == "not_assigned"
+ assert sent[2]["code"] == 4403
+ assert reached == []
diff --git a/core/tests/switch_core/gateway/test_hosted_workers.py b/core/tests/switch_core/gateway/test_hosted_workers.py
index 386611c5c..43708d6ca 100644
--- a/core/tests/switch_core/gateway/test_hosted_workers.py
+++ b/core/tests/switch_core/gateway/test_hosted_workers.py
@@ -13,11 +13,16 @@ from uuid import uuid4
import httpx
import pytest
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI, HTTPException, WebSocketDisconnect
from sqlalchemy import event as orm_event
from sqlalchemy import select
-from switch_core.bridges.agent.api.handlers import connection_placements, poll_events
+from switch_core.bridges.agent.api import handlers
+from switch_core.bridges.agent.api.handlers import (
+ connection_placements,
+ connection_socket,
+ poll_events,
+)
from switch_core.bridges.agent.api.hosted_cutover_routes import (
router as hosted_cutover_router,
)
@@ -1231,3 +1236,134 @@ async def test_relay_with_real_auth_opens_two_transactions(worker_app):
json={"message": {"health": True}, "timeout_ms": 50},
)
).status_code == 401
+
+
+class _Socket:
+ """The handler's side of a WebSocket, in this test's event loop.
+
+ Records what is sent and how it closes, and hands the handler whatever the
+ test queues as the client's messages; `None` is the client going away.
+ """
+
+ def __init__(self) -> None:
+ self.sent: list[dict[str, Any]] = []
+ self.close_code: int | None = None
+ self._incoming: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
+ self._changed = asyncio.Event()
+
+ async def accept(self) -> None:
+ return None
+
+ async def send_json(self, message: dict[str, Any]) -> None:
+ self.sent.append(message)
+ self._changed.set()
+
+ async def receive_json(self) -> dict[str, Any]:
+ message = await self._incoming.get()
+ if message is None:
+ raise WebSocketDisconnect(1000)
+ return message
+
+ async def close(self, code: int = 1000) -> None:
+ if self.close_code is None:
+ self.close_code = code
+ self._changed.set()
+
+ def leave(self) -> None:
+ self._incoming.put_nowait(None)
+
+ async def frames(self, count: int) -> list[tuple[str, dict[str, Any]]]:
+ """The next `count` frames sent, pings skipped."""
+
+ def received() -> list[tuple[str, dict[str, Any]]]:
+ return [(m["event"], m["data"]) for m in self.sent if m["event"] != "ping"]
+
+ async def wait() -> None:
+ while len(received()) < count:
+ self._changed.clear()
+ await self._changed.wait()
+
+ await asyncio.wait_for(wait(), timeout=2)
+ return received()[:count]
+
+
+async def _open_socket(
+ service,
+ agent: Agent,
+ *,
+ capability: str | None,
+ connection_id: str | None = None,
+ speaks: int = 7,
+ state_version: int | None = 1,
+) -> tuple[_Socket, asyncio.Task[None]]:
+ socket = _Socket()
+ task = asyncio.create_task(
+ connection_socket(
+ websocket=socket, # type: ignore[arg-type]
+ agent_id=agent.id,
+ agent=agent,
+ protocol=service,
+ config=service.config,
+ connection_id=connection_id or str(uuid4()),
+ scope="all",
+ spawn_capable=True,
+ protocol_version=speaks,
+ worker_capability=capability,
+ host_boot_id="boot-a",
+ host_instance_id="instance-a",
+ worker_state_version=state_version,
+ )
+ )
+ return socket, task
+
+
+ ("change", "refused"),
+ [
+ ({"capability": None}, (403, "worker_capability_required")),
+ ({"capability": "not-it"}, (403, "worker_capability_obsolete")),
+ ({"speaks": 6}, (426, "upgrade_required")),
+ ({"state_version": None}, (426, "upgrade_required")),
+ ],
+)
+async def test_the_socket_refuses_a_worker_open_as_the_stream_does(
+ worker_app, change: dict[str, Any], refused: tuple[int, str]
+):
+ _, _, agent_id, service, factory, prepared = worker_app
+ agent = await _agent(factory, agent_id)
+ opening = {"capability": prepared["worker_capability"], **change}
+ assert await _refusal(_open(service, agent, **opening)) == refused
+
+ socket, task = await _open_socket(service, agent, **opening)
+ await asyncio.wait_for(task, timeout=2)
+
+ status, code = refused
+ ((event, data),) = await socket.frames(1)
+ assert event == "refused"
+ assert data["status"] == status
+ assert data["detail"]["code"] == code
+ assert socket.close_code == 4000 + status
+ assert service.connections.for_agent(agent_id) == []
+
+
+async def test_a_worker_over_the_socket_is_sent_its_frames(worker_app, monkeypatch):
+ # Pings every 50 ms, so the socket notices its client leave promptly.
+ monkeypatch.setattr(handlers, "HEARTBEAT_INTERVAL_SECONDS", 0.05)
+ _, _, agent_id, service, factory, prepared = worker_app
+ agent = await _agent(factory, agent_id)
+ conn_id = str(uuid4())
+ socket, task = await _open_socket(
+ service, agent, capability=prepared["worker_capability"], connection_id=conn_id
+ )
+ try:
+ (state, _), (attached, data) = await socket.frames(2)
+ assert (state, attached) == ("connection_state", "worker_attached")
+ assert set(data) == WORKER_ATTACHED_FIELDS
+ conn = service.connections.get(conn_id)
+ assert conn.worker is not None and conn.stream_transport == "websocket"
+
+ assert service.connections.ring_worker(agent_id, "wake", {"entries": []})
+ assert (await socket.frames(3))[2] == ("wake", {"entries": []})
+ finally:
+ socket.leave()
+ await asyncio.wait_for(task, timeout=2)
diff --git a/core/tests/switch_core/session_activity/test_command_relay.py b/core/tests/switch_core/session_activity/test_command_relay.py
index 4e34b0bbe..8f95f2ce6 100644
--- a/core/tests/switch_core/session_activity/test_command_relay.py
+++ b/core/tests/switch_core/session_activity/test_command_relay.py
@@ -1,20 +1,20 @@
from __future__ import annotations
import asyncio
-import json
import httpx
import pytest
from fastapi import FastAPI
from switch_core.bridges.agent.protocol.agent_connections import (
+ PROTOCOL_VERSION,
SESSION_COMMAND_PROTOCOL_REVISION,
AgentConnectionRegistry,
ClientDeclaration,
)
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
from switch_core.bridges.agent.protocol.hosted_workers import WorkerBinding
-from switch_core.bridges.agent.protocol.stream import event_stream
+from switch_core.bridges.agent.protocol.stream import event_frames
from switch_core.db.models import User
from switch_core.gateway.agent_sessions import router
from switch_core.gateway.auth import get_current_user
@@ -33,7 +33,9 @@ class _Protocol:
self.connections = registry
-def _watcher(registry: AgentConnectionRegistry, *, speaks: int, scope: str = "all"):
+def _watcher(
+ registry: AgentConnectionRegistry, *, speaks: int | None, scope: str = "all"
+):
conn = registry.open(
agent_id=AGENT,
connection_id=f"conn-{scope}-{speaks}",
@@ -44,7 +46,7 @@ def _watcher(registry: AgentConnectionRegistry, *, speaks: int, scope: str = "al
declaration=ClientDeclaration(speaks=speaks),
expected_generation=None,
)
- stream = event_stream(
+ stream = event_frames(
conn=conn,
registry=registry,
buffer=EventBuffer(sequence_base=0),
@@ -75,15 +77,15 @@ async def owner(session_factory, people) -> str:
async def test_a_relayed_session_command_reaches_the_watcher_stream() -> None:
registry = AgentConnectionRegistry()
- _, stream = _watcher(registry, speaks=SESSION_COMMAND_PROTOCOL_REVISION)
+ _, stream = _watcher(registry, speaks=PROTOCOL_VERSION)
try:
await anext(stream) # connection_state
waiting = asyncio.create_task(anext(stream))
await asyncio.sleep(0.05)
assert registry.relay_session_command(AGENT, FRAME, worker_only=False) is True
- text = (await asyncio.wait_for(waiting, 5)).decode()
- assert "event: session_command\n" in text
- assert json.loads(text.split("data: ", 1)[1]) == FRAME
+ frame = await asyncio.wait_for(waiting, 5)
+ assert frame.event == "session_command"
+ assert frame.data == FRAME
finally:
await stream.aclose()
@@ -91,8 +93,8 @@ async def test_a_relayed_session_command_reaches_the_watcher_stream() -> None:
@pytest.mark.parametrize(
("speaks", "scope"),
[
- (SESSION_COMMAND_PROTOCOL_REVISION - 1, "all"),
- (SESSION_COMMAND_PROTOCOL_REVISION, "single"),
+ (None, "all"),
+ (PROTOCOL_VERSION, "single"),
],
)
async def test_a_command_nobody_can_take_is_not_relayed(speaks, scope) -> None:
@@ -109,10 +111,8 @@ async def test_switch_does_not_answer_where_an_agents_sessions_are(
) -> None:
"""The agent's room watcher owns its connection and placements; Console asks it."""
registry = AgentConnectionRegistry()
- _watcher(registry, speaks=SESSION_COMMAND_PROTOCOL_REVISION)
- registry.place_session(
- AGENT, "session-1", "room-1", f"conn-all-{SESSION_COMMAND_PROTOCOL_REVISION}"
- )
+ _watcher(registry, speaks=PROTOCOL_VERSION)
+ registry.place_session(AGENT, "session-1", "room-1", f"conn-all-{PROTOCOL_VERSION}")
async with _client(session_factory, registry, owner) as client:
assert (await client.get("/agent-sessions/room-health")).status_code == 404
diff --git a/core/tests/switch_core/session_activity/test_stop_press.py b/core/tests/switch_core/session_activity/test_stop_press.py
index bcc949617..6c8653d6c 100644
--- a/core/tests/switch_core/session_activity/test_stop_press.py
+++ b/core/tests/switch_core/session_activity/test_stop_press.py
@@ -7,7 +7,7 @@ from sqlalchemy import update
from switch_core.addressing import owner_only_policy
from switch_core.bridges.agent.protocol.agent_connections import (
- SESSION_COMMAND_PROTOCOL_REVISION,
+ PROTOCOL_VERSION,
AgentConnectionRegistry,
ClientDeclaration,
)
@@ -49,7 +49,7 @@ def _watching(registry: AgentConnectionRegistry):
delivery_filter="addressed",
spawn_capable=False,
cursor=0,
- declaration=ClientDeclaration(speaks=SESSION_COMMAND_PROTOCOL_REVISION),
+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
expected_generation=None,
)
conn.stream_attached = True
diff --git a/core/tests/switch_core/session_activity/test_stream_outcomes.py b/core/tests/switch_core/session_activity/test_stream_outcomes.py
index c500044cf..44e5b796c 100644
--- a/core/tests/switch_core/session_activity/test_stream_outcomes.py
+++ b/core/tests/switch_core/session_activity/test_stream_outcomes.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import contextlib
-import json
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
@@ -12,13 +11,12 @@ from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool
from switch_core.bridges.agent.protocol.agent_connections import (
- APPROVAL_OUTCOME_PROTOCOL_REVISION,
PROTOCOL_VERSION,
AgentConnectionRegistry,
ClientDeclaration,
)
from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
-from switch_core.bridges.agent.protocol.stream import event_stream
+from switch_core.bridges.agent.protocol.stream import Frame, event_frames
from switch_core.db.models import ApprovalRequest
from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.outcomes import ApprovalOutcomes
@@ -61,7 +59,7 @@ def _open_stream(approvals, *, agent_id=AGENT, scope="all", speaks=PROTOCOL_VERS
declaration=ClientDeclaration(speaks=speaks),
expected_generation=None,
)
- return event_stream(
+ return event_frames(
conn=conn,
registry=registry,
buffer=EventBuffer(sequence_base=0),
@@ -69,10 +67,9 @@ def _open_stream(approvals, *, agent_id=AGENT, scope="all", speaks=PROTOCOL_VERS
)
-def _outcome(frame: bytes) -> dict:
- text = frame.decode()
- assert "event: approval_outcome\n" in text, text
- return json.loads(text.split("data: ", 1)[1])
+def _outcome(frame: Frame) -> dict:
+ assert frame.event == "approval_outcome", frame
+ return frame.data
async def _no_frame_within(stream, seconds: float) -> None:
@@ -228,7 +225,7 @@ async def test_a_stream_without_approvals_carries_none(service, people):
await stream.aclose()
-async def test_a_client_older_than_the_outcome_revision_is_sent_none(
+async def test_a_client_that_declared_no_revision_is_sent_none(
service, approvals, people
):
# An older client hands unknown frames to its room-event path and breaks.
@@ -240,10 +237,9 @@ async def test_a_client_older_than_the_outcome_revision_is_sent_none(
answer=pick("allow"),
answerer=PlatformPerson(people.owner),
)
- for speaks in (APPROVAL_OUTCOME_PROTOCOL_REVISION - 1, None):
- stream = _open_stream(approvals, speaks=speaks)
- try:
- await anext(stream)
- await _no_frame_within(stream, 0.3)
- finally:
- await stream.aclose()
+ stream = _open_stream(approvals, speaks=None)
+ try:
+ await anext(stream)
+ await _no_frame_within(stream, 0.3)
+ finally:
+ await stream.aclose()