Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Move the agent connection to one WebSocket, server and runtime (CHOO-3013) (#652)
Abel Dantas8a419be67612
diff --git a/artifacts.yaml b/artifacts.yamlindex e465cb4da..a6090ab16 100644--- a/artifacts.yaml+++ b/artifacts.yaml@@ -181,9 +181,24 @@ contracts:# attached worker alone is sent `worker_attached`, `relay`, `relay_cancel`,# `wake`, `mailbox_cancel`, `operation` and `credential`. Nothing changes# for a local agent, so `accepts` stays at 1.+ #+ # Revision 8 moves the connection onto one WebSocket,+ # `/agents/{id}/connection/ws`, which carries the frames as JSON+ # (`event`, `data`, `id`) and the heartbeat: the server sends `ping` every+ # heartbeat interval and the client answers `pong` with its cursor. The+ # token is checked once, when the socket opens. A hosted worker opens it+ # with the headers and under the rules revision 7 set for the stream. It+ # is meant to replace the Server-Sent Events stream and+ # `POST /connection/beat`, which would make it+ # the first revision that is not additive, but that removal is deferred:+ # the stream and the beat are kept beside the socket for a compatibility+ # window, so `accepts` stays at 1 on both sides. switch-core serves either+ # transport, and agent-runtime prefers the socket and falls back to the+ # stream and its beat against a server with no socket. Both go, and+ # `accepts` rises to 8, once no client still connects over the stream.artifacts:- switch-core: { speaks: 7, accepts: 1 }- agent-runtime: { speaks: 7, accepts: 1 }+ switch-core: { speaks: 8, accepts: 1 }+ agent-runtime: { speaks: 8, accepts: 1 }gateway-api:description: >-diff --git a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.tsindex f8a3bf8b4..ef695c384 100644--- a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.test.ts@@ -23,7 +23,7 @@ function makeDeps(opts: {endpoint?: string;}) {const missing = opts.missingBinaries ?? [];- const nodeVersion = opts.nodeVersion ?? 'v18.19.0';+ const nodeVersion = opts.nodeVersion ?? 'v22.12.0';const exec = vi.fn(async (command: string, _args: string[]) => {// Combined working-dir + required-tools probe (`sh -c <script>`). A missing// dir or exhausted channel rejects; otherwise missing tools and node's@@ -119,7 +119,7 @@ describe('preflightRemoteSession', () => {it('fails loud when node is present but too old', async () => {const deps = makeDeps({ nodeVersion: 'v12.22.9', credsFile: CREDS_FILE });await expect(preflightRemoteSession(deps)).rejects.toThrow(- /has Node v12.22.9, but the Switch Console sidecar needs Node 18 or newer/+ /has Node v12.22.9, but the Switch Console sidecar needs Node 22 or newer/);});diff --git a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.tsindex 43a0a0f76..cda59aa53 100644--- a/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts+++ b/console/apps/switch-console-desktop/src/main/core/sessions/remote-session-preflight.ts@@ -15,9 +15,8 @@ import { parseSwitchAgentCredentials } from '@main/core/switch-rooms/switch-cred* exec context prepends `cd <workDir> &&`, so a missing dir rejects the whole* command, while present-but-missing tools and node's version are reported on* stdout. A too-old node is rejected here rather than surfacing later as an- * opaque "sidecar exited during startup" — the sidecar bundle (and this- * module's own reachability probe) rely on `fetch`/optional chaining, which- * only stabilised in Node 18;+ * opaque "sidecar exited during startup" — the sidecar holds its agent+ * connection on Node's built-in WebSocket, stable only from Node 22;* 2. the agent's Switch creds exist on the remote host — checked at the agent's* provider-neutral per-agent path (`.switch/agents/<name>.json`) first, then* the legacy `.claude/settings.local.json` for un-migrated installs@@ -32,9 +31,9 @@ import { parseSwitchAgentCredentials } from '@main/core/switch-rooms/switch-credconst REQUIRED_BINARIES = ['node', 'git'] as const;const REACHABILITY_TIMEOUT_MS = 5000;-// Global `fetch` and `AbortSignal.timeout` (used by the reachability probe below-// and throughout the sidecar bundle) are only stable from Node 18.-const MIN_NODE_MAJOR = 18;+// The sidecar holds its agent connection on Node's built-in `WebSocket`, which+// is only stable from Node 22.+const MIN_NODE_MAJOR = 22;// Prints `missing <tool>` for each absent binary and, when node is present,// `node <version>` (e.g. `node v18.19.0`), then exits 0 — so a present workingdiff --git a/console/packages/agent-controller/src/relay.test.ts b/console/packages/agent-controller/src/relay.test.tsindex a8fa429ca..423c0bd5d 100644--- a/console/packages/agent-controller/src/relay.test.ts+++ b/console/packages/agent-controller/src/relay.test.ts@@ -212,8 +212,13 @@ async function post(path: string, body: unknown, bearer = token): Promise<Respon}describe('the relay as the agent protocol, read by the real SwitchEventStream', () => {- it('serves the relay’s protocol range as switch-core declares its own', () => {- expect(RELAY_AGENT_PROTOCOL).toEqual(CONTRACTS['agent-protocol']['switch-core']);+ it('serves switch-core’s revisions up to the socket, which the runtime still accepts', () => {+ const core = CONTRACTS['agent-protocol']['switch-core'];+ const runtime = CONTRACTS['agent-protocol']['agent-runtime'];+ // The relay serves the event stream, not the socket of agent-protocol 8.+ expect(RELAY_AGENT_PROTOCOL).toEqual({ speaks: 7, accepts: core.accepts });+ expect(runtime.accepts).toBeLessThanOrEqual(RELAY_AGENT_PROTOCOL.speaks);+ expect(RELAY_AGENT_PROTOCOL.accepts).toBeLessThanOrEqual(runtime.speaks);});it('connects, then delivers addressed events in order, with their ids and missed counts', async () => {diff --git a/console/packages/agent-controller/src/relay.ts b/console/packages/agent-controller/src/relay.tsindex dc2d7c00e..f1adbeb0a 100644--- a/console/packages/agent-controller/src/relay.ts+++ b/console/packages/agent-controller/src/relay.ts@@ -28,7 +28,12 @@ import type {* `AgentHub` instead; the room of its calls comes from there too.*/-/** The agent-protocol revisions the relay serves, as switch-core declares its own. */+/**+ * The agent-protocol revisions the relay serves: switch-core's, up to the+ * socket. Revision 8 moved the connection onto a WebSocket, which the relay+ * does not serve; it serves the event stream and its beat, which the runtime+ * falls back to against a server that speaks 7 or older.+ */export const RELAY_AGENT_PROTOCOL = { speaks: 7, accepts: 1 } as const;/** From this revision a client names its connection incarnation on every beat and room request. */const FENCED_PROTOCOL_REVISION = 2;diff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.tsindex 3b2e38ae9..4fba00fe4 100644--- a/console/packages/shared/src/artifacts.ts+++ b/console/packages/shared/src/artifacts.ts@@ -40,8 +40,8 @@ export function artifactVersion(artifact: ArtifactName): string {export const CONTRACTS = {// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.'agent-protocol': {- 'switch-core': { speaks: 7, accepts: 1 },- 'agent-runtime': { speaks: 7, accepts: 1 },+ 'switch-core': { speaks: 8, accepts: 1 },+ 'agent-runtime': { speaks: 8, accepts: 1 },},// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.'gateway-api': {diff --git a/console/packages/switch-agent-runtime/package.json b/console/packages/switch-agent-runtime/package.jsonindex d6f434eb3..3b7b32f11 100644--- a/console/packages/switch-agent-runtime/package.json+++ b/console/packages/switch-agent-runtime/package.json@@ -44,6 +44,6 @@"vitest": "^5.0.1"},"engines": {- "node": ">=20"+ "node": ">=22"}}diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.tsindex 3b2e38ae9..4fba00fe4 100644--- a/console/packages/switch-agent-runtime/src/artifacts.ts+++ b/console/packages/switch-agent-runtime/src/artifacts.ts@@ -40,8 +40,8 @@ export function artifactVersion(artifact: ArtifactName): string {export const CONTRACTS = {// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.'agent-protocol': {- 'switch-core': { speaks: 7, accepts: 1 },- 'agent-runtime': { speaks: 7, accepts: 1 },+ 'switch-core': { speaks: 8, accepts: 1 },+ 'agent-runtime': { speaks: 8, accepts: 1 },},// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.'gateway-api': {diff --git a/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts b/console/packages/switch-agent-runtime/src/event-stream.old-server.test.tsnew file mode 100644index 000000000..d53477775--- /dev/null+++ b/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts@@ -0,0 +1,2296 @@+import { afterEach, describe, expect, it, vi } from 'vitest';+import {+ BEAT_INTERVAL_MS,+ BEAT_SETTLE_LIMIT_MS,+ EVICTION_CREDENTIALS_REJECTED,+ EVICTION_LAUNCH_SUPERSEDED,+ EVICTION_TAKEN_OVER,+ SwitchEventStream,+ WORKER_CAPABILITY_OBSOLETE,+ WorkerCallError,+ type Eviction,+ type SwitchEventStreamDeps,+} from './event-stream';++/**+ * The runtime against a server from before the socket (agent-protocol 7 and+ * older).+ *+ * Such a server has no `/connection/ws`: the upgrade fails before it opens,+ * which Node's WebSocket reports as an error and a 1006 close and nothing+ * more. The runtime falls back to the event stream and its own heartbeat, and+ * has to behave there exactly as it did before the socket, so these are the+ * tests it had then, run with a socket that never opens. The ones on choosing+ * the transport are at the end.+ *+ * Two ways a client can retry something that can never succeed were measured+ * on a live deployment: a stream re-declaring a room that had been deleted,+ * refused on every open, reopening at once; and a heartbeat treating "your+ * connection is gone" as a normal answer and beating on at full cadence.+ * Neither could self-heal, and between them they produced most of the error+ * volume on that deployment.+ */++const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };++function silentLog() {+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };+}++function encodeFrame(name: string, data: unknown): Uint8Array {+ return new TextEncoder().encode(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);+}++/** The frame a server opens every stream with, naming the incarnation this+ * client is attached to. No `rooms`, so it says nothing about the declared set+ * a test may be asserting on. */+function connected(generation = 0): Uint8Array {+ return encodeFrame('connection_state', { connection_id: 'conn-1', generation });+}++/** A stream body carrying one frame and then closing, the way the server ends+ * a displaced stream. */+function frameThenClose(name: string, data: unknown): ReadableStream<Uint8Array> {+ return new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(encodeFrame(name, data));+ controller.close();+ },+ });+}++/**+ * A connected stream body that stays open, so the loop neither reconnects nor+ * spins.+ *+ * It announces the connection first because the heartbeat waits for that: a+ * client that has not been told which incarnation it is cannot fence its own+ * tick, and a body that never announces is a connection that never beats.+ */+function openForever(): ReadableStream<Uint8Array> {+ return new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected());+ },+ });+}++/**+ * A connected stream body that ends when the client drops the socket.+ *+ * `openForever` cannot: its reader is parked on a read that never returns, so+ * a reopen is invisible to the test rather than absent. A body that honours+ * the request's signal is what a real socket does, and what a test asserting+ * on reopens needs.+ */+function openUntilAborted(init: { signal: AbortSignal }): ReadableStream<Uint8Array> {+ return new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected());+ init.signal.addEventListener('abort', () => controller.close(), { once: true });+ },+ });+}++/** The incarnation each beat carried, in order. */+function beatGenerations(fetchMock: { mock: { calls: unknown[][] } }): (number | null)[] {+ return fetchMock.mock.calls+ .filter((call) => String(call[0]).includes('connection/beat'))+ .map(+ (call) =>+ (JSON.parse((call[1] as { body: string }).body) as { generation: number | null }).generation+ );+}++function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {+ return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));+}++/**+ * The socket an old server's upgrade gives: it never opens. Its error and+ * close arrive on the microtask queue, the way a real socket's arrive as+ * separate tasks, once the client has attached its listeners.+ */+class SocketThatNeverOpens {+ static opened: string[] = [];+ readonly CONNECTING = 0;+ readonly OPEN = 1;+ readyState = 0;+ private readonly listeners = new Map<string, Set<(event: unknown) => void>>();++ constructor(url: string) {+ SocketThatNeverOpens.opened.push(url);+ queueMicrotask(() => {+ this.readyState = 3;+ for (const listener of [...(this.listeners.get('error') ?? [])]) listener({});+ for (const listener of [...(this.listeners.get('close') ?? [])])+ listener({ code: 1006, reason: '' });+ });+ }++ addEventListener(type: string, listener: (event: unknown) => void): void {+ const set = this.listeners.get(type) ?? new Set();+ set.add(listener);+ this.listeners.set(type, set);+ }++ removeEventListener(type: string, listener: (event: unknown) => void): void {+ this.listeners.get(type)?.delete(listener);+ }++ send(): void {+ throw new Error('nothing is sent on a socket that never opened');+ }++ close(): void {}+}++function makeStream(+ fetchMock: ReturnType<typeof vi.fn>,+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }+) {+ vi.stubGlobal('WebSocket', SocketThatNeverOpens);+ SocketThatNeverOpens.opened = [];+ vi.stubGlobal('fetch', fetchMock);+ const abort = new AbortController();+ const log = silentLog();+ const stream = new SwitchEventStream({+ creds,+ connectionId: 'conn-1',+ scope: 'single',+ filter: 'all',+ onEvent: () => {},+ onGap: () => {},+ onEvicted: () => {},+ worker: null,+ log,+ signal: abort.signal,+ ...deps,+ });+ stream.start();+ return { stream, abort, log };+}++async function flush(times = 8): Promise<void> {+ for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));+}++afterEach(() => {+ vi.unstubAllGlobals();+ vi.useRealTimers();+});++describe('a room the server refuses', () => {+ /** The refusal switch-core returns when a declared room no longer exists. */+ function roomGone(roomId: string) {+ return {+ ok: false,+ status: 403,+ body: null,+ text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),+ };+ }++ it('is dropped and not declared again', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events')) {+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ return String(url).includes('rooms=')+ ? roomGone('room-dead')+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const rejected: { roomId: string; status: number }[] = [];+ const reported: string[][] = [];+ const { abort, log } = makeStream(fetchMock, {+ rooms: ['room-dead'],+ onRooms: (rooms) => reported.push(rooms),+ onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),+ });+ await flush();++ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(2);+ expect(opens[0]).toContain('rooms=room-dead');+ expect(opens[1]).not.toContain('rooms=');+ expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);+ expect(reported).toEqual([[]]);+ expect(log.error).toHaveBeenCalled();+ abort.abort();+ });++ it('keeps the rooms the refusal does not name', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ const u = String(url);+ if (!u.includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ return u.includes('room-dead')+ ? roomGone('room-dead')+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });+ await flush();++ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(2);+ expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');+ abort.abort();+ });++ it('stays a transport error when the body names no declared room', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events')) {+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ return {+ ok: false,+ status: 404,+ body: null,+ text: async (): Promise<string> => JSON.stringify({ detail: 'No such connection' }),+ };+ });+ const rejected: string[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onRoomRejected: ({ roomId }) => rejected.push(roomId),+ });+ await vi.advanceTimersByTimeAsync(0);++ // One open, then the backoff, not an immediate retry, and nothing dropped.+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(rejected).toEqual([]);+ abort.abort();+ });+});++describe('credentials the server rejects', () => {+ function refusal(status: number, detail: string) {+ return {+ ok: false,+ status,+ body: null,+ text: async (): Promise<string> => JSON.stringify({ detail }),+ };+ }++ /** Open once, be refused, and let a long while pass. */+ async function afterRefusal(status: number, detail: string) {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? refusal(status, detail)+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const evicted: Eviction[] = [];+ const { abort, log } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onEvicted: (eviction) => evicted.push(eviction),+ });+ await vi.advanceTimersByTimeAsync(0);+ const settled = fetchMock.mock.calls.length;+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ abort.abort();+ return { fetchMock, evicted, log, settled };+ }++ it('ends the stream and the heartbeat on a 401, and says so once', async () => {+ const { fetchMock, evicted, log, settled } = await afterRefusal(401, 'Invalid agent token');++ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ // Nothing beyond the beat already in flight when the refusal landed.+ expect(fetchMock.mock.calls).toHaveLength(settled);+ expect(evicted).toHaveLength(1);+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);+ expect(evicted[0]?.reason).toContain('credentials');+ expect(evicted[0]?.reason).toContain('401');+ expect(log.error).toHaveBeenCalled();+ });++ it('ends the stream on a 403 that names no declared room', async () => {+ const { fetchMock, evicted, settled } = await afterRefusal(+ 403,+ 'Agent is not a member of this room'+ );++ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(fetchMock.mock.calls).toHaveLength(settled);+ expect(evicted).toHaveLength(1);+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);+ expect(evicted[0]?.reason).toContain('credentials');+ expect(evicted[0]?.reason).toContain('403');+ });++ it('ends the stream and the heartbeat on a 401 heartbeat, and says so once', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }+ : refusal(401, 'Invalid agent token')+ );+ const evicted: Eviction[] = [];+ const { abort, log } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onEvicted: (eviction) => evicted.push(eviction),+ });+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS + 1);+ const settled = fetchMock.mock.calls.length;+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ abort.abort();++ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);+ expect(fetchMock.mock.calls).toHaveLength(settled);+ expect(evicted).toHaveLength(1);+ expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);+ expect(evicted[0]?.reason).toContain('credentials');+ expect(evicted[0]?.reason).toContain('401');+ expect(log.error).toHaveBeenCalledTimes(1);+ });++ it('tells a session start to the room itself only where the server says it leaves that to it', async () => {+ for (const [said, expected] of [+ [{ announce_session_starts: true }, true],+ [{}, false],+ ] as const) {+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: frameThenClose('connection_state', {+ connection_id: 'conn-1',+ generation: 0,+ ...said,+ }),+ text: async (): Promise<string> => '',+ }+ : new Promise<never>(() => {})+ );+ let connected = false;+ const { stream, abort } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onConnected: () => {+ connected = true;+ },+ });+ expect(stream.announcesSessionStarts).toBe(false);+ for (let i = 0; i < 50 && !connected; i += 1) await flush();+ expect(stream.announcesSessionStarts).toBe(expected);+ abort.abort();+ vi.unstubAllGlobals();+ }+ });++ it('tells the owner once when the stream and the heartbeat are refused together', async () => {+ vi.useFakeTimers();+ let opens = 0;+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events')) {+ // Connected once (the heartbeat has nothing to beat for until then)+ // and refused on every reopen after that.+ opens += 1;+ if (opens === 1) {+ return {+ ok: true,+ status: 200,+ body: frameThenClose('connection_state', { connection_id: 'conn-1', generation: 0 }),+ text: async (): Promise<string> => '',+ };+ }+ await new Promise((r) => setTimeout(r, BEAT_INTERVAL_MS));+ return refusal(401, 'Invalid agent token');+ }+ // Slow enough to still be in flight when the reopen is refused: both+ // doors closing at once is the case this test is about.+ await new Promise((r) => setTimeout(r, 2 * BEAT_INTERVAL_MS));+ return refusal(401, 'Invalid agent token');+ });+ const evicted: Eviction[] = [];+ const { abort, log } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onEvicted: (eviction) => evicted.push(eviction),+ });+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ abort.abort();++ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);+ expect(evicted).toHaveLength(1);+ expect(log.error).toHaveBeenCalledTimes(1);+ });++ it.each([503, 429])('keeps reconnecting after HTTP %i', async (status) => {+ vi.useFakeTimers();+ let opens = 0;+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ opens += 1;+ return opens === 1+ ? { ok: false, status, body: null, text: async (): Promise<string> => 'try later' }+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(2000);++ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(evicted).toEqual([]);+ abort.abort();+ });++ it('keeps reconnecting after a network error', async () => {+ vi.useFakeTimers();+ let opens = 0;+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ opens += 1;+ if (opens === 1) throw new TypeError('fetch failed');+ return { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(2000);++ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(evicted).toEqual([]);+ abort.abort();+ });+});++describe('the heartbeat', () => {+ /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */+ async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events'))+ return {+ ok: true,+ status: 200,+ body: openForever(),+ text: async (): Promise<string> => '',+ };+ return { ok: false, status, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });+ await vi.advanceTimersByTimeAsync(windowMs);+ const beats = urlsFor(fetchMock, 'connection/beat').length;+ abort.abort();+ return beats;+ }++ it('backs off when the connection is rejected, instead of beating at full rate', async () => {+ const windowMs = 20 * BEAT_INTERVAL_MS;+ // At the base cadence this window holds ~20 beats. Doubling from the base+ // gives 4s, 8s, 16s…, four requests in the same window.+ expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);+ expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);+ });++ it('keeps backing off the longer the rejection lasts', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events'))+ return {+ ok: true,+ status: 200,+ body: openForever(),+ text: async (): Promise<string> => '',+ };+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(30_000);+ const early = urlsFor(fetchMock, 'connection/beat').length;+ await vi.advanceTimersByTimeAsync(30_000);+ const late = urlsFor(fetchMock, 'connection/beat').length - early;++ expect(late).toBeLessThan(early);+ abort.abort();+ });++ it('sends nothing until the server has said which incarnation it is', async () => {+ vi.useFakeTimers();+ const server = { announce: (): void => {} };+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ server.announce = () => controller.enqueue(connected(7));+ },+ }),+ text: async (): Promise<string> => '',+ };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);+ // An unfenced tick is what a displaced client sends, and the server refuses+ // it. Beating before the first frame would make every healthy connection+ // open with one.+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(0);++ server.announce();+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);++ const beats = fetchMock.mock.calls.filter((c) => String(c[0]).includes('connection/beat'));+ expect(beats.length).toBeGreaterThan(0);+ // And it carries the incarnation that frame named, so the server can fence it.+ const [, sent] = beats[0] as unknown as [string, { body: string }];+ expect(JSON.parse(sent.body).generation).toBe(7);+ abort.abort();+ });++ it('waits for the new incarnation after its own reconnect, instead of standing down', async () => {+ vi.useFakeTimers();+ // A server that fences: every attach is a new incarnation, and a tick+ // naming an older one is refused the way a displaced client's is.+ const server = { generation: 0, announce: (): void => {}, drop: (): void => {} };+ const fetchMock = vi.fn(async (url: string, init: { body: string }) => {+ if (String(url).includes('/events')) {+ server.generation += 1;+ const attached = server.generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ const announce = (): void => controller.enqueue(connected(attached));+ // The first attach is announced at once; the second is held, so+ // the beat falls due inside the window this test is about.+ if (attached === 1) announce();+ else server.announce = announce;+ server.drop = () => controller.close();+ },+ }),+ text: async (): Promise<string> => '',+ };+ }+ const sent = (JSON.parse(init.body) as { generation: number | null }).generation;+ if (sent === server.generation)+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),+ };+ });+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });++ await vi.advanceTimersByTimeAsync(2 * BEAT_INTERVAL_MS);+ const first = beatGenerations(fetchMock);+ expect(first.length).toBeGreaterThan(0);+ expect(first.every((generation) => generation === 1)).toBe(true);++ server.drop();+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);+ // Nothing went out while the reopen was in flight: the incarnation we hold+ // is the one before it, and a tick carrying that is a stand-down.+ expect(beatGenerations(fetchMock)).toHaveLength(first.length);+ expect(evicted).toEqual([]);++ server.announce();+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ const resumed = beatGenerations(fetchMock).slice(first.length);+ expect(resumed.length).toBeGreaterThan(0);+ expect(resumed.every((generation) => generation === 2)).toBe(true);+ expect(evicted).toEqual([]);+ abort.abort();+ });++ it('does not stand down over a beat that was already in flight when it reopened', async () => {+ vi.useFakeTimers();+ // The gate stops a beat *starting* inside the reattach window. It cannot+ // recall one that left before the gate shut, and that beat is answered+ // against the incarnation the reopen replaced, a truthful `taken_over`+ // about a takeover this client performed on itself.+ const server = {+ generation: 0,+ announce: (): void => {},+ drop: (): void => {},+ answerBeat: (): void => {},+ beatsHeld: 0,+ };+ const fetchMock = vi.fn(async (url: string, init: { body: string }) => {+ if (String(url).includes('/events')) {+ const claimed = new URL(String(url)).searchParams.get('expected_generation');+ if (claimed !== null && Number(claimed) !== server.generation) {+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),+ };+ }+ server.generation += 1;+ const attached = server.generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected(attached));+ server.drop = () => controller.close();+ },+ }),+ text: async (): Promise<string> => '',+ };+ }+ const sent = (JSON.parse(init.body) as { generation: number | null }).generation;+ // Held rather than answered: the point of the test is an answer that+ // arrives after the connection has moved on beneath it.+ server.beatsHeld += 1;+ await new Promise<void>((resolve) => {+ server.answerBeat = resolve;+ });+ if (sent === server.generation)+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),+ };+ });+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });++ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ expect(server.beatsHeld).toBe(1);+ expect(beatGenerations(fetchMock)).toEqual([1]);++ // Reopen with that beat still outstanding. The bound expires, the open goes+ // ahead, and the server is at 2 by the time the old beat is answered.+ server.drop();+ await vi.advanceTimersByTimeAsync(BEAT_SETTLE_LIMIT_MS + 10 * BEAT_INTERVAL_MS);+ expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(1);++ server.answerBeat();+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);++ // Undecidable, so inert. Standing down here stops a client that nothing+ // has taken anything from.+ expect(evicted).toEqual([]);+ const resumed = beatGenerations(fetchMock).slice(1);+ expect(resumed.length).toBeGreaterThan(0);+ expect(resumed.every((generation) => generation === 2)).toBe(true);+ abort.abort();+ });++ it('stands down when its reattach is refused, rather than trying again', async () => {+ vi.useFakeTimers();+ // The other half: a client that missed its eviction and comes back. The+ // server refuses the reattach without disturbing the holder, and this is+ // the only thing left that can tell the loser it lost.+ const server = { generation: 0, drop: (): void => {} };+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ const claimed = new URL(String(url)).searchParams.get('expected_generation');+ if (claimed !== null) {+ // Someone else attached while we were away, so our claim is stale.+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),+ };+ }+ server.generation += 1;+ const attached = server.generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected(attached));+ server.drop = () => controller.close();+ },+ }),+ text: async (): Promise<string> => '',+ };+ });+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });++ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ server.drop();+ await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);++ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);+ // And it stopped asking: retrying is how the pair trade the connection.+ const attempts = urlsFor(fetchMock, 'expected_generation=1').length;+ await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);+ expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(attempts);+ abort.abort();+ });++ it('returns to the base cadence once a beat lands', async () => {+ vi.useFakeTimers();+ let reject = true;+ // A socket that ends when the client drops it: each refusal reopens, and+ // the heartbeat is held from the reopen until the new stream is named.+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (String(url).includes('/events'))+ return {+ ok: true,+ status: 200,+ body: openUntilAborted(init),+ text: async (): Promise<string> => '',+ };+ if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(30_000);+ reject = false;+ await vi.advanceTimersByTimeAsync(30_000);+ const recovered = urlsFor(fetchMock, 'connection/beat').length;+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);++ expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);+ abort.abort();+ });+});++describe('the heartbeat after a reconnect', () => {+ /**+ * A server that reattaches on every open, can drop the current stream, and+ * records when each stream was opened and each beat arrived. The server's+ * own rule is what these tests hold the client to: a new connection that+ * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.+ */+ const HEARTBEAT_TTL_MS = 6000;++ /** Either end may close a stream first; the second close is not an error here. */+ function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {+ try {+ controller.close();+ } catch {+ // Already closed by the other side.+ }+ }++ function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {+ const opens: number[] = [];+ const beats: number[] = [];+ let generation = 0;+ let current: ReadableStreamDefaultController<Uint8Array> | null = null;+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (String(url).includes('/events')) {+ opens.push(Date.now());+ generation += 1;+ const gen = generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ current = controller;+ controller.enqueue(connected(gen));+ init.signal.addEventListener('abort', () => closeQuietly(controller), {+ once: true,+ });+ },+ }),+ text: async (): Promise<string> => '',+ };+ }+ beats.push(Date.now());+ const reply = beatReply();+ if (reply === 'fail') throw new TypeError('fetch failed');+ if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ /** The server ends the current stream, as it does a lapsed connection. */+ const drop = (): void => {+ if (current) closeQuietly(current);+ };+ return { fetchMock, opens, beats, drop };+ }++ it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {+ // The lockout: beats fail for a while, so the gap between them climbs to+ // its 30-second cap. The stream then reopens. The server starts its+ // 6-second clock on the new connection, and the client used to sit out+ // the rest of the 30 seconds before saying anything, lose the connection+ // for silence, and repeat that on every connection it opened afterwards.+ vi.useFakeTimers();+ let failing = true;+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ // Long enough of failed beats for the gap to reach its cap.+ await vi.advanceTimersByTimeAsync(70_000);+ failing = false;+ server.drop();+ await vi.advanceTimersByTimeAsync(40_000);++ const reopenedAt = server.opens[1];+ expect(reopenedAt).toBeDefined();+ const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);+ expect(firstBeatAfter).toBeDefined();+ expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);+ abort.abort();+ });++ it('keeps beating at the base cadence on the new connection once it lands', async () => {+ vi.useFakeTimers();+ let failing = true;+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(70_000);+ failing = false;+ server.drop();+ await vi.advanceTimersByTimeAsync(5_000);+ const settled = server.beats.length;+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);++ expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);+ abort.abort();+ });++ it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {+ // The brake the old ordering provided, which beating promptly on every+ // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The+ // wait now sits before each reopen and doubles, so a refusing server sees+ // a handful of reopens, not one every couple of seconds per agent.+ vi.useFakeTimers();+ const server = reattachingServer(() => 404);+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(60_000);++ // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.+ expect(server.opens.length).toBeLessThanOrEqual(6);+ abort.abort();+ });++ it('does not throw away a connection the stream reopened while it waited', async () => {+ // A lapsed connection's socket is closed by the server, and the stream+ // loop reopens it by itself. If that lands during the heartbeat's wait+ // after a refusal, reopening again would discard a healthy connection.+ vi.useFakeTimers();+ let refuse = true;+ const server = reattachingServer(() => (refuse ? 404 : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ expect(server.beats.length).toBe(1);+ refuse = false;+ // The server ends the stream now, inside the heartbeat's wait.+ server.drop();+ await vi.advanceTimersByTimeAsync(20_000);++ // One open for the start, one for the stream's own reopen, not a third+ // from the heartbeat discarding it.+ expect(server.opens.length).toBe(2);+ abort.abort();+ });+});++describe('reopening while an open is still in flight', () => {+ /**+ * A server that keeps the incarnation the way the real one does: every open+ * it receives makes a new one, whether or not the client stays to read the+ * answer, and a reattach claiming an older one is refused as a takeover.+ * The next open can be held unanswered, the way a slow connection holds it.+ */+ function incarnationServer() {+ const state = {+ generation: 0,+ opens: 0,+ beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',+ subscribeDrops: false,+ holdNext: false,+ release: (): void => {},+ drop: (): void => {},+ };+ const body = (gen: number, signal: AbortSignal) =>+ new ReadableStream<Uint8Array>({+ start(controller) {+ const close = (): void => {+ try {+ controller.close();+ } catch {+ // already closed+ }+ };+ state.drop = close;+ controller.enqueue(connected(gen));+ signal.addEventListener('abort', close, { once: true });+ },+ });+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (String(url).includes('connection/subscribe')) {+ if (!state.subscribeDrops)+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ state.subscribeDrops = false;+ // The stream ends while the claim is still in flight.+ state.holdNext = true;+ state.drop();+ await new Promise((r) => setTimeout(r, 3_000));+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ if (!String(url).includes('/events')) {+ if (state.beat === 'slow-404-and-drop') {+ // The stream ends while this beat is in flight, and its answer comes+ // back just after the client has started reopening.+ state.beat = 'ok';+ state.holdNext = true;+ state.drop();+ await new Promise((r) => setTimeout(r, 1_200));+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ }+ if (state.beat === 404)+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ state.opens += 1;+ const claimed = new URL(String(url)).searchParams.get('expected_generation');+ if (claimed !== null && Number(claimed) < state.generation)+ return {+ ok: false,+ status: 409,+ body: null,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),+ };+ // The server acts on the open the moment it arrives.+ state.generation += 1;+ const gen = state.generation;+ const answer = {+ ok: true,+ status: 200,+ body: body(gen, init.signal),+ text: async (): Promise<string> => '',+ };+ if (!state.holdNext) return answer;+ state.holdNext = false;+ return new Promise((resolve, reject) => {+ state.release = () => resolve(answer);+ init.signal.addEventListener(+ 'abort',+ () => reject(new DOMException('aborted', 'AbortError')),+ { once: true }+ );+ });+ });+ return { state, fetchMock };+ }++ it('does not cancel an open the server has already acted on, and so does not stand down', async () => {+ // The self-takeover seen on the pilot. The server closes the stream, the+ // client starts reopening, and a beat sent just before comes back 404.+ // Reopening in answer used to cancel the open already in flight, which+ // the server had acted on, moving the connection to a new incarnation.+ // The client never read that answer, reattached claiming the old one, was+ // refused as a takeover, and stood down for good. The "other client" was+ // its own cancelled request.+ vi.useFakeTimers();+ const { state, fetchMock } = incarnationServer();+ const onEvicted = vi.fn();+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });++ await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);+ expect(state.generation).toBe(1);++ state.beat = 'slow-404-and-drop';+ // Long enough for the refused beat's wait to end while the open is held.+ await vi.advanceTimersByTimeAsync(12_000);+ state.release();+ await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);++ expect(onEvicted).not.toHaveBeenCalled();+ expect(state.opens).toBe(2);+ expect(beatGenerations(fetchMock).at(-1)).toBe(2);+ abort.abort();+ });++ it('carries out a repoint asked for mid-open once that open is answered', async () => {+ // The other way into the same hole: a room change reopens to include the+ // new room. Mid-open, it must wait for the answer rather than cancel it,+ // and must still happen, since the open in flight was built without it.+ vi.useFakeTimers();+ const { state, fetchMock } = incarnationServer();+ const onEvicted = vi.fn();+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);++ state.subscribeDrops = true;+ const repointing = stream.repoint('room-b');+ // The claim returns while the stream's reopen is held unanswered.+ await vi.advanceTimersByTimeAsync(5_000);+ state.release();+ await vi.advanceTimersByTimeAsync(5_000);+ await repointing;++ expect(onEvicted).not.toHaveBeenCalled();+ const opens = urlsFor(fetchMock, '/events');+ expect(opens.at(-1)).toContain('rooms=room-b');+ abort.abort();+ });+});++describe('a connection another client takes over', () => {+ it('stands down when the room it repoints to is refused, without reopening', async () => {+ // `repoint` claims the room *before* it reopens, so the open's own fence+ // is too late to protect the winner: by the time it refuses, a displaced+ // client has already rewritten the winner's rooms. The claim carries the+ // incarnation for that reason, and a refusal ends this client.+ const opens: string[] = [];+ const subscribes: (number | null)[] = [];+ const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {+ if (String(url).includes('/events')) {+ opens.push(String(url));+ return {+ ok: true,+ status: 200,+ body: openForever(),+ text: async (): Promise<string> => '',+ };+ }+ if (String(url).includes('connection/subscribe')) {+ const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };+ subscribes.push(sent.generation);+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'not your connection' } }),+ };+ }+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ const evicted: Eviction[] = [];+ const { stream, abort } = makeStream(fetchMock, {+ rooms: ['!old'],+ onEvicted: (e) => evicted.push(e),+ });+ await flush();+ const opensBefore = opens.length;++ await stream.repoint('!new');+ await flush();++ // The claim named the incarnation the frame gave us, not nothing.+ expect(subscribes).toEqual([0]);+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);+ // No reopen: coming back would be the takeover the refusal just denied.+ expect(opens.length).toBe(opensBefore);+ abort.abort();+ });++ it('does not claim a room before the frame that names its incarnation', async () => {+ // Claiming nothing is how a client too old to have an incarnation gets+ // through, so a claim sent in the window before the first frame would go+ // through the same door, and that window is exactly when this client may+ // already have been displaced without knowing it.+ let announce: () => void = () => {};+ const subscribes: (number | null)[] = [];+ const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {+ if (String(url).includes('/events'))+ return {+ ok: true,+ status: 200,+ // Attached, but silent: the server has not said which incarnation+ // this socket is yet.+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ announce = () => controller.enqueue(connected());+ },+ }),+ text: async (): Promise<string> => '',+ };+ if (String(url).includes('connection/subscribe')) {+ const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };+ subscribes.push(sent.generation);+ }+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ const { stream, abort } = makeStream(fetchMock, { rooms: ['!old'] });+ await flush();++ const repointed = stream.repoint('!new');+ await flush();++ expect(subscribes).toEqual([]);++ announce();+ await repointed;++ expect(subscribes).toEqual([0]);+ abort.abort();+ });++ it('halts instead of reopening, because reopening would be a takeover back', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: frameThenClose('evicted', {+ code: 'taken_over',+ reason: 'another stream attached to this connection',+ room_id: null,+ }),+ text: async (): Promise<string> => '',+ }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(5 * 60_000);++ // Two clients reopening each other's connection is the loop this prevents:+ // whoever reopens wins, so a displaced client that reopens starts it again.+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(evicted).toEqual([+ {+ code: EVICTION_TAKEN_OVER,+ reason: 'another stream attached to this connection',+ roomId: null,+ },+ ]);+ abort.abort();+ });++ it('stands down when its heartbeat is refused, rather than taking the connection back', async () => {+ vi.useFakeTimers();+ // The case the SSE frame cannot cover: this client's socket dropped before+ // the eviction reached it, so the refused heartbeat is the only way it ever+ // hears that it lost.+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }+ : {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({+ detail: {+ code: 'taken_over',+ message: 'connection conn-1 was reopened since incarnation 0',+ },+ }),+ }+ );+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(5 * 60_000);++ // One beat and no second open: reopening is a takeover, so a loser that+ // reopens pulls the connection straight back off the client that won it.+ expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);+ abort.abort();+ });++ it('still reopens on a refusal that names no code, the way an old server sends it', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: openUntilAborted(init),+ text: async (): Promise<string> => '',+ }+ : {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: 'connection conn-1 has no stream attached' }),+ }+ );+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(5 * 60_000);++ // Only the takeover is terminal. Everything else a 409 can mean is still+ // something a reopen fixes, and a server too old to say which is one of them.+ expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);+ expect(evicted).toEqual([]);+ abort.abort();+ });++ it('reads an old server’s wording as a takeover when it sends no code', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: frameThenClose('evicted', {+ reason: 'another stream attached to this connection',+ }),+ text: async (): Promise<string> => '',+ }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(5 * 60_000);++ expect(evicted[0]?.code).toBe(EVICTION_TAKEN_OVER);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ abort.abort();+ });+});++describe('a stream the server closes cleanly', () => {+ /** Opens, closes at once, forever: the shape of a contested connection. */+ function closesAtOnce() {+ return vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ controller.close();+ },+ }),+ text: async (): Promise<string> => '',+ }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ }++ it('waits before reopening rather than reconnecting at full rate', async () => {+ vi.useFakeTimers();+ const fetchMock = closesAtOnce();+ const { abort } = makeStream(fetchMock, { rooms: [] });++ // A minute of 1s, 2s, 4s… is seven opens. Without pacing a clean close it+ // is an unbounded spin, limited only by how fast the server can answer.+ await vi.advanceTimersByTimeAsync(60_000);++ expect(urlsFor(fetchMock, '/events').length).toBeLessThanOrEqual(8);+ abort.abort();+ });++ it('keeps backing off rather than resetting on every handshake', async () => {+ vi.useFakeTimers();+ const fetchMock = closesAtOnce();+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(60_000);+ const early = urlsFor(fetchMock, '/events').length;+ await vi.advanceTimersByTimeAsync(60_000);+ const late = urlsFor(fetchMock, '/events').length - early;++ // An open is not evidence of a working stream. Resetting the curve on the+ // handshake held two contending clients at a reconnect a second forever.+ expect(late).toBeLessThan(early);+ abort.abort();+ });+});++it('does not acknowledge a delivery before the consumer has saved it', async () => {+ let finish: () => void = () => {};+ const saved = new Promise<void>((resolve) => {+ finish = resolve;+ });+ const onEvent = vi.fn(() => saved);+ const fetchMock = vi.fn(async (url: string) => {+ if (!url.includes('/events')) return Response.json({});+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(+ new TextEncoder().encode(+ 'id: 7\nevent: message\ndata: {"type":"message","room_id":"room","sequence":7}\n\n'+ )+ );+ },+ })+ );+ });+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room'], startCursor: 6, onEvent });+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());+ expect(stream.position).toBe(6);+ finish();+ await vi.waitFor(() => expect(stream.position).toBe(7));+ abort.abort();+});++it('replays from an explicitly saved zero cursor instead of starting at head', async () => {+ const fetchMock = vi+ .fn()+ .mockImplementation(async (url: string) =>+ url.includes('/events?')+ ? { ok: true, body: openForever() }+ : { ok: true, json: async () => ({ rooms: [] }) }+ );+ const { abort } = makeStream(fetchMock, { rooms: [], startCursor: 0 });+ try {+ await flush();+ expect(new URL(urlsFor(fetchMock, '/events?')[0]).searchParams.get('start_from')).toBe('0');+ } finally {+ abort.abort();+ }+});++function resetFrames(url: string): Response {+ if (!url.includes('/events')) return Response.json({});+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(+ new TextEncoder().encode(+ 'event: gap\ndata: {"from_sequence":0,"resumed_at":0,"reason":"buffer reset"}\n\n' ++ 'id: 1\nevent: message\ndata: {"type":"message","room_id":"room","sequence":1}\n\n'+ )+ );+ },+ })+ );+}++it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {+ let finish!: () => void;+ const saved = new Promise<void>((resolve) => {+ finish = resolve;+ });+ const onGap = vi.fn(() => saved);+ const onEvent = vi.fn();+ const { stream, abort } = makeStream(+ vi.fn(async (url: string) => resetFrames(url)),+ { rooms: ['room'], startCursor: 4, onGap, onEvent }+ );+ try {+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());+ expect(onGap).toHaveBeenCalledWith({+ fromSequence: 0,+ resumedAt: 0,+ cursorReset: true,+ reason: 'buffer reset',+ });+ expect(stream.position).toBe(4);+ expect(onEvent).not.toHaveBeenCalled();+ finish();+ await vi.waitFor(() => expect(stream.position).toBe(1));+ expect(onEvent).toHaveBeenCalledOnce();+ } finally {+ abort.abort();+ }+});++it('does not advance or deliver when persisting the reset checkpoint fails', async () => {+ const onGap = vi.fn(async () => {+ throw new Error('journal unavailable');+ });+ const onEvent = vi.fn();+ const { stream, abort } = makeStream(+ vi.fn(async (url: string) => resetFrames(url)),+ { rooms: ['room'], startCursor: 4, onGap, onEvent }+ );+ try {+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());+ expect(stream.position).toBe(4);+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+});++describe('the permission to start a session', () => {+ function reopenable() {+ return vi.fn(async (url: string, init: { signal: AbortSignal }) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: openUntilAborted(init),+ text: async (): Promise<string> => '',+ }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ }++ it('is redeclared on the wire when it changes under a live connection', async () => {+ // Turning automatic sessions off while the controller is connected has to+ // reach the server, or it goes on promising a session nothing will start.+ const fetchMock = reopenable();+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ await flush();+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(urlsFor(fetchMock, '/events')[0]).toContain('spawn_capable=true');++ stream.setSpawnCapable(false);+ await flush();++ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(2);+ expect(opens[1]).not.toContain('spawn_capable');+ // A reattach, not a takeover: the incarnation goes with it, so the server+ // recognises the same client rather than evicting it in favour of itself.+ expect(opens[1]).toContain('expected_generation=0');+ abort.abort();+ });++ /**+ * A server that fences reattaches the way the real one does: every accepted+ * open makes a new incarnation, and an open claiming an older one is refused.+ * `holdStateOn` withholds the frame naming the incarnation for one open, so a+ * test can act in the window where the client has not been told yet.+ */+ function fencingServer(holdStateOn: number) {+ let generation = 0;+ let opens = 0;+ const announce: Array<() => void> = [];+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ const claimed = new URL(String(url)).searchParams.get('expected_generation');+ if (claimed !== null && Number(claimed) !== generation)+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: EVICTION_TAKEN_OVER } }),+ };+ opens += 1;+ generation += 1;+ const held = opens === holdStateOn;+ const named = generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ let closed = false;+ const send = () => {+ if (!closed) controller.enqueue(connected(named));+ };+ if (held) announce.push(send);+ else send();+ init.signal.addEventListener(+ 'abort',+ () => {+ closed = true;+ controller.close();+ },+ { once: true }+ );+ },+ }),+ text: async (): Promise<string> => '',+ };+ });+ return { fetchMock, announce };+ }++ it('holds a second change until the server has answered the first', async () => {+ // Redeclaring reattaches, which claims the incarnation this client believes+ // it holds. A second change sent before the first open was answered claims+ // the one before it, which the server has moved past, and refuses as a+ // takeover. That would take the agent's only connection off the air over+ // two clicks in a row.+ const onEvicted = vi.fn();+ const { fetchMock, announce } = fencingServer(2);+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true, onEvicted });+ try {+ await flush();+ stream.setSpawnCapable(false);+ await flush();+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);++ // The reopen is out but unanswered, so this one has nothing safe to claim.+ stream.setSpawnCapable(true);+ await flush();+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);++ announce[0]();+ await flush();++ // Without the hold, this is where the connection has already gone: the+ // third open claims the incarnation the second one replaced.+ expect(onEvicted).not.toHaveBeenCalled();+ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(3);+ expect(opens[2]).toContain('spawn_capable=true');+ expect(opens[2]).toContain('expected_generation=2');+ } finally {+ abort.abort();+ }+ });++ it('carries nothing when a change made in that window is taken back', async () => {+ const { fetchMock, announce } = fencingServer(2);+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ try {+ await flush();+ stream.setSpawnCapable(false);+ await flush();+ stream.setSpawnCapable(true);+ stream.setSpawnCapable(false);+ await flush();++ announce[0]();+ await flush();++ // The held open already declares what the setting settled back to.+ expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ } finally {+ abort.abort();+ }+ });++ it('does not reopen the socket when it is set to what it already is', async () => {+ const fetchMock = reopenable();+ const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ await flush();++ stream.setSpawnCapable(true);+ await flush();++ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ abort.abort();+ });+});++it('routes command wakeups separately from room events and their cursor', async () => {+ const onCommands = vi.fn();+ const onEvent = vi.fn();+ const fetchMock = vi.fn(async (url: string) =>+ url.includes('/events')+ ? new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(connected());+ controller.enqueue(encodeFrame('session_commands', { session_ids: ['session'] }));+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ )+ : Response.json({})+ );+ const { abort } = makeStream(fetchMock, { rooms: [], scope: 'all', onCommands, onEvent });+ try {+ await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+});++function streaming(...frames: Uint8Array[]) {+ return vi.fn(async (url: string) =>+ url.includes('/events')+ ? new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(connected());+ for (const frame of frames) controller.enqueue(frame);+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ )+ : Response.json({})+ );+}++it('hands an approval outcome to its own callback, not the room-event path', async () => {+ const onApprovalOutcome = vi.fn();+ const onEvent = vi.fn();+ const outcome = {+ session_id: 'session',+ request_id: 'permission',+ state: 'answered',+ answer: '0',+ answered_by: '@person:test',+ answered_at: '2026-09-24T12:00:00Z',+ };+ const { abort } = makeStream(streaming(encodeFrame('approval_outcome', outcome)), {+ rooms: [],+ scope: 'all',+ onApprovalOutcome,+ onEvent,+ });+ try {+ await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+});++it('drops a frame it does not know that is not a room event', async () => {+ const onEvent = vi.fn();+ const { abort } = makeStream(+ streaming(+ encodeFrame('something_new', { detail: 'from a later server' }),+ encodeFrame('approval_outcome', { session_id: 'session' }),+ encodeFrame('message', { type: 'message', room_id: 'room', payload: {} })+ ),+ { rooms: [], scope: 'all', onEvent }+ );+ try {+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));+ expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });+ } finally {+ abort.abort();+ }+});++it('hands a relayed session command to its own callback', async () => {+ const onSessionCommand = vi.fn();+ const onEvent = vi.fn();+ const command = {+ contractVersion: 1,+ commandId: 'command-1',+ sessionId: 'session',+ epoch: 'epoch',+ origin: { surface: 'console', actorId: 'owner', roomId: null, threadId: null, messageId: null },+ body: { type: 'turn.interrupt', turnId: 'turn' },+ };+ const { abort } = makeStream(+ streaming(+ encodeFrame('session_command', { commandId: 'no-session' }),+ encodeFrame('session_command', command)+ ),+ { rooms: [], scope: 'all', onSessionCommand, onEvent }+ );+ try {+ await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));+ expect(onSessionCommand).toHaveBeenCalledTimes(1);+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+});++it('hands a room another connection took over to its own callback', async () => {+ const onRoomReleased = vi.fn();+ const onEvent = vi.fn();+ const { abort } = makeStream(+ streaming(+ encodeFrame('room_released', { session_id: 'no-room' }),+ encodeFrame('room_released', { room_id: 'room', session_id: 'session' }),+ encodeFrame('room_released', { room_id: 'other', session_id: null })+ ),+ { rooms: [], scope: 'all', onRoomReleased, onEvent }+ );+ try {+ await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));+ expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });+ expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+});++it('says when the server has confirmed an open', async () => {+ const onConnected = vi.fn();+ const { abort } = makeStream(streaming(), { rooms: [], scope: 'all', onConnected });+ try {+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));+ } finally {+ abort.abort();+ }+});++it('says when an open stream ended and it is trying again', async () => {+ const onConnected = vi.fn();+ const onDisconnected = vi.fn();+ let opens = 0;+ const fetchMock = vi.fn(async (url: string) => {+ if (!url.includes('/events')) return Response.json({});+ opens += 1;+ if (opens === 1)+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(connected());+ controller.close();+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ );+ return new Response(openForever(), { headers: { 'content-type': 'text/event-stream' } });+ });+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ scope: 'all',+ onConnected,+ onDisconnected,+ });+ try {+ await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));+ expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });+ expect(onConnected).toHaveBeenCalledTimes(1);+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });+ expect(onDisconnected).toHaveBeenCalledTimes(1);+ } finally {+ abort.abort();+ }+});++it('states placements on the attached incarnation, and raises on a refusal', async () => {+ const bodies: unknown[] = [];+ let refuse = false;+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {+ if (url.includes('/events'))+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(connected(4));+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ );+ if (url.includes('connection/placements')) {+ bodies.push(JSON.parse(init!.body!));+ return refuse ? new Response('not a member', { status: 403 }) : Response.json({ ok: true });+ }+ return Response.json({});+ });+ const { stream, abort } = makeStream(fetchMock, { rooms: [], scope: 'all' });+ try {+ await stream.replacePlacements({ session: 'room' });+ expect(bodies).toEqual([+ { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },+ ]);+ refuse = true;+ await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');+ } finally {+ abort.abort();+ }+});++describe('a hosted worker', () => {+ const worker = { capability: 'cap-1', bootId: 'boot-1', instanceId: 'i-1', stateVersion: 1 };++ function refused(status: number, code: string) {+ return {+ ok: false,+ status,+ body: null,+ text: async (): Promise<string> => JSON.stringify({ detail: { code, message: code } }),+ };+ }++ it('states its capability and host on the open', async () => {+ const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({+ ok: true,+ status: 200,+ body: openForever(),+ text: async (): Promise<string> => '',+ }));+ const { abort } = makeStream(fetchMock, { rooms: [], worker });+ await flush();+ const open = fetchMock.mock.calls.find((c) => String(c[0]).includes('/events'));+ expect(open?.[1]?.headers).toMatchObject({+ 'X-Switch-Worker-Capability': 'cap-1',+ 'X-Switch-Host-Boot-Id': 'boot-1',+ 'X-Switch-Host-Instance-Id': 'i-1',+ 'X-Switch-Worker-State-Version': '1',+ });+ abort.abort();+ });++ it.each([+ [403, WORKER_CAPABILITY_OBSOLETE],+ [403, 'worker_capability_required'],+ [426, 'upgrade_required'],+ [403, 'hosted_worker_only'],+ ])('stops for good on a %i %s open', async (status, code) => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async () => refused(status, code));+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ worker,+ onEvicted: (e) => evicted.push(e),+ });+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(evicted.map((e) => e.code)).toEqual([code]);+ abort.abort();+ });++ it('backs off and retries while another worker is attached', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async () => refused(409, 'worker_already_attached'));+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ worker,+ onEvicted: (e) => evicted.push(e),+ });+ await vi.advanceTimersByTimeAsync(10_000);+ expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);+ expect(urlsFor(fetchMock, '/events').length).toBeLessThan(6);+ expect(evicted).toEqual([]);+ abort.abort();+ });++ it('halts on a launch_superseded eviction', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? {+ ok: true,+ status: 200,+ body: frameThenClose('evicted', { code: 'launch_superseded', reason: 'bumped' }),+ text: async (): Promise<string> => '',+ }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const evicted: Eviction[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ worker,+ onEvicted: (e) => evicted.push(e),+ });+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(evicted.map((e) => e.code)).toEqual([EVICTION_LAUNCH_SUPERSEDED]);+ abort.abort();+ });++ it('hands worker frames over in order and not as room events', async () => {+ const body = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected(4));+ controller.enqueue(encodeFrame('worker_attached', { launch_revision: 3 }));+ controller.enqueue(encodeFrame('operation', { id: 'op-1' }));+ },+ });+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const frames: [string, unknown][] = [];+ const events: unknown[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ worker,+ onEvent: (e) => void events.push(e),+ onWorkerFrame: (name, data) => void frames.push([name, data]),+ });+ await flush();+ expect(frames).toEqual([+ ['worker_attached', { launch_revision: 3 }],+ ['operation', { id: 'op-1' }],+ ]);+ expect(events).toEqual([]);+ abort.abort();+ });++ it('sends an up-call as its connection and incarnation, and raises a refusal with its code', async () => {+ const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => {+ const u = String(url);+ if (u.includes('/events'))+ return {+ ok: true,+ status: 200,+ body: openForever(),+ text: async (): Promise<string> => '',+ };+ if (u.endsWith('/claim')) return refused(409, 'already_claimed');+ return {+ ok: true,+ status: 200,+ text: async (): Promise<string> => '{"queued_operations":[]}',+ };+ });+ const { stream, abort } = makeStream(fetchMock, { rooms: [], worker });+ await expect(+ stream.workerCall('/agents/agent-1/connection/idle', { busy: false })+ ).resolves.toEqual({+ queued_operations: [],+ });+ const idle = fetchMock.mock.calls.find((c) => String(c[0]).endsWith('/connection/idle'));+ expect(String(idle?.[0])).toBe('https://switch.test/agents/agent-1/connection/idle');+ expect(JSON.parse(String(idle?.[1]?.body))).toEqual({+ busy: false,+ connection_id: 'conn-1',+ generation: 0,+ });+ const claim = stream.workerCall('/hosted/operations/op-1/claim', {});+ await expect(claim).rejects.toBeInstanceOf(WorkerCallError);+ await expect(claim).rejects.toMatchObject({ status: 409, code: 'already_claimed' });+ abort.abort();+ });+});++describe('test_gap_above_cursor_is_not_cursor_reset', () => {+ it('reports a restart gap resuming above the cursor as no cursor reset', async () => {+ const body = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected());+ controller.enqueue(+ new TextEncoder().encode(+ 'id: 7\nevent: message\ndata: {"type":"message","room_id":"r"}\n\n'+ )+ );+ controller.enqueue(+ encodeFrame('gap', {+ from_sequence: 7,+ resumed_at: 2 ** 32 - 1,+ reason: 'server restarted',+ rooms: [],+ })+ );+ },+ });+ const fetchMock = vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];+ const { stream, abort } = makeStream(fetchMock, { rooms: [], onGap: (g) => void gaps.push(g) });+ await flush();+ expect(gaps).toEqual([expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false })]);+ expect(stream.position).toBe(2 ** 32 - 1);+ abort.abort();+ });+});++it('does not give its connection up to itself over a request answered across its own reopen', async () => {+ // Seen on a real host: placements stated under incarnation 4, the stream+ // reopened (to 5) before the answer came, and the answer, "taken over:+ // reopened since 4, now at 5", named this very client as the new holder.+ // It stood down for good, with nobody else anywhere near the connection.+ const opened: ReadableStreamDefaultController<Uint8Array>[] = [];+ let answerPlacements: (response: Response) => void = () => {};+ const fetchMock = vi.fn(async (url: string) => {+ if (url.includes('/events')) {+ const generation = 4 + opened.length;+ return new Response(+ new ReadableStream({+ start(controller) {+ opened.push(controller);+ controller.enqueue(connected(generation));+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ );+ }+ if (url.includes('connection/placements'))+ return new Promise<Response>((resolve) => {+ answerPlacements = resolve;+ });+ return Response.json({});+ });+ const evicted: Eviction[] = [];+ const onConnected = vi.fn();+ const { stream, abort } = makeStream(fetchMock, {+ rooms: [],+ scope: 'all',+ onConnected,+ onEvicted: (eviction) => evicted.push(eviction),+ });+ try {+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));+ const stating = stream.replacePlacements({ session: 'room' });+ await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));++ opened[0]!.close();+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });+ answerPlacements(+ Response.json(+ {+ detail: {+ code: 'taken_over',+ message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',+ },+ },+ { status: 409 }+ )+ );++ await expect(stating).rejects.toThrow('409');+ expect(evicted).toEqual([]);+ } finally {+ abort.abort();+ }+});++it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ if (url.includes('/events'))+ return new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(connected(4));+ },+ }),+ { headers: { 'content-type': 'text/event-stream' } }+ );+ if (url.includes('connection/placements'))+ return Response.json(+ { detail: { code: 'taken_over', message: 'another client holds it' } },+ { status: 409 }+ );+ return Response.json({});+ });+ const evicted: Eviction[] = [];+ const { stream, abort } = makeStream(fetchMock, {+ rooms: [],+ scope: 'all',+ onEvicted: (eviction) => evicted.push(eviction),+ });+ try {+ await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');+ expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);+ } finally {+ abort.abort();+ }+});++describe('the heartbeat across its own reopen after a refusal', () => {+ /**+ * What took every in-Console watcher off the air when a laptop woke+ * (2026-10-01, four agents within ten seconds). The connection had lapsed+ * while the machine slept, so the first beat after waking was refused with a+ * 404. The heartbeat asked the stream to reopen and went straight back to+ * its gate, which it had found open a moment before the stream loop closed+ * it for the reopen. One beat went out under the lapsed incarnation right+ * behind the open that replaced it. The server, holding the new incarnation,+ * answered that beat `taken_over`, and the client believed it: it stood down+ * over a connection it had just made itself. In the server log the open and+ * the stale beat arrived 11 ms apart, every time.+ */+ function lapsingServer() {+ let generation = 0;+ let lapsed = false;+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {+ if (String(url).includes('/events')) {+ generation += 1;+ lapsed = false;+ const gen = generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(connected(gen));+ init.signal.addEventListener(+ 'abort',+ () => {+ try {+ controller.error(new DOMException('The operation was aborted.', 'AbortError'));+ } catch {+ // Already closed.+ }+ },+ { once: true }+ );+ },+ }),+ text: async (): Promise<string> => '',+ };+ }+ if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };+ const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;+ if (claimed !== null && claimed !== generation) {+ return {+ ok: false,+ status: 409,+ text: async (): Promise<string> =>+ JSON.stringify({+ detail: {+ code: 'taken_over',+ message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,+ },+ }),+ };+ }+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ /** The server sweeps the connection for silence; its socket stays up+ * client-side, the way a socket does across a sleep. */+ const lapse = (): void => {+ lapsed = true;+ };+ return { fetchMock, lapse };+ }++ it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {+ vi.useFakeTimers();+ const server = lapsingServer();+ const evicted: Eviction[] = [];+ const { abort } = makeStream(server.fetchMock, {+ rooms: [],+ onEvicted: (eviction) => evicted.push(eviction),+ });++ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);+ server.lapse();+ await vi.advanceTimersByTimeAsync(30_000);++ // The beat refused with a 404 is the last one under the lapsed+ // incarnation: what follows it on the wire is the reopen, not another beat+ // that the server could only take for someone else's.+ const sent = server.fetchMock.mock.calls.map((call) =>+ String(call[0]).includes('/events')+ ? 'open'+ : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`+ );+ const reopen = sent.indexOf('open', 1);+ expect(reopen).toBeGreaterThan(0);+ expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);+ expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);+ expect(evicted).toEqual([]);+ abort.abort();+ });+});++describe('choosing the transport', () => {+ /** A stream body that announces a server speaking `protocol` (none for a+ * server from before the declaration), then ends. */+ function announcingThenClose(protocol?: number): ReadableStream<Uint8Array> {+ return frameThenClose('connection_state', {+ connection_id: 'conn-1',+ generation: 0,+ ...(protocol === undefined ? {} : { protocol }),+ });+ }++ function serving(body: () => ReadableStream<Uint8Array>) {+ return vi.fn(async (url: string) =>+ String(url).includes('/events')+ ? { ok: true, status: 200, body: body(), text: async (): Promise<string> => '' }+ : { ok: true, status: 200, text: async (): Promise<string> => '' }+ );+ }++ // 6 is a server before the hosted worker, 7 one with it: neither has a socket.+ it.each([6, 7])(+ 'falls back to the event stream, then opens it straight away for a server speaking %i',+ async (protocol) => {+ vi.useFakeTimers();+ const fetchMock = serving(() => announcingThenClose(protocol));+ const { abort } = makeStream(fetchMock, { rooms: [] });++ // The stream ends at once each time: opens at 0, 1 and 3 seconds.+ await vi.advanceTimersByTimeAsync(4000);++ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);+ expect(SocketThatNeverOpens.opened).toHaveLength(1);+ abort.abort();+ }+ );++ it('declares a range a server from before the socket shares', async () => {+ const fetchMock = serving(() => openForever());+ const { abort } = makeStream(fetchMock, { rooms: [] });+ await flush();++ const params = new URL(urlsFor(fetchMock, '/events')[0]!).searchParams;+ expect(params.get('protocol')).toBe('8');+ expect(Number(params.get('protocol_accepts'))).toBeLessThanOrEqual(6);+ abort.abort();+ });++ it('keeps trying the socket first when neither answers, and says it could not connect', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async () => {+ throw new TypeError('fetch failed');+ });+ const disconnected: string[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: [],+ onDisconnected: ({ error }) => disconnected.push(error),+ });++ await vi.advanceTimersByTimeAsync(4000);++ // A network failure decides nothing: every open still starts with the socket.+ expect(SocketThatNeverOpens.opened).toHaveLength(3);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);+ expect(disconnected[0]).toContain('could not connect');+ abort.abort();+ });++ it('goes back to the socket once the server says it speaks a revision with one', async () => {+ vi.useFakeTimers();+ // The socket failed to open (a proxy, say) but the server is a new one.+ const fetchMock = serving(() => announcingThenClose(8));+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(2000);++ expect(SocketThatNeverOpens.opened).toHaveLength(2);+ abort.abort();+ });++ it('moves to the socket at once when the stream it went to straight away has been removed', async () => {+ vi.useFakeTimers();+ let opens = 0;+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ opens += 1;+ if (opens === 1)+ return {+ ok: true,+ status: 200,+ body: announcingThenClose(6),+ text: async (): Promise<string> => '',+ };+ return {+ ok: false,+ status: 410,+ body: null,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'transport_removed', message: 'gone' } }),+ };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ // The first stream ends at once; the next open, a second later, is the+ // stream again and is refused as removed.+ await vi.advanceTimersByTimeAsync(1500);+ expect(SocketThatNeverOpens.opened).toHaveLength(2);++ // Then the backoff, not a spin between the two.+ await vi.advanceTimersByTimeAsync(1000);+ expect(SocketThatNeverOpens.opened).toHaveLength(2);+ expect(urlsFor(fetchMock, '/events')).toHaveLength(3);+ abort.abort();+ });+});diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.tsindex 516e3c6e2..2fa9b01c1 100644--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts@@ -1,106 +1,208 @@-import { afterEach, describe, expect, it, vi } from 'vitest';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import {- BEAT_INTERVAL_MS,- BEAT_SETTLE_LIMIT_MS,EVICTION_CREDENTIALS_REJECTED,EVICTION_LAUNCH_SUPERSEDED,EVICTION_TAKEN_OVER,SwitchEventStream,WORKER_CAPABILITY_OBSOLETE,- WorkerCallError,type Eviction,type SwitchEventStreamDeps,} from './event-stream';/**- * Two ways a client can retry something that can never succeed.+ * The client side of the agent connection, held to what the server does.*- * Both were measured on a live deployment: a stream re-declaring a room that- * had been deleted, refused on every open, reopening at once; and a heartbeat- * treating "your connection is gone" as a normal answer and beating on at full- * cadence. Neither could self-heal, and between them they produced most of the- * error volume on that deployment.+ * Much of what is pinned here was measured on a live deployment: a stream+ * re-declaring a room that had been deleted, refused on every open, reopening+ * at once; a client cancelling its own open and then standing down over the+ * incarnation that open made. Neither could self-heal.+ *+ * The server end is a fake WebSocket the test drives by hand: it accepts,+ * sends frames and pings, refuses, and closes, and records what the client+ * sent back. Its events are dispatched on the microtask queue, the way a real+ * socket's arrive as separate tasks, so nothing a test sends is seen before+ * the client has attached its listeners.*/const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };-function silentLog() {- return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };+type Listener = (event: { data?: string; code?: number; reason?: string }) => void;++/** What the server does with each socket the client opens, and its index. */+type Script = (socket: FakeSocket, index: number) => void;++/** Every socket opened in the current test, and the script that answers them. */+const server: { sockets: FakeSocket[]; script: Script } = {+ sockets: [],+ script: () => {},+};++/** Open and announce incarnation 0: a healthy, attached connection. */+const attach: Script = (socket) => {+ socket.open();+ socket.announce();+};++class FakeSocket {+ static readonly CONNECTING = 0;+ static readonly OPEN = 1;+ static readonly CLOSING = 2;+ static readonly CLOSED = 3;+ readonly CONNECTING = 0;+ readonly OPEN = 1;+ readonly CLOSING = 2;+ readonly CLOSED = 3;++ readyState = 0;+ readonly url: string;+ readonly headers: Record<string, string>;+ /** Every message the client sent, parsed. */+ readonly sent: unknown[] = [];+ /** The code the client closed with, if it was the one to close. */+ closedByClient: number | null = null;+ private ending = false;+ private readonly listeners = new Map<string, Set<Listener>>();++ constructor(url: string, init?: { headers?: Record<string, string> }) {+ this.url = url;+ this.headers = { ...init?.headers };+ server.sockets.push(this);+ server.script(this, server.sockets.length - 1);+ }++ get params(): URLSearchParams {+ return new URL(this.url).searchParams;+ }++ addEventListener(type: string, listener: Listener): void {+ const set = this.listeners.get(type) ?? new Set<Listener>();+ set.add(listener);+ this.listeners.set(type, set);+ }++ removeEventListener(type: string, listener: Listener): void {+ this.listeners.get(type)?.delete(listener);+ }++ send(data: string): void {+ this.sent.push(JSON.parse(data));+ }++ close(code = 1000, reason = ''): void {+ if (this.ending) return;+ this.closedByClient = code;+ this.readyState = this.CLOSING;+ this.end(code, reason);+ }++ // The server's side.++ open(): void {+ this.dispatch(+ 'open',+ {},+ () => this.readyState === this.CONNECTING,+ () => {+ this.readyState = this.OPEN;+ }+ );+ }++ frame(event: string, data: unknown, id?: number): void {+ const message = JSON.stringify({ event, data, ...(id === undefined ? {} : { id }) });+ this.dispatch('message', { data: message }, () => this.readyState === this.OPEN);+ }++ /** The frame a server opens every connection with, naming its incarnation.+ * No `rooms`, so it says nothing about the declared set a test may be+ * asserting on. */+ announce(generation = 0): void {+ this.frame('connection_state', { connection_id: 'conn-1', generation });+ }++ ping(): void {+ this.frame('ping', {});+ }++ /** Refuse the open the way the server does: say why, then close with the status. */+ refuse(status: number, detail: unknown): void {+ this.open();+ this.frame('refused', { status, detail });+ this.drop(4000 + status);+ }++ /** The server closes the socket. */+ drop(code = 1000, reason = ''): void {+ this.end(code, reason);+ }++ /** The connection fails underneath both ends, before or after it opened. */+ fail(): void {+ this.dispatch('error', {}, () => this.readyState !== this.CLOSED);+ this.end(1006, '');+ }++ private end(code: number, reason: string): void {+ if (this.ending) return;+ this.ending = true;+ this.dispatch(+ 'close',+ { code, reason },+ () => true,+ () => {+ this.readyState = this.CLOSED;+ }+ );+ }++ private dispatch(+ type: string,+ event: { data?: string; code?: number; reason?: string },+ live: () => boolean,+ apply?: () => void+ ): void {+ queueMicrotask(() => {+ if (!live()) return;+ apply?.();+ for (const listener of [...(this.listeners.get(type) ?? [])]) listener(event);+ });+ }}-function encodeFrame(name: string, data: unknown): Uint8Array {- return new TextEncoder().encode(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);+function serve(script: Script): void {+ server.script = script;}-/** The frame a server opens every stream with, naming the incarnation this- * client is attached to. No `rooms`, so it says nothing about the declared set- * a test may be asserting on. */-function connected(generation = 0): Uint8Array {- return encodeFrame('connection_state', { connection_id: 'conn-1', generation });+function urls(): string[] {+ return server.sockets.map((socket) => socket.url);}-/** A stream body carrying one frame and then closing, the way the server ends- * a displaced stream. */-function frameThenClose(name: string, data: unknown): ReadableStream<Uint8Array> {- return new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(encodeFrame(name, data));- controller.close();- },- });+function silentLog() {+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };}-/**- * A connected stream body that stays open, so the loop neither reconnects nor- * spins.- *- * It announces the connection first because the heartbeat waits for that: a- * client that has not been told which incarnation it is cannot fence its own- * tick, and a body that never announces is a connection that never beats.- */-function openForever(): ReadableStream<Uint8Array> {- return new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected());- },- });+/** A request on the side that the server answered. */+function answer(status = 200, body = '') {+ return { ok: status >= 200 && status < 300, status, text: async (): Promise<string> => body };}-/**- * A connected stream body that ends when the client drops the socket.- *- * `openForever` cannot: its reader is parked on a read that never returns, so- * a reopen is invisible to the test rather than absent. A body that honours- * the request's signal is what a real socket does, and what a test asserting- * on reopens needs.- */-function openUntilAborted(init: { signal: AbortSignal }): ReadableStream<Uint8Array> {- return new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected());- init.signal.addEventListener('abort', () => controller.close(), { once: true });- },- });+function takenOver(message: string) {+ return answer(409, JSON.stringify({ detail: { code: 'taken_over', message } }));}-/** The incarnation each beat carried, in order. */-function beatGenerations(fetchMock: { mock: { calls: unknown[][] } }): (number | null)[] {- return fetchMock.mock.calls- .filter((call) => String(call[0]).includes('connection/beat'))- .map(- (call) =>- (JSON.parse((call[1] as { body: string }).body) as { generation: number | null }).generation- );+function okFetch() {+ return vi.fn(async (_url: string, _init?: { body?: string }) => answer());}-function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {+function postsTo(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));}function makeStream(- fetchMock: ReturnType<typeof vi.fn>,- deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] },+ fetchMock: ReturnType<typeof vi.fn> = okFetch()) {+ vi.stubGlobal('WebSocket', FakeSocket);vi.stubGlobal('fetch', fetchMock);const abort = new AbortController();const log = silentLog();@@ -118,51 +220,135 @@ function makeStream(...deps,});stream.start();- return { stream, abort, log };+ return { stream, abort, log, fetchMock };}async function flush(times = 8): Promise<void> {for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));}+beforeEach(() => {+ server.sockets = [];+ server.script = attach;+});+afterEach(() => {vi.unstubAllGlobals();vi.useRealTimers();});+describe('the socket', () => {+ it('carries the token as a bearer header', async () => {+ const { abort } = makeStream({ rooms: [] });+ await flush();++ expect(server.sockets).toHaveLength(1);+ expect(server.sockets[0]?.headers).toEqual({ Authorization: 'Bearer tok' });+ abort.abort();+ });++ it('opens over wss for an https endpoint and ws for an http one', async () => {+ const secure = makeStream({ rooms: [] });+ await flush();+ secure.abort.abort();+ const plain = makeStream({+ rooms: [],+ creds: { ...creds, apiEndpoint: 'http://localhost:8000' },+ });+ await flush();+ plain.abort.abort();++ const [first, second] = urls();+ expect(first).toMatch(/^wss:\/\/switch\.test\/agents\/agent-1\/connection\/ws\?/);+ expect(second).toMatch(/^ws:\/\/localhost:8000\/agents\/agent-1\/connection\/ws\?/);+ expect(new URL(first!).searchParams.get('connection_id')).toBe('conn-1');+ });+});++describe('the heartbeat', () => {+ it('answers a ping at once with a pong carrying the cursor', async () => {+ serve((socket) => {+ attach(socket, 0);+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 3 }, 3);+ });+ const { abort } = makeStream({ rooms: ['room'] });+ await flush();++ server.sockets[0]!.ping();+ await flush();++ expect(server.sockets[0]!.sent).toEqual([{ type: 'pong', cursor: 3 }]);+ abort.abort();+ });++ it('answers a ping while a slow handler is still running, without waiting for it', async () => {+ // Handling an event can take as long as the agent likes. A pong queued+ // behind it would have the server sweep a connection that is busy, not gone.+ let finish: () => void = () => {};+ const onEvent = vi.fn(+ () =>+ new Promise<void>((resolve) => {+ finish = resolve;+ })+ );+ serve((socket) => {+ attach(socket, 0);+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 7 }, 7);+ });+ const { abort } = makeStream({ rooms: ['room'], startCursor: 6, onEvent });+ await flush();+ expect(onEvent).toHaveBeenCalledOnce();++ server.sockets[0]!.ping();+ await flush();+ // The cursor it names is the last event fully handled, not the one in hand.+ expect(server.sockets[0]!.sent).toEqual([{ type: 'pong', cursor: 6 }]);++ finish();+ await flush();+ server.sockets[0]!.ping();+ await flush();+ expect(server.sockets[0]!.sent).toEqual([+ { type: 'pong', cursor: 6 },+ { type: 'pong', cursor: 7 },+ ]);+ abort.abort();+ });++ it('makes no requests on the side to stay alive', async () => {+ vi.useFakeTimers();+ const { abort, fetchMock } = makeStream({ rooms: [] });++ await vi.advanceTimersByTimeAsync(60_000);++ expect(fetchMock).not.toHaveBeenCalled();+ expect(server.sockets).toHaveLength(1);+ abort.abort();+ });+});+describe('a room the server refuses', () => {- /** The refusal switch-core returns when a declared room no longer exists. */- function roomGone(roomId: string) {- return {- ok: false,- status: 403,- body: null,- text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),- };- }+ /** The refusal switch-core sends when a declared room no longer exists. */+ const roomGone = (socket: FakeSocket, roomId: string): void =>+ socket.refuse(403, `Room not found: ${roomId}`);it('is dropped and not declared again', async () => {- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events')) {- return { ok: true, status: 200, text: async (): Promise<string> => '' };- }- return String(url).includes('rooms=')- ? roomGone('room-dead')- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };- });+ serve((socket) =>+ socket.params.has('rooms') ? roomGone(socket, 'room-dead') : attach(socket, 0)+ );const rejected: { roomId: string; status: number }[] = [];const reported: string[][] = [];- const { abort, log } = makeStream(fetchMock, {+ const { abort, log } = makeStream({rooms: ['room-dead'],onRooms: (rooms) => reported.push(rooms),onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),});await flush();- const opens = urlsFor(fetchMock, '/events');+ const opens = server.sockets.map((socket) => socket.params);expect(opens).toHaveLength(2);- expect(opens[0]).toContain('rooms=room-dead');- expect(opens[1]).not.toContain('rooms=');+ expect(opens[0]?.get('rooms')).toBe('room-dead');+ expect(opens[1]?.has('rooms')).toBe(false);expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);expect(reported).toEqual([[]]);expect(log.error).toHaveBeenCalled();@@ -170,689 +356,159 @@ describe('a room the server refuses', () => {});it('keeps the rooms the refusal does not name', async () => {- const fetchMock = vi.fn(async (url: string) => {- const u = String(url);- if (!u.includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- return u.includes('room-dead')- ? roomGone('room-dead')- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };- });- const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });+ serve((socket) =>+ socket.params.get('rooms')?.includes('room-dead')+ ? roomGone(socket, 'room-dead')+ : attach(socket, 0)+ );+ const { abort } = makeStream({ rooms: ['room-dead', 'room-live'] });await flush();- const opens = urlsFor(fetchMock, '/events');- expect(opens).toHaveLength(2);- expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');+ expect(server.sockets).toHaveLength(2);+ expect(server.sockets[1]?.params.get('rooms')).toBe('room-live');abort.abort();});- it('stays a transport error when the body names no declared room', async () => {+ it('stays a transport error when the refusal names no declared room', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events')) {- return { ok: true, status: 200, text: async (): Promise<string> => '' };- }- return {- ok: false,- status: 404,- body: null,- text: async (): Promise<string> => JSON.stringify({ detail: 'No such connection' }),- };- });+ serve((socket) => socket.refuse(404, 'No such connection'));const rejected: string[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort } = makeStream({rooms: ['room-live'],onRoomRejected: ({ roomId }) => rejected.push(roomId),});await vi.advanceTimersByTimeAsync(0);- // One open, then the backoff — not an immediate retry, and nothing dropped.- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ // One open, then the backoff: not an immediate retry, and nothing dropped.+ expect(server.sockets).toHaveLength(1);expect(rejected).toEqual([]);abort.abort();});});describe('credentials the server rejects', () => {- function refusal(status: number, detail: string) {- return {- ok: false,- status,- body: null,- text: async (): Promise<string> => JSON.stringify({ detail }),- };- }-/** Open once, be refused, and let a long while pass. */- async function afterRefusal(status: number, detail: string) {+ async function afterRefusal(refuse: Script) {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? refusal(status, detail)- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );+ serve(refuse);const evicted: Eviction[] = [];- const { abort, log } = makeStream(fetchMock, {+ const { abort, log, fetchMock } = makeStream({rooms: ['room-live'],onEvicted: (eviction) => evicted.push(eviction),});- await vi.advanceTimersByTimeAsync(0);- const settled = fetchMock.mock.calls.length;await vi.advanceTimersByTimeAsync(5 * 60_000);abort.abort();- return { fetchMock, evicted, log, settled };+ return { evicted, log, fetchMock };}- it('ends the stream and the heartbeat on a 401, and says so once', async () => {- const { fetchMock, evicted, log, settled } = await afterRefusal(401, 'Invalid agent token');+ it('ends the stream when the socket is closed for its token, and says so once', async () => {+ // An auth failure closes with 4401 and no `refused` frame before it.+ const { evicted, log, fetchMock } = await afterRefusal((socket) => {+ socket.open();+ socket.drop(4401, 'Invalid agent token');+ });- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);- // Nothing beyond the beat already in flight when the refusal landed.- expect(fetchMock.mock.calls).toHaveLength(settled);+ expect(server.sockets).toHaveLength(1);+ expect(fetchMock).not.toHaveBeenCalled();expect(evicted).toHaveLength(1);expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);expect(evicted[0]?.reason).toContain('credentials');expect(evicted[0]?.reason).toContain('401');- expect(log.error).toHaveBeenCalled();+ expect(evicted[0]?.reason).toContain('Invalid agent token');+ expect(log.error).toHaveBeenCalledTimes(1);});it('ends the stream on a 403 that names no declared room', async () => {- const { fetchMock, evicted, settled } = await afterRefusal(- 403,- 'Agent is not a member of this room'+ const { evicted } = await afterRefusal((socket) =>+ socket.refuse(403, 'Agent is not a member of this room'));- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);- expect(fetchMock.mock.calls).toHaveLength(settled);+ expect(server.sockets).toHaveLength(1);expect(evicted).toHaveLength(1);expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);expect(evicted[0]?.reason).toContain('credentials');expect(evicted[0]?.reason).toContain('403');});- it('ends the stream and the heartbeat on a 401 heartbeat, and says so once', async () => {+ it.each([503, 429])('keeps reconnecting after a refusal with status %i', async (status) => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }- : refusal(401, 'Invalid agent token')+ serve((socket, index) =>+ index === 0 ? socket.refuse(status, 'try later') : attach(socket, 0));const evicted: Eviction[] = [];- const { abort, log } = makeStream(fetchMock, {- rooms: ['room-live'],- onEvicted: (eviction) => evicted.push(eviction),- });- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS + 1);- const settled = fetchMock.mock.calls.length;- await vi.advanceTimersByTimeAsync(5 * 60_000);- abort.abort();-- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);- expect(fetchMock.mock.calls).toHaveLength(settled);- expect(evicted).toHaveLength(1);- expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);- expect(evicted[0]?.reason).toContain('credentials');- expect(evicted[0]?.reason).toContain('401');- expect(log.error).toHaveBeenCalledTimes(1);- });-- it('tells a session start to the room itself only where the server says it leaves that to it', async () => {- for (const [said, expected] of [- [{ announce_session_starts: true }, true],- [{}, false],- ] as const) {- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: frameThenClose('connection_state', {- connection_id: 'conn-1',- generation: 0,- ...said,- }),- text: async (): Promise<string> => '',- }- : new Promise<never>(() => {})- );- let connected = false;- const { stream, abort } = makeStream(fetchMock, {- rooms: ['room-live'],- onConnected: () => {- connected = true;- },- });- expect(stream.announcesSessionStarts).toBe(false);- for (let i = 0; i < 50 && !connected; i += 1) await flush();- expect(stream.announcesSessionStarts).toBe(expected);- abort.abort();- vi.unstubAllGlobals();- }- });-- it('tells the owner once when the stream and the heartbeat are refused together', async () => {- vi.useFakeTimers();- let opens = 0;- const fetchMock = vi.fn(async (url: string) => {- if (String(url).includes('/events')) {- // Connected once — the heartbeat has nothing to beat for until then —- // and refused on every reopen after that.- opens += 1;- if (opens === 1) {- return {- ok: true,- status: 200,- body: frameThenClose('connection_state', { connection_id: 'conn-1', generation: 0 }),- text: async (): Promise<string> => '',- };- }- await new Promise((r) => setTimeout(r, BEAT_INTERVAL_MS));- return refusal(401, 'Invalid agent token');- }- // Slow enough to still be in flight when the reopen is refused: both- // doors closing at once is the case this test is about.- await new Promise((r) => setTimeout(r, 2 * BEAT_INTERVAL_MS));- return refusal(401, 'Invalid agent token');- });- const evicted: Eviction[] = [];- const { abort, log } = makeStream(fetchMock, {- rooms: ['room-live'],- onEvicted: (eviction) => evicted.push(eviction),- });- await vi.advanceTimersByTimeAsync(5 * 60_000);- abort.abort();-- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);- expect(evicted).toHaveLength(1);- expect(log.error).toHaveBeenCalledTimes(1);- });-- it.each([503, 429])('keeps reconnecting after HTTP %i', async (status) => {- vi.useFakeTimers();- let opens = 0;- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- opens += 1;- return opens === 1- ? { ok: false, status, body: null, text: async (): Promise<string> => 'try later' }- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };- });- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort } = makeStream({rooms: [],onEvicted: (eviction) => evicted.push(eviction),});await vi.advanceTimersByTimeAsync(2000);- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(server.sockets).toHaveLength(2);expect(evicted).toEqual([]);abort.abort();});it('keeps reconnecting after a network error', async () => {vi.useFakeTimers();- let opens = 0;- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- opens += 1;- if (opens === 1) throw new TypeError('fetch failed');- return { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };- });+ serve((socket, index) => (index === 0 ? socket.fail() : attach(socket, 0)));const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const disconnected: string[] = [];+ const { abort } = makeStream({rooms: [],onEvicted: (eviction) => evicted.push(eviction),+ onDisconnected: ({ error }) => disconnected.push(error),});await vi.advanceTimersByTimeAsync(2000);- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(server.sockets).toHaveLength(2);expect(evicted).toEqual([]);+ expect(disconnected).toHaveLength(1);+ expect(disconnected[0]).toContain('could not connect');abort.abort();});});-describe('the heartbeat', () => {- /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */- async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {+describe('a server restart', () => {+ it('comes back within a second of the server, not on the doubling backoff', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) => {- if (String(url).includes('/events'))- return {- ok: true,- status: 200,- body: openForever(),- text: async (): Promise<string> => '',- };- return { ok: false, status, text: async (): Promise<string> => '' };- });- const { abort } = makeStream(fetchMock, { rooms: [] });- await vi.advanceTimersByTimeAsync(windowMs);- const beats = urlsFor(fetchMock, 'connection/beat').length;- abort.abort();- return beats;- }-- it('backs off when the connection is rejected, instead of beating at full rate', async () => {- const windowMs = 20 * BEAT_INTERVAL_MS;- // At the base cadence this window holds ~20 beats. Doubling from the base- // gives 4s, 8s, 16s… — four requests in the same window.- expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);- expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);- });-- it('keeps backing off the longer the rejection lasts', async () => {- vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) => {- if (String(url).includes('/events'))- return {- ok: true,- status: 200,- body: openForever(),- text: async (): Promise<string> => '',- };- return { ok: false, status: 404, text: async (): Promise<string> => '' };- });- const { abort } = makeStream(fetchMock, { rooms: [] });-- await vi.advanceTimersByTimeAsync(30_000);- const early = urlsFor(fetchMock, 'connection/beat').length;- await vi.advanceTimersByTimeAsync(30_000);- const late = urlsFor(fetchMock, 'connection/beat').length - early;-- expect(late).toBeLessThan(early);- abort.abort();- });-- it('sends nothing until the server has said which incarnation it is', async () => {- vi.useFakeTimers();- const server = { announce: (): void => {} };- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- server.announce = () => controller.enqueue(connected(7));- },- }),- text: async (): Promise<string> => '',- };- });- const { abort } = makeStream(fetchMock, { rooms: [] });-- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);- // An unfenced tick is what a displaced client sends, and the server refuses- // it. Beating before the first frame would make every healthy connection- // open with one.- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(0);-- server.announce();- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);-- const beats = fetchMock.mock.calls.filter((c) => String(c[0]).includes('connection/beat'));- expect(beats.length).toBeGreaterThan(0);- // And it carries the incarnation that frame named, so the server can fence it.- const [, sent] = beats[0] as unknown as [string, { body: string }];- expect(JSON.parse(sent.body).generation).toBe(7);- abort.abort();- });-- it('waits for the new incarnation after its own reconnect, instead of standing down', async () => {- vi.useFakeTimers();- // A server that fences: every attach is a new incarnation, and a tick- // naming an older one is refused the way a displaced client's is.- const server = { generation: 0, announce: (): void => {}, drop: (): void => {} };- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {- if (String(url).includes('/events')) {- server.generation += 1;- const attached = server.generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- const announce = (): void => controller.enqueue(connected(attached));- // The first attach is announced at once; the second is held, so- // the beat falls due inside the window this test is about.- if (attached === 1) announce();- else server.announce = announce;- server.drop = () => controller.close();- },- }),- text: async (): Promise<string> => '',- };+ // Open, then the server restarts: it closes with 1012, refuses three+ // attempts while it is down, then takes the fifth.+ serve((socket, index) => {+ if (index === 0) {+ attach(socket, 0);+ queueMicrotask(() => socket.drop(1012, 'service restart'));+ } else if (index < 4) {+ socket.fail();+ } else {+ attach(socket, 0);}- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;- if (sent === server.generation)- return { ok: true, status: 200, text: async (): Promise<string> => '' };- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),- };});- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });-- await vi.advanceTimersByTimeAsync(2 * BEAT_INTERVAL_MS);- const first = beatGenerations(fetchMock);- expect(first.length).toBeGreaterThan(0);- expect(first.every((generation) => generation === 1)).toBe(true);-- server.drop();- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);- // Nothing went out while the reopen was in flight: the incarnation we hold- // is the one before it, and a tick carrying that is a stand-down.- expect(beatGenerations(fetchMock)).toHaveLength(first.length);- expect(evicted).toEqual([]);-- server.announce();- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);- const resumed = beatGenerations(fetchMock).slice(first.length);- expect(resumed.length).toBeGreaterThan(0);- expect(resumed.every((generation) => generation === 2)).toBe(true);- expect(evicted).toEqual([]);- abort.abort();- });-- it('does not stand down over a beat that was already in flight when it reopened', async () => {- vi.useFakeTimers();- // The gate stops a beat *starting* inside the reattach window. It cannot- // recall one that left before the gate shut, and that beat is answered- // against the incarnation the reopen replaced — a truthful `taken_over`- // about a takeover this client performed on itself.- const server = {- generation: 0,- announce: (): void => {},- drop: (): void => {},- answerBeat: (): void => {},- beatsHeld: 0,- };- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {- if (String(url).includes('/events')) {- const claimed = new URL(String(url)).searchParams.get('expected_generation');- if (claimed !== null && Number(claimed) !== server.generation) {- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),- };- }- server.generation += 1;- const attached = server.generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected(attached));- server.drop = () => controller.close();- },- }),- text: async (): Promise<string> => '',- };- }- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;- // Held rather than answered: the point of the test is an answer that- // arrives after the connection has moved on beneath it.- server.beatsHeld += 1;- await new Promise<void>((resolve) => {- server.answerBeat = resolve;- });- if (sent === server.generation)- return { ok: true, status: 200, text: async (): Promise<string> => '' };- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),- };- });- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });+ const { abort } = makeStream({ rooms: [] });- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);- expect(server.beatsHeld).toBe(1);- expect(beatGenerations(fetchMock)).toEqual([1]);+ await vi.advanceTimersByTimeAsync(4000);- // Reopen with that beat still outstanding. The bound expires, the open goes- // ahead, and the server is at 2 by the time the old beat is answered.- server.drop();- await vi.advanceTimersByTimeAsync(BEAT_SETTLE_LIMIT_MS + 10 * BEAT_INTERVAL_MS);- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(1);-- server.answerBeat();- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);-- // Undecidable, so inert. Standing down here stops a client that nothing- // has taken anything from.- expect(evicted).toEqual([]);- const resumed = beatGenerations(fetchMock).slice(1);- expect(resumed.length).toBeGreaterThan(0);- expect(resumed.every((generation) => generation === 2)).toBe(true);+ // On the doubling backoff the fifth attempt would come after 1+2+4+8 s.+ expect(server.sockets).toHaveLength(5);abort.abort();});- it('stands down when its reattach is refused, rather than trying again', async () => {+ it('keeps the doubling backoff for an ending that is not a restart', async () => {vi.useFakeTimers();- // The other half: a client that missed its eviction and comes back. The- // server refuses the reattach without disturbing the holder, and this is- // the only thing left that can tell the loser it lost.- const server = { generation: 0, drop: (): void => {} };- const fetchMock = vi.fn(async (url: string) => {- if (!String(url).includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- const claimed = new URL(String(url)).searchParams.get('expected_generation');- if (claimed !== null) {- // Someone else attached while we were away, so our claim is stale.- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),- };+ serve((socket, index) => {+ if (index === 0) {+ attach(socket, 0);+ queueMicrotask(() => socket.drop(1006));+ } else {+ socket.fail();}- server.generation += 1;- const attached = server.generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected(attached));- server.drop = () => controller.close();- },- }),- text: async (): Promise<string> => '',- };- });- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });-- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);- server.drop();- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);-- expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);- // And it stopped asking: retrying is how the pair trade the connection.- const attempts = urlsFor(fetchMock, 'expected_generation=1').length;- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(attempts);- abort.abort();- });-- it('returns to the base cadence once a beat lands', async () => {- vi.useFakeTimers();- let reject = true;- // A socket that ends when the client drops it: each refusal reopens, and- // the heartbeat is held from the reopen until the new stream is named.- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {- if (String(url).includes('/events'))- return {- ok: true,- status: 200,- body: openUntilAborted(init),- text: async (): Promise<string> => '',- };- if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };- return { ok: true, status: 200, text: async (): Promise<string> => '' };});- const { abort } = makeStream(fetchMock, { rooms: [] });+ const { abort } = makeStream({ rooms: [] });- await vi.advanceTimersByTimeAsync(30_000);- reject = false;- await vi.advanceTimersByTimeAsync(30_000);- const recovered = urlsFor(fetchMock, 'connection/beat').length;- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);+ await vi.advanceTimersByTimeAsync(4000);- expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);- abort.abort();- });-});--describe('the heartbeat after a reconnect', () => {- /**- * A server that reattaches on every open, can drop the current stream, and- * records when each stream was opened and each beat arrived. The server's- * own rule is what these tests hold the client to: a new connection that- * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.- */- const HEARTBEAT_TTL_MS = 6000;-- /** Either end may close a stream first; the second close is not an error here. */- function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {- try {- controller.close();- } catch {- // Already closed by the other side.- }- }-- function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {- const opens: number[] = [];- const beats: number[] = [];- let generation = 0;- let current: ReadableStreamDefaultController<Uint8Array> | null = null;- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {- if (String(url).includes('/events')) {- opens.push(Date.now());- generation += 1;- const gen = generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- current = controller;- controller.enqueue(connected(gen));- init.signal.addEventListener('abort', () => closeQuietly(controller), {- once: true,- });- },- }),- text: async (): Promise<string> => '',- };- }- beats.push(Date.now());- const reply = beatReply();- if (reply === 'fail') throw new TypeError('fetch failed');- if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };- return { ok: true, status: 200, text: async (): Promise<string> => '' };- });- /** The server ends the current stream, as it does a lapsed connection. */- const drop = (): void => {- if (current) closeQuietly(current);- };- return { fetchMock, opens, beats, drop };- }-- it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {- // The lockout: beats fail for a while, so the gap between them climbs to- // its 30-second cap. The stream then reopens. The server starts its- // 6-second clock on the new connection — and the client used to sit out- // the rest of the 30 seconds before saying anything, lose the connection- // for silence, and repeat that on every connection it opened afterwards.- vi.useFakeTimers();- let failing = true;- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));- const { abort } = makeStream(server.fetchMock, { rooms: [] });-- // Long enough of failed beats for the gap to reach its cap.- await vi.advanceTimersByTimeAsync(70_000);- failing = false;- server.drop();- await vi.advanceTimersByTimeAsync(40_000);-- const reopenedAt = server.opens[1];- expect(reopenedAt).toBeDefined();- const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);- expect(firstBeatAfter).toBeDefined();- expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);- abort.abort();- });-- it('keeps beating at the base cadence on the new connection once it lands', async () => {- vi.useFakeTimers();- let failing = true;- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));- const { abort } = makeStream(server.fetchMock, { rooms: [] });-- await vi.advanceTimersByTimeAsync(70_000);- failing = false;- server.drop();- await vi.advanceTimersByTimeAsync(5_000);- const settled = server.beats.length;- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);-- expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);- abort.abort();- });-- it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {- // The brake the old ordering provided, which beating promptly on every- // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The- // wait now sits before each reopen and doubles, so a refusing server sees- // a handful of reopens, not one every couple of seconds per agent.- vi.useFakeTimers();- const server = reattachingServer(() => 404);- const { abort } = makeStream(server.fetchMock, { rooms: [] });-- await vi.advanceTimersByTimeAsync(60_000);-- // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.- expect(server.opens.length).toBeLessThanOrEqual(6);- abort.abort();- });-- it('does not throw away a connection the stream reopened while it waited', async () => {- // A lapsed connection's socket is closed by the server, and the stream- // loop reopens it by itself. If that lands during the heartbeat's wait- // after a refusal, reopening again would discard a healthy connection.- vi.useFakeTimers();- let refuse = true;- const server = reattachingServer(() => (refuse ? 404 : 'ok'));- const { abort } = makeStream(server.fetchMock, { rooms: [] });-- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);- expect(server.beats.length).toBe(1);- refuse = false;- // The server ends the stream now, inside the heartbeat's wait.- server.drop();- await vi.advanceTimersByTimeAsync(20_000);-- // One open for the start, one for the stream's own reopen — not a third- // from the heartbeat discarding it.- expect(server.opens.length).toBe(2);+ // Attempts at 1 s and 3 s: the next is not due until 7 s.+ expect(server.sockets).toHaveLength(3);abort.abort();});});@@ -865,137 +521,98 @@ describe('reopening while an open is still in flight', () => {* The next open can be held unanswered, the way a slow connection holds it.*/function incarnationServer() {- const state = {- generation: 0,- opens: 0,- beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',- subscribeDrops: false,- holdNext: false,- release: (): void => {},- drop: (): void => {},- };- const body = (gen: number, signal: AbortSignal) =>- new ReadableStream<Uint8Array>({- start(controller) {- const close = (): void => {- try {- controller.close();- } catch {- // already closed- }- };- state.drop = close;- controller.enqueue(connected(gen));- signal.addEventListener('abort', close, { once: true });- },- });- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {- if (String(url).includes('connection/subscribe')) {- if (!state.subscribeDrops)- return { ok: true, status: 200, text: async (): Promise<string> => '' };- state.subscribeDrops = false;- // The stream ends while the claim is still in flight.- state.holdNext = true;- state.drop();- await new Promise((r) => setTimeout(r, 3_000));- return { ok: true, status: 200, text: async (): Promise<string> => '' };- }- if (!String(url).includes('/events')) {- if (state.beat === 'slow-404-and-drop') {- // The stream ends while this beat is in flight, and its answer comes- // back just after the client has started reopening.- state.beat = 'ok';- state.holdNext = true;- state.drop();- await new Promise((r) => setTimeout(r, 1_200));- return { ok: false, status: 404, text: async (): Promise<string> => '' };- }- if (state.beat === 404)- return { ok: false, status: 404, text: async (): Promise<string> => '' };- return { ok: true, status: 200, text: async (): Promise<string> => '' };+ const state = { generation: 0, holdNext: false, release: (): void => {} };+ serve((socket) => {+ const claimed = socket.params.get('expected_generation');+ if (claimed !== null && Number(claimed) < state.generation) {+ socket.refuse(409, { code: 'taken_over', message: 'reattach refused' });+ return;}- state.opens += 1;- const claimed = new URL(String(url)).searchParams.get('expected_generation');- if (claimed !== null && Number(claimed) < state.generation)- return {- ok: false,- status: 409,- body: null,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),- };// The server acts on the open the moment it arrives.state.generation += 1;- const gen = state.generation;- const answer = {- ok: true,- status: 200,- body: body(gen, init.signal),- text: async (): Promise<string> => '',+ const generation = state.generation;+ const accept = (): void => {+ socket.open();+ socket.announce(generation);};- if (!state.holdNext) return answer;+ if (!state.holdNext) return accept();state.holdNext = false;- return new Promise((resolve, reject) => {- state.release = () => resolve(answer);- init.signal.addEventListener(- 'abort',- () => reject(new DOMException('aborted', 'AbortError')),- { once: true }- );- });+ state.release = accept;});- return { state, fetchMock };+ return state;}it('does not cancel an open the server has already acted on, and so does not stand down', async () => {// The self-takeover seen on the pilot. The server closes the stream, the- // client starts reopening, and a beat sent just before comes back 404.- // Reopening in answer used to cancel the open already in flight — which- // the server had acted on, moving the connection to a new incarnation.- // The client never read that answer, reattached claiming the old one, was- // refused as a takeover, and stood down for good. The "other client" was- // its own cancelled request.+ // client starts reopening, and something asks for another reopen while+ // that open is unanswered. Cancelling the open in flight does not undo it+ // server-side: the client never reads the incarnation it made, reattaches+ // claiming the old one, is refused as a takeover, and stands down for+ // good. The "other client" was its own cancelled request.vi.useFakeTimers();- const { state, fetchMock } = incarnationServer();+ const state = incarnationServer();const onEvicted = vi.fn();- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });-- await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true, onEvicted });+ await vi.advanceTimersByTimeAsync(0);expect(state.generation).toBe(1);- state.beat = 'slow-404-and-drop';- // Long enough for the refused beat's wait to end while the open is held.+ state.holdNext = true;+ server.sockets[0]!.drop();+ await vi.advanceTimersByTimeAsync(1_000);+ expect(server.sockets).toHaveLength(2);++ stream.setSpawnCapable(false);await vi.advanceTimersByTimeAsync(12_000);+ // Left to be answered rather than cancelled.+ expect(server.sockets).toHaveLength(2);+ expect(server.sockets[1]!.closedByClient).toBeNull();+state.release();- await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);+ await vi.advanceTimersByTimeAsync(1_000);expect(onEvicted).not.toHaveBeenCalled();- expect(state.opens).toBe(2);- expect(beatGenerations(fetchMock).at(-1)).toBe(2);+ expect(server.sockets).toHaveLength(3);+ const carried = server.sockets[2]!.params;+ expect(carried.get('expected_generation')).toBe('2');+ expect(carried.has('spawn_capable')).toBe(false);+ expect(state.generation).toBe(3);abort.abort();});it('carries out a repoint asked for mid-open once that open is answered', async () => {// The other way into the same hole: a room change reopens to include the- // new room. Mid-open, it must wait for the answer rather than cancel it —- // and must still happen, since the open in flight was built without it.+ // new room. Mid-open, it must wait for the answer rather than cancel it,+ // and must still happen, since the open in flight may have been built+ // without it.vi.useFakeTimers();- const { state, fetchMock } = incarnationServer();+ const state = incarnationServer();+ let subscribeDrops = true;+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) => {+ if (url.includes('connection/subscribe') && subscribeDrops) {+ subscribeDrops = false;+ // The stream ends while the claim is still in flight.+ state.holdNext = true;+ server.sockets.at(-1)!.drop();+ await new Promise((r) => setTimeout(r, 3_000));+ }+ return answer();+ });const onEvicted = vi.fn();- const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ const { stream, abort } = makeStream({ rooms: ['room-a'], onEvicted }, fetchMock);+ await vi.advanceTimersByTimeAsync(0);- state.subscribeDrops = true;const repointing = stream.repoint('room-b');// The claim returns while the stream's reopen is held unanswered.await vi.advanceTimersByTimeAsync(5_000);+ expect(server.sockets).toHaveLength(2);+ expect(server.sockets[1]!.closedByClient).toBeNull();state.release();await vi.advanceTimersByTimeAsync(5_000);await repointing;expect(onEvicted).not.toHaveBeenCalled();- const opens = urlsFor(fetchMock, '/events');- expect(opens.at(-1)).toContain('rooms=room-b');+ expect(server.sockets.at(-1)!.params.get('rooms')).toBe('room-b');+ expect(server.sockets.at(-1)!.params.get('expected_generation')).toBe('2');abort.abort();});});@@ -1006,37 +623,19 @@ describe('a connection another client takes over', () => {// is too late to protect the winner: by the time it refuses, a displaced// client has already rewritten the winner's rooms. The claim carries the// incarnation for that reason, and a refusal ends this client.- const opens: string[] = [];const subscribes: (number | null)[] = [];- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {- if (String(url).includes('/events')) {- opens.push(String(url));- return {- ok: true,- status: 200,- body: openForever(),- text: async (): Promise<string> => '',- };- }- if (String(url).includes('connection/subscribe')) {- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };- subscribes.push(sent.generation);- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: 'taken_over', message: 'not your connection' } }),- };- }- return { ok: true, status: 200, text: async (): Promise<string> => '' };+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {+ if (!url.includes('connection/subscribe')) return answer();+ subscribes.push((JSON.parse(init?.body ?? '{}') as { generation: number | null }).generation);+ return takenOver('not your connection');});const evicted: Eviction[] = [];- const { stream, abort } = makeStream(fetchMock, {- rooms: ['!old'],- onEvicted: (e) => evicted.push(e),- });+ const { stream, abort } = makeStream(+ { rooms: ['!old'], onEvicted: (e) => evicted.push(e) },+ fetchMock+ );await flush();- const opensBefore = opens.length;+ const opensBefore = server.sockets.length;await stream.repoint('!new');await flush();@@ -1045,38 +644,25 @@ describe('a connection another client takes over', () => {expect(subscribes).toEqual([0]);expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);// No reopen: coming back would be the takeover the refusal just denied.- expect(opens.length).toBe(opensBefore);+ expect(server.sockets).toHaveLength(opensBefore);abort.abort();});it('does not claim a room before the frame that names its incarnation', async () => {// Claiming nothing is how a client too old to have an incarnation gets// through, so a claim sent in the window before the first frame would go- // through the same door — and that window is exactly when this client may+ // through the same door, and that window is exactly when this client may// already have been displaced without knowing it.- let announce: () => void = () => {};+ serve((socket) => socket.open());const subscribes: (number | null)[] = [];- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {- if (String(url).includes('/events'))- return {- ok: true,- status: 200,- // Attached, but silent: the server has not said which incarnation- // this socket is yet.- body: new ReadableStream<Uint8Array>({- start(controller) {- announce = () => controller.enqueue(connected());- },- }),- text: async (): Promise<string> => '',- };- if (String(url).includes('connection/subscribe')) {- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };- subscribes.push(sent.generation);- }- return { ok: true, status: 200, text: async (): Promise<string> => '' };+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {+ if (url.includes('connection/subscribe'))+ subscribes.push(+ (JSON.parse(init?.body ?? '{}') as { generation: number | null }).generation+ );+ return answer();});- const { stream, abort } = makeStream(fetchMock, { rooms: ['!old'] });+ const { stream, abort } = makeStream({ rooms: ['!old'] }, fetchMock);await flush();const repointed = stream.repoint('!new');@@ -1084,7 +670,7 @@ describe('a connection another client takes over', () => {expect(subscribes).toEqual([]);- announce();+ server.sockets[0]!.announce();await repointed;expect(subscribes).toEqual([0]);@@ -1093,22 +679,17 @@ describe('a connection another client takes over', () => {it('halts instead of reopening, because reopening would be a takeover back', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: frameThenClose('evicted', {- code: 'taken_over',- reason: 'another stream attached to this connection',- room_id: null,- }),- text: async (): Promise<string> => '',- }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );+ serve((socket) => {+ socket.open();+ socket.frame('evicted', {+ code: 'taken_over',+ reason: 'another stream attached to this connection',+ room_id: null,+ });+ socket.drop();+ });const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort } = makeStream({rooms: [],onEvicted: (eviction) => evicted.push(eviction),});@@ -1117,7 +698,7 @@ describe('a connection another client takes over', () => {// Two clients reopening each other's connection is the loop this prevents:// whoever reopens wins, so a displaced client that reopens starts it again.- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(server.sockets).toHaveLength(1);expect(evicted).toEqual([{code: EVICTION_TAKEN_OVER,@@ -1128,61 +709,43 @@ describe('a connection another client takes over', () => {abort.abort();});- it('stands down when its heartbeat is refused, rather than taking the connection back', async () => {+ it('stands down when its reattach is refused, rather than trying again', async () => {+ // A client that missed its eviction and comes back. The server refuses+ // the reattach without disturbing the holder, and this is the only thing+ // left that can tell the loser it lost.vi.useFakeTimers();- // The case the SSE frame cannot cover: this client's socket dropped before- // the eviction reached it, so the refused heartbeat is the only way it ever- // hears that it lost.- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }- : {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({- detail: {- code: 'taken_over',- message: 'connection conn-1 was reopened since incarnation 0',- },- }),- }- );- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {- rooms: [],- onEvicted: (eviction) => evicted.push(eviction),+ let generation = 0;+ serve((socket) => {+ if (socket.params.has('expected_generation')) {+ // Someone else attached while we were away, so our claim is stale.+ socket.refuse(409, { code: 'taken_over', message: 'reattach refused' });+ return;+ }+ generation += 1;+ socket.open();+ socket.announce(generation);});+ const evicted: Eviction[] = [];+ const { abort } = makeStream({ rooms: [], onEvicted: (e) => evicted.push(e) });- await vi.advanceTimersByTimeAsync(5 * 60_000);+ await vi.advanceTimersByTimeAsync(0);+ server.sockets[0]!.drop();+ await vi.advanceTimersByTimeAsync(30_000);- // One beat and no second open: reopening is a takeover, so a loser that- // reopens pulls the connection straight back off the client that won it.- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);+ expect(server.sockets).toHaveLength(2);+ expect(server.sockets[1]!.params.get('expected_generation')).toBe('1');+ // And it stopped asking: retrying is how the pair trade the connection.+ await vi.advanceTimersByTimeAsync(5 * 60_000);+ expect(server.sockets).toHaveLength(2);abort.abort();});it('still reopens on a refusal that names no code, the way an old server sends it', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: openUntilAborted(init),- text: async (): Promise<string> => '',- }- : {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: 'connection conn-1 has no stream attached' }),- }- );+ serve((socket) => socket.refuse(409, 'connection conn-1 has no stream attached'));const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort } = makeStream({rooms: [],onEvicted: (eviction) => evicted.push(eviction),});@@ -1191,27 +754,20 @@ describe('a connection another client takes over', () => {// Only the takeover is terminal. Everything else a 409 can mean is still// something a reopen fixes, and a server too old to say which is one of them.- expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);+ expect(server.sockets.length).toBeGreaterThan(1);expect(evicted).toEqual([]);abort.abort();});it('reads an old server’s wording as a takeover when it sends no code', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: frameThenClose('evicted', {- reason: 'another stream attached to this connection',- }),- text: async (): Promise<string> => '',- }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );+ serve((socket) => {+ socket.open();+ socket.frame('evicted', { reason: 'another stream attached to this connection' });+ socket.drop();+ });const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort } = makeStream({rooms: [],onEvicted: (eviction) => evicted.push(eviction),});@@ -1219,200 +775,190 @@ describe('a connection another client takes over', () => {await vi.advanceTimersByTimeAsync(5 * 60_000);expect(evicted[0]?.code).toBe(EVICTION_TAKEN_OVER);- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(server.sockets).toHaveLength(1);abort.abort();});});describe('a stream the server closes cleanly', () => {- /** Opens, closes at once, forever — the shape of a contested connection. */- function closesAtOnce() {- return vi.fn(async (url: string) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- controller.close();- },- }),- text: async (): Promise<string> => '',- }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );- }+ /** Opens, closes at once, forever: the shape of a contested connection. */+ const closesAtOnce: Script = (socket) => {+ socket.open();+ socket.drop();+ };it('waits before reopening rather than reconnecting at full rate', async () => {vi.useFakeTimers();- const fetchMock = closesAtOnce();- const { abort } = makeStream(fetchMock, { rooms: [] });+ serve(closesAtOnce);+ const { abort } = makeStream({ rooms: [] });// A minute of 1s, 2s, 4s… is seven opens. Without pacing a clean close it// is an unbounded spin, limited only by how fast the server can answer.await vi.advanceTimersByTimeAsync(60_000);- expect(urlsFor(fetchMock, '/events').length).toBeLessThanOrEqual(8);+ expect(server.sockets.length).toBeLessThanOrEqual(8);abort.abort();});it('keeps backing off rather than resetting on every handshake', async () => {vi.useFakeTimers();- const fetchMock = closesAtOnce();- const { abort } = makeStream(fetchMock, { rooms: [] });+ serve(closesAtOnce);+ const { abort } = makeStream({ rooms: [] });await vi.advanceTimersByTimeAsync(60_000);- const early = urlsFor(fetchMock, '/events').length;+ const early = server.sockets.length;await vi.advanceTimersByTimeAsync(60_000);- const late = urlsFor(fetchMock, '/events').length - early;+ const late = server.sockets.length - early;// An open is not evidence of a working stream. Resetting the curve on the// handshake held two contending clients at a reconnect a second forever.expect(late).toBeLessThan(early);abort.abort();});-});-it('does not acknowledge a delivery before the consumer has saved it', async () => {- let finish: () => void = () => {};- const saved = new Promise<void>((resolve) => {- finish = resolve;- });- const onEvent = vi.fn(() => saved);- const fetchMock = vi.fn(async (url: string) => {- if (!url.includes('/events')) return Response.json({});- return new Response(- new ReadableStream({- start(controller) {- controller.enqueue(- new TextEncoder().encode(- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"room","sequence":7}\n\n'- )- );- },- })- );- });- const { stream, abort } = makeStream(fetchMock, { rooms: ['room'], startCursor: 6, onEvent });- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());- expect(stream.position).toBe(6);- finish();- await vi.waitFor(() => expect(stream.position).toBe(7));- abort.abort();-});+ it('starts the backoff over once a stream has lasted', async () => {+ vi.useFakeTimers();+ const openedAt: number[] = [];+ serve((socket, index) => {+ openedAt.push(Date.now());+ // Four that close at once climb the curve; the fifth stays up.+ if (index < 4) closesAtOnce(socket, index);+ else attach(socket, index);+ });+ const { abort } = makeStream({ rooms: [] });-it('replays from an explicitly saved zero cursor instead of starting at head', async () => {- const fetchMock = vi- .fn()- .mockImplementation(async (url: string) =>- url.includes('/events?')- ? { ok: true, body: openForever() }- : { ok: true, json: async () => ({ rooms: [] }) }- );- const { abort } = makeStream(fetchMock, { rooms: [], startCursor: 0 });- try {- await flush();- expect(new URL(urlsFor(fetchMock, '/events?')[0]).searchParams.get('start_from')).toBe('0');- } finally {- abort.abort();- }-});+ // Opens at 0, 1, 3, 7 and 15 seconds.+ await vi.advanceTimersByTimeAsync(16_000);+ expect(server.sockets).toHaveLength(5);-function resetFrames(url: string): Response {- if (!url.includes('/events')) return Response.json({});- return new Response(- new ReadableStream({- start(controller) {- controller.enqueue(- new TextEncoder().encode(- 'event: gap\ndata: {"from_sequence":0,"resumed_at":0,"reason":"buffer reset"}\n\n' +- 'id: 1\nevent: message\ndata: {"type":"message","room_id":"room","sequence":1}\n\n'- )- );- },- })- );-}+ // Long enough to count as a working stream, then the server ends it.+ await vi.advanceTimersByTimeAsync(31_000);+ const closedAt = Date.now();+ server.sockets[4]!.drop();+ await vi.advanceTimersByTimeAsync(2_000);-it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {- let finish!: () => void;- const saved = new Promise<void>((resolve) => {- finish = resolve;+ // Back at the first step of the curve, not the 16 seconds it had reached.+ expect(server.sockets).toHaveLength(6);+ expect(openedAt[5]! - closedAt).toBeLessThanOrEqual(1_000);+ abort.abort();});- const onGap = vi.fn(() => saved);- const onEvent = vi.fn();- const { stream, abort } = makeStream(- vi.fn(async (url: string) => resetFrames(url)),- { rooms: ['room'], startCursor: 4, onGap, onEvent }- );- try {- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());- expect(onGap).toHaveBeenCalledWith({- fromSequence: 0,- resumedAt: 0,- cursorReset: true,- reason: 'buffer reset',+});++describe('the cursor', () => {+ it('does not acknowledge a delivery before the consumer has saved it', async () => {+ let finish: () => void = () => {};+ const saved = new Promise<void>((resolve) => {+ finish = resolve;+ });+ const onEvent = vi.fn(() => saved);+ serve((socket) => {+ socket.open();+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 7 }, 7);});- expect(stream.position).toBe(4);- expect(onEvent).not.toHaveBeenCalled();+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 6, onEvent });+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());+ expect(stream.position).toBe(6);finish();- await vi.waitFor(() => expect(stream.position).toBe(1));- expect(onEvent).toHaveBeenCalledOnce();- } finally {+ await vi.waitFor(() => expect(stream.position).toBe(7));abort.abort();- }-});+ });-it('does not advance or deliver when persisting the reset checkpoint fails', async () => {- const onGap = vi.fn(async () => {- throw new Error('journal unavailable');+ it('replays from an explicitly saved zero cursor instead of starting at head', async () => {+ const { abort } = makeStream({ rooms: [], startCursor: 0 });+ try {+ await flush();+ expect(server.sockets[0]?.params.get('start_from')).toBe('0');+ } finally {+ abort.abort();+ }});- const onEvent = vi.fn();- const { stream, abort } = makeStream(- vi.fn(async (url: string) => resetFrames(url)),- { rooms: ['room'], startCursor: 4, onGap, onEvent }- );- try {- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());- expect(stream.position).toBe(4);- expect(onEvent).not.toHaveBeenCalled();- } finally {++ it('starts at head when it has nothing saved, and resumes from its cursor on reconnect', async () => {+ vi.useFakeTimers();+ serve((socket, index) => {+ attach(socket, index);+ if (index === 0) {+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 12 }, 12);+ socket.drop();+ }+ });+ const { abort } = makeStream({ rooms: ['room'] });++ await vi.advanceTimersByTimeAsync(1_000);++ expect(server.sockets.map((socket) => socket.params.get('start_from'))).toEqual(['head', '12']);abort.abort();- }+ });++ /** A buffer reset, then the first event after it. */+ const reset: Script = (socket) => {+ socket.open();+ socket.frame('gap', { from_sequence: 0, resumed_at: 0, reason: 'buffer reset' });+ socket.frame('message', { type: 'message', room_id: 'room', sequence: 1 }, 1);+ };++ it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {+ let finish!: () => void;+ const saved = new Promise<void>((resolve) => {+ finish = resolve;+ });+ const onGap = vi.fn(() => saved);+ const onEvent = vi.fn();+ serve(reset);+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 4, onGap, onEvent });+ try {+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());+ expect(onGap).toHaveBeenCalledWith({+ fromSequence: 0,+ resumedAt: 0,+ cursorReset: true,+ reason: 'buffer reset',+ });+ expect(stream.position).toBe(4);+ expect(onEvent).not.toHaveBeenCalled();+ finish();+ await vi.waitFor(() => expect(stream.position).toBe(1));+ expect(onEvent).toHaveBeenCalledOnce();+ } finally {+ abort.abort();+ }+ });++ it('does not advance or deliver when persisting the reset checkpoint fails', async () => {+ const onGap = vi.fn(async () => {+ throw new Error('journal unavailable');+ });+ const onEvent = vi.fn();+ serve(reset);+ const { stream, abort } = makeStream({ rooms: ['room'], startCursor: 4, onGap, onEvent });+ try {+ await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());+ expect(stream.position).toBe(4);+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+ });});describe('the permission to start a session', () => {- function reopenable() {- return vi.fn(async (url: string, init: { signal: AbortSignal }) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: openUntilAborted(init),- text: async (): Promise<string> => '',- }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );- }-it('is redeclared on the wire when it changes under a live connection', async () => {// Turning automatic sessions off while the controller is connected has to// reach the server, or it goes on promising a session nothing will start.- const fetchMock = reopenable();- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });await flush();- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);- expect(urlsFor(fetchMock, '/events')[0]).toContain('spawn_capable=true');+ expect(server.sockets).toHaveLength(1);+ expect(server.sockets[0]?.params.get('spawn_capable')).toBe('true');stream.setSpawnCapable(false);await flush();- const opens = urlsFor(fetchMock, '/events');- expect(opens).toHaveLength(2);- expect(opens[1]).not.toContain('spawn_capable');+ expect(server.sockets).toHaveLength(2);+ expect(server.sockets[0]?.closedByClient).toBe(1000);+ expect(server.sockets[1]?.params.has('spawn_capable')).toBe(false);// A reattach, not a takeover: the incarnation goes with it, so the server// recognises the same client rather than evicting it in favour of itself.- expect(opens[1]).toContain('expected_generation=0');+ expect(server.sockets[1]?.params.get('expected_generation')).toBe('0');abort.abort();});@@ -1426,86 +972,59 @@ describe('the permission to start a session', () => {let generation = 0;let opens = 0;const announce: Array<() => void> = [];- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {- if (!String(url).includes('/events'))- return { ok: true, status: 200, text: async (): Promise<string> => '' };- const claimed = new URL(String(url)).searchParams.get('expected_generation');- if (claimed !== null && Number(claimed) !== generation)- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({ detail: { code: EVICTION_TAKEN_OVER } }),- };+ serve((socket) => {+ const claimed = socket.params.get('expected_generation');+ if (claimed !== null && Number(claimed) !== generation) {+ socket.refuse(409, { code: EVICTION_TAKEN_OVER });+ return;+ }opens += 1;generation += 1;- const held = opens === holdStateOn;const named = generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- let closed = false;- const send = () => {- if (!closed) controller.enqueue(connected(named));- };- if (held) announce.push(send);- else send();- init.signal.addEventListener(- 'abort',- () => {- closed = true;- controller.close();- },- { once: true }- );- },- }),- text: async (): Promise<string> => '',- };+ socket.open();+ if (opens === holdStateOn) announce.push(() => socket.announce(named));+ else socket.announce(named);});- return { fetchMock, announce };+ return { announce };}it('holds a second change until the server has answered the first', async () => {// Redeclaring reattaches, which claims the incarnation this client believes// it holds. A second change sent before the first open was answered claims- // the one before it — which the server has moved past, and refuses as a+ // the one before it, which the server has moved past and refuses as a// takeover. That would take the agent's only connection off the air over// two clicks in a row.const onEvicted = vi.fn();- const { fetchMock, announce } = fencingServer(2);- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true, onEvicted });+ const { announce } = fencingServer(2);+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true, onEvicted });try {await flush();stream.setSpawnCapable(false);await flush();- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(server.sockets).toHaveLength(2);// The reopen is out but unanswered, so this one has nothing safe to claim.stream.setSpawnCapable(true);await flush();- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(server.sockets).toHaveLength(2);- announce[0]();+ announce[0]!();await flush();// Without the hold, this is where the connection has already gone: the// third open claims the incarnation the second one replaced.expect(onEvicted).not.toHaveBeenCalled();- const opens = urlsFor(fetchMock, '/events');- expect(opens).toHaveLength(3);- expect(opens[2]).toContain('spawn_capable=true');- expect(opens[2]).toContain('expected_generation=2');+ expect(server.sockets).toHaveLength(3);+ expect(server.sockets[2]?.params.get('spawn_capable')).toBe('true');+ expect(server.sockets[2]?.params.get('expected_generation')).toBe('2');} finally {abort.abort();}});it('carries nothing when a change made in that window is taken back', async () => {- const { fetchMock, announce } = fencingServer(2);- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ const { announce } = fencingServer(2);+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });try {await flush();stream.setSpawnCapable(false);@@ -1514,262 +1033,278 @@ describe('the permission to start a session', () => {stream.setSpawnCapable(false);await flush();- announce[0]();+ announce[0]!();await flush();// The held open already declares what the setting settled back to.- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);+ expect(server.sockets).toHaveLength(2);} finally {abort.abort();}});it('does not reopen the socket when it is set to what it already is', async () => {- const fetchMock = reopenable();- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });+ const { stream, abort } = makeStream({ rooms: [], spawnCapable: true });await flush();stream.setSpawnCapable(true);await flush();- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(server.sockets).toHaveLength(1);abort.abort();});});-it('routes command wakeups separately from room events and their cursor', async () => {- const onCommands = vi.fn();- const onEvent = vi.fn();- const fetchMock = vi.fn(async (url: string) =>- url.includes('/events')- ? new Response(- new ReadableStream({- start(controller) {- controller.enqueue(connected());- controller.enqueue(encodeFrame('session_commands', { session_ids: ['session'] }));- },- }),- { headers: { 'content-type': 'text/event-stream' } }- )- : Response.json({})- );- const { abort } = makeStream(fetchMock, { rooms: [], scope: 'all', onCommands, onEvent });- try {- await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));- expect(onEvent).not.toHaveBeenCalled();- } finally {- abort.abort();+describe('frames that are not room events', () => {+ /** An attached connection that then sends these frames. */+ function streaming(...frames: [string, unknown][]): void {+ serve((socket) => {+ attach(socket, 0);+ for (const [event, data] of frames) socket.frame(event, data);+ });}-});-function streaming(...frames: Uint8Array[]) {- return vi.fn(async (url: string) =>- url.includes('/events')- ? new Response(- new ReadableStream({- start(controller) {- controller.enqueue(connected());- for (const frame of frames) controller.enqueue(frame);- },- }),- { headers: { 'content-type': 'text/event-stream' } }- )- : Response.json({})- );-}+ it('routes command wakeups separately from room events and their cursor', async () => {+ const onCommands = vi.fn();+ const onEvent = vi.fn();+ streaming(['session_commands', { session_ids: ['session'] }]);+ const { stream, abort } = makeStream({ rooms: [], scope: 'all', onCommands, onEvent });+ try {+ await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));+ expect(onEvent).not.toHaveBeenCalled();+ expect(stream.position).toBe(0);+ } finally {+ abort.abort();+ }+ });-it('hands an approval outcome to its own callback, not the room-event path', async () => {- const onApprovalOutcome = vi.fn();- const onEvent = vi.fn();- const outcome = {- session_id: 'session',- request_id: 'permission',- state: 'answered',- answer: '0',- answered_by: '@person:test',- answered_at: '2026-09-24T12:00:00Z',- };- const { abort } = makeStream(streaming(encodeFrame('approval_outcome', outcome)), {- rooms: [],- scope: 'all',- onApprovalOutcome,- onEvent,+ it('hands an approval outcome to its own callback, not the room-event path', async () => {+ const onApprovalOutcome = vi.fn();+ const onEvent = vi.fn();+ const outcome = {+ session_id: 'session',+ request_id: 'permission',+ state: 'answered',+ answer: '0',+ answered_by: '@person:test',+ answered_at: '2026-09-24T12:00:00Z',+ };+ streaming(['approval_outcome', outcome]);+ const { abort } = makeStream({ rooms: [], scope: 'all', onApprovalOutcome, onEvent });+ try {+ await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }});- try {- await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));- expect(onEvent).not.toHaveBeenCalled();- } finally {- abort.abort();- }-});-it('drops a frame it does not know that is not a room event', async () => {- const onEvent = vi.fn();- const { abort } = makeStream(+ it('drops a frame it does not know that is not a room event', async () => {+ const onEvent = vi.fn();streaming(- encodeFrame('something_new', { detail: 'from a later server' }),- encodeFrame('approval_outcome', { session_id: 'session' }),- encodeFrame('message', { type: 'message', room_id: 'room', payload: {} })- ),- { rooms: [], scope: 'all', onEvent }- );- try {- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));- expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });- } finally {- abort.abort();- }-});+ ['something_new', { detail: 'from a later server' }],+ ['approval_outcome', { session_id: 'session' }],+ ['message', { type: 'message', room_id: 'room', payload: {} }]+ );+ const { abort } = makeStream({ rooms: [], scope: 'all', onEvent });+ try {+ await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));+ expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });+ } finally {+ abort.abort();+ }+ });-it('hands a relayed session command to its own callback', async () => {- const onSessionCommand = vi.fn();- const onEvent = vi.fn();- const command = {- contractVersion: 1,- commandId: 'command-1',- sessionId: 'session',- epoch: 'epoch',- origin: { surface: 'console', actorId: 'owner', roomId: null, threadId: null, messageId: null },- body: { type: 'turn.interrupt', turnId: 'turn' },- };- const { abort } = makeStream(- streaming(- encodeFrame('session_command', { commandId: 'no-session' }),- encodeFrame('session_command', command)- ),- { rooms: [], scope: 'all', onSessionCommand, onEvent }- );- try {- await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));- expect(onSessionCommand).toHaveBeenCalledTimes(1);- expect(onEvent).not.toHaveBeenCalled();- } finally {- abort.abort();- }-});+ it('hands a relayed session command to its own callback', async () => {+ const onSessionCommand = vi.fn();+ const onEvent = vi.fn();+ const command = {+ contractVersion: 1,+ commandId: 'command-1',+ sessionId: 'session',+ epoch: 'epoch',+ origin: {+ surface: 'console',+ actorId: 'owner',+ roomId: null,+ threadId: null,+ messageId: null,+ },+ body: { type: 'turn.interrupt', turnId: 'turn' },+ };+ streaming(['session_command', { commandId: 'no-session' }], ['session_command', command]);+ const { abort } = makeStream({ rooms: [], scope: 'all', onSessionCommand, onEvent });+ try {+ await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));+ expect(onSessionCommand).toHaveBeenCalledTimes(1);+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+ });-it('hands a room another connection took over to its own callback', async () => {- const onRoomReleased = vi.fn();- const onEvent = vi.fn();- const { abort } = makeStream(+ it('hands a room another connection took over to its own callback', async () => {+ const onRoomReleased = vi.fn();+ const onEvent = vi.fn();streaming(- encodeFrame('room_released', { session_id: 'no-room' }),- encodeFrame('room_released', { room_id: 'room', session_id: 'session' }),- encodeFrame('room_released', { room_id: 'other', session_id: null })- ),- { rooms: [], scope: 'all', onRoomReleased, onEvent }- );- try {- await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));- expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });- expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });- expect(onEvent).not.toHaveBeenCalled();- } finally {- abort.abort();- }-});--it('says when the server has confirmed an open', async () => {- const onConnected = vi.fn();- const { abort } = makeStream(streaming(), { rooms: [], scope: 'all', onConnected });- try {- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));- } finally {- abort.abort();- }+ ['room_released', { session_id: 'no-room' }],+ ['room_released', { room_id: 'room', session_id: 'session' }],+ ['room_released', { room_id: 'other', session_id: null }]+ );+ const { abort } = makeStream({ rooms: [], scope: 'all', onRoomReleased, onEvent });+ try {+ await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));+ expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });+ expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();+ }+ });});-it('says when an open stream ended and it is trying again', async () => {- const onConnected = vi.fn();- const onDisconnected = vi.fn();- let opens = 0;- const fetchMock = vi.fn(async (url: string) => {- if (!url.includes('/events')) return Response.json({});- opens += 1;- if (opens === 1)- return new Response(- new ReadableStream({- start(controller) {- controller.enqueue(connected());- controller.close();- },- }),- { headers: { 'content-type': 'text/event-stream' } }- );- return new Response(openForever(), { headers: { 'content-type': 'text/event-stream' } });+describe('the connection’s lifecycle', () => {+ it('says when the server has confirmed an open', async () => {+ const onConnected = vi.fn();+ const { abort } = makeStream({ rooms: [], scope: 'all', onConnected });+ try {+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));+ } finally {+ abort.abort();+ }});- const { abort } = makeStream(fetchMock, {- rooms: [],- scope: 'all',- onConnected,- onDisconnected,++ it('says when an open stream ended and it is trying again', async () => {+ vi.useFakeTimers();+ serve((socket, index) => {+ attach(socket, index);+ if (index === 0) socket.drop();+ });+ const onConnected = vi.fn();+ const onDisconnected = vi.fn();+ const { abort } = makeStream({ rooms: [], scope: 'all', onConnected, onDisconnected });+ try {+ await vi.advanceTimersByTimeAsync(0);+ expect(onDisconnected).toHaveBeenCalledTimes(1);+ expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });+ expect(onConnected).toHaveBeenCalledTimes(1);++ await vi.advanceTimersByTimeAsync(1_000);+ expect(onConnected).toHaveBeenCalledTimes(2);+ expect(onDisconnected).toHaveBeenCalledTimes(1);+ } finally {+ abort.abort();+ }});- try {- await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));- expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });- expect(onConnected).toHaveBeenCalledTimes(1);- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });- expect(onDisconnected).toHaveBeenCalledTimes(1);- } finally {- abort.abort();- }});-it('states placements on the attached incarnation, and raises on a refusal', async () => {- const bodies: unknown[] = [];- let refuse = false;- const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {- if (url.includes('/events'))- return new Response(- new ReadableStream({- start(controller) {- controller.enqueue(connected(4));- },- }),- { headers: { 'content-type': 'text/event-stream' } }- );- if (url.includes('connection/placements')) {+describe('placements', () => {+ it('states placements on the attached incarnation, and raises on a refusal', async () => {+ serve((socket) => {+ socket.open();+ socket.announce(4);+ });+ const bodies: unknown[] = [];+ let refuse = false;+ const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {+ if (!url.includes('connection/placements')) return answer();bodies.push(JSON.parse(init!.body!));- return refuse ? new Response('not a member', { status: 403 }) : Response.json({ ok: true });+ return refuse ? answer(403, 'not a member') : answer(200, '{"ok":true}');+ });+ const { stream, abort } = makeStream({ rooms: [], scope: 'all' }, fetchMock);+ try {+ await stream.replacePlacements({ session: 'room' });+ expect(bodies).toEqual([+ { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },+ ]);+ refuse = true;+ await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');+ } finally {+ abort.abort();+ }+ });++ it('does not give its connection up to itself over a request answered across its own reopen', async () => {+ // Seen on a real host: placements stated under incarnation 4, the stream+ // reopened (to 5) before the answer came, and the answer, "taken over:+ // reopened since 4, now at 5", named this very client as the new holder.+ // It stood down for good, with nobody else anywhere near the connection.+ vi.useFakeTimers();+ serve((socket, index) => {+ socket.open();+ socket.announce(4 + index);+ });+ let answerPlacements: (response: ReturnType<typeof answer>) => void = () => {};+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) =>+ url.includes('connection/placements')+ ? new Promise<ReturnType<typeof answer>>((resolve) => {+ answerPlacements = resolve;+ })+ : answer()+ );+ const evicted: Eviction[] = [];+ const onConnected = vi.fn();+ const { stream, abort } = makeStream(+ {+ rooms: [],+ scope: 'all',+ onConnected,+ onEvicted: (eviction) => evicted.push(eviction),+ },+ fetchMock+ );+ try {+ await vi.advanceTimersByTimeAsync(0);+ expect(onConnected).toHaveBeenCalledTimes(1);+ const stating = stream.replacePlacements({ session: 'room' });+ const settled = expect(stating).rejects.toThrow('409');+ await vi.advanceTimersByTimeAsync(0);+ expect(postsTo(fetchMock, 'connection/placements')).toHaveLength(1);++ server.sockets[0]!.drop();+ await vi.advanceTimersByTimeAsync(1_000);+ expect(onConnected).toHaveBeenCalledTimes(2);+ answerPlacements(+ takenOver('connection conn-1 has been reopened since incarnation 4 and is now at 5')+ );++ await settled;+ expect(evicted).toEqual([]);+ } finally {+ abort.abort();+ }+ });++ it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {+ serve((socket) => {+ socket.open();+ socket.announce(4);+ });+ const fetchMock = vi.fn(async (url: string, _init?: { body?: string }) =>+ url.includes('connection/placements') ? takenOver('another client holds it') : answer()+ );+ const evicted: Eviction[] = [];+ const { stream, abort } = makeStream(+ { rooms: [], scope: 'all', onEvicted: (eviction) => evicted.push(eviction) },+ fetchMock+ );+ try {+ await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');+ expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);+ } finally {+ abort.abort();}- return Response.json({});});- const { stream, abort } = makeStream(fetchMock, { rooms: [], scope: 'all' });- try {- await stream.replacePlacements({ session: 'room' });- expect(bodies).toEqual([- { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },- ]);- refuse = true;- await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');- } finally {- abort.abort();- }});-describe('a hosted worker', () => {+describe('a hosted worker over the socket', () => {const worker = { capability: 'cap-1', bootId: 'boot-1', instanceId: 'i-1', stateVersion: 1 };- function refused(status: number, code: string) {- return {- ok: false,- status,- body: null,- text: async (): Promise<string> => JSON.stringify({ detail: { code, message: code } }),- };- }-- it('states its capability and host on the open', async () => {- const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => ({- ok: true,- status: 200,- body: openForever(),- text: async (): Promise<string> => '',- }));- const { abort } = makeStream(fetchMock, { rooms: [], worker });+ it('states its capability and host on the socket it opens', async () => {+ const { abort } = makeStream({ rooms: [], worker });await flush();- const open = fetchMock.mock.calls.find((c) => String(c[0]).includes('/events'));- expect(open?.[1]?.headers).toMatchObject({+ expect(server.sockets[0]?.headers).toEqual({+ Authorization: 'Bearer tok','X-Switch-Worker-Capability': 'cap-1','X-Switch-Host-Boot-Id': 'boot-1','X-Switch-Host-Instance-Id': 'i-1',@@ -1778,353 +1313,84 @@ describe('a hosted worker', () => {abort.abort();});+ it('sends no worker headers when it is not a worker', async () => {+ const { abort } = makeStream({ rooms: [] });+ await flush();+ expect(server.sockets[0]?.headers).toEqual({ Authorization: 'Bearer tok' });+ abort.abort();+ });+it.each([[403, WORKER_CAPABILITY_OBSOLETE],[403, 'worker_capability_required'],[426, 'upgrade_required'],[403, 'hosted_worker_only'],- ])('stops for good on a %i %s open', async (status, code) => {+ ])('stops for good on a %i %s refusal, and does not try the stream', async (status, code) => {vi.useFakeTimers();- const fetchMock = vi.fn(async () => refused(status, code));+ serve((socket) => socket.refuse(status, { code, message: code }));const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {+ const { abort, fetchMock } = makeStream({rooms: [],worker,onEvicted: (e) => evicted.push(e),});await vi.advanceTimersByTimeAsync(5 * 60_000);- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(server.sockets).toHaveLength(1);+ expect(postsTo(fetchMock, '/events')).toEqual([]);expect(evicted.map((e) => e.code)).toEqual([code]);abort.abort();});it('backs off and retries while another worker is attached', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async () => refused(409, 'worker_already_attached'));+ serve((socket) => socket.refuse(409, { code: 'worker_already_attached', message: 'attached' }));const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {- rooms: [],- worker,- onEvicted: (e) => evicted.push(e),- });+ const { abort } = makeStream({ rooms: [], worker, onEvicted: (e) => evicted.push(e) });await vi.advanceTimersByTimeAsync(10_000);- expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);- expect(urlsFor(fetchMock, '/events').length).toBeLessThan(6);+ expect(server.sockets.length).toBeGreaterThan(1);+ expect(server.sockets.length).toBeLessThan(6);expect(evicted).toEqual([]);abort.abort();});it('halts on a launch_superseded eviction', async () => {vi.useFakeTimers();- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? {- ok: true,- status: 200,- body: frameThenClose('evicted', { code: 'launch_superseded', reason: 'bumped' }),- text: async (): Promise<string> => '',- }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );- const evicted: Eviction[] = [];- const { abort } = makeStream(fetchMock, {- rooms: [],- worker,- onEvicted: (e) => evicted.push(e),+ serve((socket) => {+ attach(socket, 0);+ socket.frame('evicted', { code: 'launch_superseded', reason: 'bumped', room_id: null });+ socket.drop();});+ const evicted: Eviction[] = [];+ const { abort } = makeStream({ rooms: [], worker, onEvicted: (e) => evicted.push(e) });await vi.advanceTimersByTimeAsync(5 * 60_000);- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(server.sockets).toHaveLength(1);expect(evicted.map((e) => e.code)).toEqual([EVICTION_LAUNCH_SUPERSEDED]);abort.abort();});it('hands worker frames over in order and not as room events', async () => {- const body = new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected(4));- controller.enqueue(encodeFrame('worker_attached', { launch_revision: 3 }));- controller.enqueue(encodeFrame('operation', { id: 'op-1' }));- },+ serve((socket) => {+ attach(socket, 0);+ socket.frame('worker_attached', { launch_revision: 3 });+ socket.frame('operation', { id: 'op-1' });});- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );const frames: [string, unknown][] = [];- const events: unknown[] = [];- const { abort } = makeStream(fetchMock, {+ const onEvent = vi.fn();+ const { abort } = makeStream({rooms: [],worker,- onEvent: (e) => void events.push(e),+ onEvent,onWorkerFrame: (name, data) => void frames.push([name, data]),});- await flush();- expect(frames).toEqual([- ['worker_attached', { launch_revision: 3 }],- ['operation', { id: 'op-1' }],- ]);- expect(events).toEqual([]);- abort.abort();- });-- it('sends an up-call as its connection and incarnation, and raises a refusal with its code', async () => {- const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => {- const u = String(url);- if (u.includes('/events'))- return {- ok: true,- status: 200,- body: openForever(),- text: async (): Promise<string> => '',- };- if (u.endsWith('/claim')) return refused(409, 'already_claimed');- return {- ok: true,- status: 200,- text: async (): Promise<string> => '{"queued_operations":[]}',- };- });- const { stream, abort } = makeStream(fetchMock, { rooms: [], worker });- await expect(- stream.workerCall('/agents/agent-1/connection/idle', { busy: false })- ).resolves.toEqual({- queued_operations: [],- });- const idle = fetchMock.mock.calls.find((c) => String(c[0]).endsWith('/connection/idle'));- expect(String(idle?.[0])).toBe('https://switch.test/agents/agent-1/connection/idle');- expect(JSON.parse(String(idle?.[1]?.body))).toEqual({- busy: false,- connection_id: 'conn-1',- generation: 0,- });- const claim = stream.workerCall('/hosted/operations/op-1/claim', {});- await expect(claim).rejects.toBeInstanceOf(WorkerCallError);- await expect(claim).rejects.toMatchObject({ status: 409, code: 'already_claimed' });- abort.abort();- });-});--describe('test_gap_above_cursor_is_not_cursor_reset', () => {- it('reports a restart gap resuming above the cursor as no cursor reset', async () => {- const body = new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected());- controller.enqueue(- new TextEncoder().encode(- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"r"}\n\n'- )- );- controller.enqueue(- encodeFrame('gap', {- from_sequence: 7,- resumed_at: 2 ** 32 - 1,- reason: 'server restarted',- rooms: [],- })- );- },- });- const fetchMock = vi.fn(async (url: string) =>- String(url).includes('/events')- ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }- : { ok: true, status: 200, text: async (): Promise<string> => '' }- );- const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];- const { stream, abort } = makeStream(fetchMock, { rooms: [], onGap: (g) => void gaps.push(g) });- await flush();- expect(gaps).toEqual([expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false })]);- expect(stream.position).toBe(2 ** 32 - 1);- abort.abort();- });-});--it('does not give its connection up to itself over a request answered across its own reopen', async () => {- // Seen on a real host: placements stated under incarnation 4, the stream- // reopened (to 5) before the answer came, and the answer — "taken over:- // reopened since 4, now at 5" — named this very client as the new holder.- // It stood down for good, with nobody else anywhere near the connection.- const opened: ReadableStreamDefaultController<Uint8Array>[] = [];- let answerPlacements: (response: Response) => void = () => {};- const fetchMock = vi.fn(async (url: string) => {- if (url.includes('/events')) {- const generation = 4 + opened.length;- return new Response(- new ReadableStream({- start(controller) {- opened.push(controller);- controller.enqueue(connected(generation));- },- }),- { headers: { 'content-type': 'text/event-stream' } }- );+ try {+ await vi.waitFor(() => expect(frames).toHaveLength(2));+ expect(frames).toEqual([+ ['worker_attached', { launch_revision: 3 }],+ ['operation', { id: 'op-1' }],+ ]);+ expect(onEvent).not.toHaveBeenCalled();+ } finally {+ abort.abort();}- if (url.includes('connection/placements'))- return new Promise<Response>((resolve) => {- answerPlacements = resolve;- });- return Response.json({});- });- const evicted: Eviction[] = [];- const onConnected = vi.fn();- const { stream, abort } = makeStream(fetchMock, {- rooms: [],- scope: 'all',- onConnected,- onEvicted: (eviction) => evicted.push(eviction),- });- try {- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));- const stating = stream.replacePlacements({ session: 'room' });- await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));-- opened[0]!.close();- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });- answerPlacements(- Response.json(- {- detail: {- code: 'taken_over',- message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',- },- },- { status: 409 }- )- );-- await expect(stating).rejects.toThrow('409');- expect(evicted).toEqual([]);- } finally {- abort.abort();- }-});--it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {- const fetchMock = vi.fn(async (url: string) => {- if (url.includes('/events'))- return new Response(- new ReadableStream({- start(controller) {- controller.enqueue(connected(4));- },- }),- { headers: { 'content-type': 'text/event-stream' } }- );- if (url.includes('connection/placements'))- return Response.json(- { detail: { code: 'taken_over', message: 'another client holds it' } },- { status: 409 }- );- return Response.json({});- });- const evicted: Eviction[] = [];- const { stream, abort } = makeStream(fetchMock, {- rooms: [],- scope: 'all',- onEvicted: (eviction) => evicted.push(eviction),- });- try {- await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');- expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);- } finally {- abort.abort();- }-});--describe('the heartbeat across its own reopen after a refusal', () => {- /**- * What took every in-Console watcher off the air when a laptop woke- * (2026-10-01, four agents within ten seconds). The connection had lapsed- * while the machine slept, so the first beat after waking was refused with a- * 404. The heartbeat asked the stream to reopen and went straight back to- * its gate — which it had found open a moment before the stream loop closed- * it for the reopen. One beat went out under the lapsed incarnation right- * behind the open that replaced it. The server, holding the new incarnation,- * answered that beat `taken_over`, and the client believed it: it stood down- * over a connection it had just made itself. In the server log the open and- * the stale beat arrived 11 ms apart, every time.- */- function lapsingServer() {- let generation = 0;- let lapsed = false;- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {- if (String(url).includes('/events')) {- generation += 1;- lapsed = false;- const gen = generation;- return {- ok: true,- status: 200,- body: new ReadableStream<Uint8Array>({- start(controller) {- controller.enqueue(connected(gen));- init.signal.addEventListener(- 'abort',- () => {- try {- controller.error(new DOMException('The operation was aborted.', 'AbortError'));- } catch {- // Already closed.- }- },- { once: true }- );- },- }),- text: async (): Promise<string> => '',- };- }- if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };- const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;- if (claimed !== null && claimed !== generation) {- return {- ok: false,- status: 409,- text: async (): Promise<string> =>- JSON.stringify({- detail: {- code: 'taken_over',- message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,- },- }),- };- }- return { ok: true, status: 200, text: async (): Promise<string> => '' };- });- /** The server sweeps the connection for silence; its socket stays up- * client-side, the way a socket does across a sleep. */- const lapse = (): void => {- lapsed = true;- };- return { fetchMock, lapse };- }-- it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {- vi.useFakeTimers();- const server = lapsingServer();- const evicted: Eviction[] = [];- const { abort } = makeStream(server.fetchMock, {- rooms: [],- onEvicted: (eviction) => evicted.push(eviction),- });-- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);- server.lapse();- await vi.advanceTimersByTimeAsync(30_000);-- // The beat refused with a 404 is the last one under the lapsed- // incarnation: what follows it on the wire is the reopen, not another beat- // that the server could only take for someone else's.- const sent = server.fetchMock.mock.calls.map((call) =>- String(call[0]).includes('/events')- ? 'open'- : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`- );- const reopen = sent.indexOf('open', 1);- expect(reopen).toBeGreaterThan(0);- expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);- expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);- expect(evicted).toEqual([]);- abort.abort();});});diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.tsindex e9147bdd3..e7d963c4a 100644--- a/console/packages/switch-agent-runtime/src/event-stream.ts+++ b/console/packages/switch-agent-runtime/src/event-stream.ts@@ -1,6 +1,6 @@import { contractRange } from './artifacts';import { ReattachFence } from './reattach-fence';-import { readSse, type SseFrame } from './sse';+import { readSse } from './sse';import type { AgentBridgeEvent, SwitchCredentials } from './types';import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';@@ -8,44 +8,68 @@ import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';const AGENT_PROTOCOL = contractRange('agent-protocol', RUNTIME_ARTIFACT);/**- * The agent bridge's push transport (CHOO-1857), client side.+ * The agent bridge's connection (CHOO-1857), client side.*- * Replaces the long-poll: one SSE stream carries events, one heartbeat proves- * we are alive, and both are tied to a connection id we choose. The connection- * outlives its socket — losing the stream stops delivery but does not end the- * connection, so reopening within the heartbeat TTL keeps the room slot and the- * role lease and resumes from the cursor.+ * One WebSocket per connection, tied to a connection id we choose. It carries+ * the events down and the heartbeat both ways: the server sends `ping` every+ * heartbeat interval and we answer `pong` with our cursor, which is what keeps+ * the connection alive. The token is checked once, when the socket opens. The+ * connection outlives its socket: losing the socket stops delivery but does not+ * end the connection, so reconnecting within the heartbeat TTL keeps the room+ * slot and the role lease and resumes from the cursor.*- * Two things this buys that the poll could not:+ * A server from before the socket (agent-protocol 7 and older) has none, and+ * is spoken to the way this runtime used to: the same frames as a Server-Sent+ * Events stream, and a heartbeat we send ourselves, `POST /connection/beat`+ * every two seconds with the cursor. Which one is decided on each open (see+ * `openFrames`). That path is kept for a compatibility window and goes when+ * those servers are gone.*- * - **Resume.** Every event carries a sequence number; we reopen with- * `Last-Event-ID` and get exactly what we missed. The poll had no cursor at- * all — the server drained the queue on read, so anything delivered while we- * were away was simply gone.- * - **One heartbeat instead of three.** `/connection/renew`, `/leases/renew`- * and `/watch/heartbeat` collapse into `/connection/beat`.- *- * A gap is never silent. If the server cannot serve our cursor it says so, and- * `onGap` fires so the caller can tell the agent to re-read context rather than- * carry on believing it saw everything.+ * Every event carries a sequence number, and a reconnect names the last one we+ * handled, so we get exactly what we missed. A gap is never silent: if the+ * server cannot serve our cursor it says so, and `onGap` fires so the caller+ * can tell the agent to re-read context rather than carry on believing it saw+ * everything.*/-/** Cadence of the connection heartbeat. Must stay well inside the server's- * 6s TTL — the server declares the connection dead without it. */+/** How long a request on the side (a room claim, placements, a beat) may take. */+const REQUEST_TIMEOUT_MS = 4000;+/** Cadence of the heartbeat we send on the event stream. Must stay well inside+ * the server's 6s TTL: the server declares the connection dead without it. The+ * socket needs none; the server pings it. */export const BEAT_INTERVAL_MS = 2000;-const BEAT_REQUEST_TIMEOUT_MS = 4000;/*** How long a reopen waits for a beat already in flight before disowning it.** A beat answered inside this window is still believed, so a takeover that- * lands while the socket happens to be reopening is acted on. Past it the+ * lands while the stream happens to be reopening is acted on. Past it the* answer cannot be told apart from one our own reopen provoked, and the reopen* matters more: it is what restores delivery, and the server gives the* connection six seconds without a beat.*/export const BEAT_SETTLE_LIMIT_MS = 1000;+/** The agent-protocol revision that brought the socket. A server that speaks+ * an older one has no socket, only the event stream. */+const SOCKET_PROTOCOL_REVISION = 8;+/** The code a server answers the event stream with once it has removed it. */+const TRANSPORT_REMOVED = 'transport_removed';const INITIAL_BACKOFF_MS = 1000;const MAX_BACKOFF_MS = 30_000;+/**+ * The close code a server sends when it is restarting (uvicorn sends it to+ * every open socket on shutdown).+ *+ * A restart is the one ending we know is short, and the doubling backoff fits+ * it badly: the socket closes as shutdown begins, so the first retries are+ * refused while the server is down and the next one lands seconds after it is+ * back. For a while after this code we retry at a short, randomised interval+ * instead, which brings the agent back within a second of the server and+ * spreads a fleet of agents over that second rather than one instant.+ */+const SERVICE_RESTART = 1012;+const RESTART_WINDOW_MS = 60_000;+const RESTART_RETRY_MIN_MS = 250;+const RESTART_RETRY_SPREAD_MS = 500;/*** How long an open socket has to last before it counts as a working stream.*@@ -83,7 +107,7 @@ export type DeliveryFilter = 'all' | 'addressed';* - `taken_over` — another client attached to this connection id. **Terminal.*** Reopening is itself a takeover, so a displaced client that retries is how* two clients trade one connection back and forth forever.- * - `heartbeat_lapsed` — we stopped ticking. Recoverable: reopen and resume.+ * - `heartbeat_lapsed` — we stopped answering pings. Recoverable: reopen and resume.* - `closed` — the server ended it for some other reason; `roomId` names the* room when it was about one.* - `credentials_rejected` — decided here rather than sent: every reopen would@@ -163,7 +187,7 @@ function evictionCode(data: Record<string, unknown>): string {}/**- * Read the code off a refusal — a rejected heartbeat or a rejected reattach.+ * Read the code off a refusal — a rejected open or reattach.** Every refusal shares one status, and they call for opposite responses: being* superseded is terminal, because attaching is itself a takeover, while the@@ -325,7 +349,7 @@ export interface SwitchEventStreamDeps {cursorReset?: boolean;}): void | Promise<void>;/** Fired when another stream took this connection over, or it was closed.- * A `taken_over` eviction has already halted both loops before this runs:+ * A `taken_over` eviction has already halted the connection before this runs:* there is nothing left to reconnect, only something to report. */onEvicted(eviction: Eviction): void;/**@@ -335,21 +359,58 @@ export interface SwitchEventStreamDeps {worker: WorkerIdentity | null;onWorkerFrame?: (frame: WorkerFrameName, data: Record<string, unknown>) => Promise<void> | void;log: EventStreamLogger;- /** Aborts the stream and the heartbeat together. */+ /** Aborts the connection. */signal: AbortSignal;}+/**+ * The server refused to open the connection: a `refused` frame, or a close+ * code of 4000 plus an HTTP status. `body` is shaped like the HTTP error body+ * the same refusal would have had (`{"detail": ...}`), so the remedies that+ * read it do not care which transport it came over.+ */+export class OpenRefused extends Error {+ constructor(+ readonly status: number,+ readonly body: string+ ) {+ super(`HTTP ${status}: ${body}`);+ this.name = 'OpenRefused';+ }+}++/** One frame off the socket: `{event, data, id}`, `id` being its sequence number. */+export type SocketFrame = { event: string; id?: string; data: Record<string, unknown> };++/** What a connection's frames travel over: the socket, or the event stream an+ * old server offers instead. */+type Transport = 'socket' | 'stream';++/** The socket closed before it opened: the server is unreachable, or has no+ * socket to offer. Only a request to the event stream tells the two apart. */+class SocketNeverOpened extends Error {+ constructor(code: number | null) {+ super(+ `could not connect to Switch (socket closed with ${code ?? 'no code'}, and the event stream did not answer either): the server is unreachable`+ );+ this.name = 'SocketNeverOpened';+ }+}++/** Node's WebSocket, which, unlike a browser's, takes request headers. */+type HeaderedWebSocket = new (url: string, init: { headers: Record<string, string> }) => WebSocket;+export class SwitchEventStream {private readonly deps: SwitchEventStreamDeps;- /** Highest sequence number received. Sent on every beat and used as- * `Last-Event-ID` when reopening. Seeded from `startCursor` when the caller+ /** Highest sequence number received. Sent with every pong and as+ * `start_from` when reopening. Seeded from `startCursor` when the caller* needs to begin behind head rather than at it. */private cursor: number;/** Aborts only the current socket, so a reconnect can replace it without* tearing down the connection. */private socketAbort: AbortController | null = null;/** Aborts the connection for good. Distinct from the caller's signal: some- * refusals can never be retried into a success, and both loops have to end. */+ * refusals can never be retried into a success, and the loop has to end. */private readonly halt = new AbortController();private rooms: string[];/**@@ -364,52 +425,64 @@ export class SwitchEventStream {/*** Which incarnation of the connection id the server last told us we are.*- * Sent on every beat so the server can refuse a tick from a client that has- * been displaced: sharing the id is what makes takeover work, and it is also- * what makes the loser's beat indistinguishable from the winner's.+ * Sent as `expected_generation` on every reopen, so the server refuses a+ * reattach from a client that has been displaced: sharing the id is what+ * makes takeover work, and it is also what makes the loser's reopen+ * indistinguishable from the winner's.** Kept across a dropped socket rather than cleared. The connection outlives* its socket, so a client whose socket merely dropped is still the holder,- * and nulling this would have it beat unfenced — or, worse, stop beating and- * lose the connection it still owns. Null only before the first- * `connection_state`, and against a server too old to send one.+ * and nulling this would have it reopen unfenced and take the connection+ * back off whoever holds it now. Null only before the first+ * `connection_state`.*/private generation: number | null = null;/** The server leaves telling a room a session is starting to this client (`connection_state`). */private announces = false;/**- * The barrier between the heartbeat and the socket it beats for.+ * Closed from the moment an open starts until the frame naming the+ * incarnation that open made arrives.*- * The heartbeat waits on it before every tick, so no beat is sent between an- * open and the frame naming the incarnation that open made: there is nothing- * current to fence such a tick with, and a stale incarnation is worse than- * none — the server reads it as a takeover and we would stand down over our- * own reconnect. Nothing is lost by waiting, since a beat with no stream- * attached is refused anyway, and a server too old to carry an incarnation- * still sends the frame.- *- * It also disowns a beat that was already in flight when the open began, for- * the same reason from the other end: that answer was decided about the- * incarnation we have just replaced.+ * A reopen asked for in between waits on it (see `reopen`): there is nothing+ * current to claim, and a stale incarnation is worse than none, since the+ * server reads it as a takeover and we would stand down over our own+ * reconnect.*/private readonly fence = new ReattachFence();+ /**+ * Reopens asked for, and how many of those the open now in progress was+ * built after — so a reopen asked for mid-open is carried out once that open+ * is answered, never by cancelling it. See `reopen`.+ */+ private reopensWanted = 0;+ /** Until when the server is taken to be restarting (see `SERVICE_RESTART`). */+ private restartingUntil = 0;+ private openCarries = 0;+ /**+ * Whether to open the event stream rather than try the socket first.+ *+ * Set when a server we reached over the stream says it speaks a revision+ * from before the socket, so each reopen does not first knock on a door that+ * is not there. Cleared as soon as the server says otherwise: a+ * `connection_state` naming a later revision, or the stream refused as+ * removed. A socket that merely fails to open never sets it; only an answer+ * from the server does, so a network failure cannot leave us on the stream.+ */+ private preferStream = false;+ /** What the open in progress travels over, and what the attached one does. */+ private openingOver: Transport = 'socket';+ private attachedOver: Transport | null = null;/*** How many times the server has named an incarnation for this client, and* how to cut the heartbeat's current wait short when it does.** A fresh attach starts the server's 6-second clock on a connection that has- * never beaten, so the heartbeat must go out promptly — whatever back-off an- * earlier, failing connection had built up. See `beatLoop`.+ * never beaten, so on the event stream the heartbeat must go out promptly,+ * whatever back-off an earlier, failing connection had built up. See+ * `beatLoop`.*/private attaches = 0;private wakeBeat: () => void = () => {};- /**- * Reopens asked for, and how many of those the open now in progress was- * built after — so a reopen asked for mid-open is carried out once that open- * is answered, never by cancelling it. See `reopen`.- */- private reopensWanted = 0;- private openCarries = 0;constructor(deps: SwitchEventStreamDeps) {this.deps = deps;@@ -475,11 +548,11 @@ export class SwitchEventStream {* quick succession would permanently stand down the agent's only connection.* The confirmation carries whatever the declaration has settled on by then* instead, and a change that cancels itself out carries nothing. This is the- * same hazard the heartbeat's fence exists for, from the other side.+ * same hazard the fence exists for.*- * Nothing to fence before the first `connection_state`, or against a server- * too old to send one: with no incarnation to claim the reopen is a plain- * attach, which cannot be refused for being stale.+ * Nothing to fence before the first `connection_state`: with no incarnation+ * to claim the reopen is a plain attach, which cannot be refused for being+ * stale.*/private redeclare(): void {if (this.declaredSpawnCapable === this.spawnCapable) return;@@ -508,19 +581,16 @@ export class SwitchEventStream {this.reopensWanted += 1;if (!this.fence.admitting) return;// Held from this instant, not from when the stream loop gets round to its- // own `detaching`. Whoever asked for the reopen — the heartbeat, after a- // refusal — goes straight back to the gate, and could otherwise find it- // still open and send one more beat under the incarnation this reopen is- // about to replace. Arriving just behind the open, that beat is refused as- // a takeover and the client stands down over a connection it made itself.+ // own `detaching`, so a second reopen asked for straight after this one+ // waits for the incarnation it makes rather than claiming the old one.this.fence.closeAdmission();this.socketAbort?.abort();}/**- * Reopen only to get attached again — the heartbeat's reason. An open- * already in flight does exactly that, so there is nothing to add to it; and- * one scheduled after a dropped socket carries the request anyway.+ * Reopen only to get attached again, the heartbeat's reason. An open already+ * in flight does exactly that, so there is nothing to add to it; and one+ * scheduled after a dropped stream carries the request anyway.*/private reattach(): void {if (this.fence.admitting) this.reopen();@@ -598,7 +668,7 @@ export class SwitchEventStream {}/** A rejected credential is not an outage: every reopen would carry the same- * token, so end both loops and tell the owner once. */+ * token, so end the connection and tell the owner once. */private rejectCredentials(status: number, body: string): void {if (this.halt.signal.aborted) return;if (this.refuseWorker(status, body)) return;@@ -622,14 +692,12 @@ export class SwitchEventStream {** A request carries the incarnation it was sent under, and the server* refuses it as taken over once the connection has moved past that. But- * this client reopens its own stream — after an eviction, a refused beat, a+ * this client reopens its own socket — after an eviction, a dropped socket, a* repoint — and a request in flight across that reopen is refused the same* way, naming as the new holder the incarnation this client now is. Standing- * down on that gives the connection up to ourselves, for good. Heartbeats- * already discard such answers through the fence; this is the same test for- * the requests that do not go through it. A real takeover after our reopen- * is still caught: the next beat, under the incarnation we now hold, is- * refused, and stands down.+ * down on that gives the connection up to ourselves, for good. A real+ * takeover after our reopen is still caught: the server tells the socket we+ * now hold with an `evicted` frame.*/private supersededOurselves(sent: number | null): boolean {return sent !== null && this.generation !== null && this.generation !== sent;@@ -638,14 +706,14 @@ export class SwitchEventStream {/*** Give the connection up to the client that now holds it.*- * Ends both loops and reports through the same callback an `evicted` frame+ * Ends the connection and reports through the same callback an `evicted` frame* does, with the same code, so whatever records the stand-down does not have* to care which door the news came through.*/private standDown(): void {if (this.halt.signal.aborted) return;this.deps.log.warn('SwitchEventStream: the connection was taken over — standing down', {- event: 'switch_beat_superseded',+ event: 'switch_stream_superseded',connectionId: this.deps.connectionId,});this.halt.abort();@@ -764,7 +832,7 @@ export class SwitchEventStream {return text ? JSON.parse(text) : null;}- private post(path: string, body: unknown, timeoutMs = BEAT_REQUEST_TIMEOUT_MS) {+ private post(path: string, body: unknown, timeoutMs = REQUEST_TIMEOUT_MS) {const { creds } = this.deps;return fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/${path}`, {method: 'POST',@@ -787,7 +855,7 @@ export class SwitchEventStream {* the same line every thirty seconds until the app is quit, which is how a* real failure elsewhere gets lost. The first failure is reported, then* powers of two, and recovery is stated so the outage has an end in the log- * as well as a beginning — the same discipline the heartbeat already applies.+ * as well as a beginning.*/private async streamLoop(): Promise<void> {const { creds, connectionId, scope, filter, log, signal } = this.deps;@@ -803,6 +871,11 @@ export class SwitchEventStream {* server is ending streams on purpose.*/const pace = async (): Promise<void> => {+ if (Date.now() < this.restartingUntil) {+ const wait = RESTART_RETRY_MIN_MS + Math.random() * RESTART_RETRY_SPREAD_MS;+ await new Promise((r) => setTimeout(r, wait));+ return;+ }await new Promise((r) => setTimeout(r, backoff));backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);};@@ -811,9 +884,10 @@ export class SwitchEventStream {const socketAbort = new AbortController();this.socketAbort = socketAbort;// Closed before the open rather than after it lands: it is this open- // that makes the new incarnation, so the heartbeat has to be held from- // here until the frame naming it arrives. A beat already in flight is- // given a moment to come back and be believed, and disowned after that.+ // that makes the new incarnation, so a reopen and the event stream's+ // heartbeat have to wait from here until the frame naming it arrives. A+ // beat already in flight is given a moment to come back and be+ // believed, and disowned after that. The socket has none in flight.await this.fence.detaching(BEAT_SETTLE_LIMIT_MS);if (signal.aborted || this.halt.signal.aborted) return;let openedAt = 0;@@ -841,66 +915,49 @@ export class SwitchEventStream {// Reattaching, so say which incarnation we believe we still are and// let the server refuse us if we are wrong. An attach is a takeover,// and a client that missed its own eviction — a partition, a dropped- // socket, a beat whose refusal never arrived — would otherwise take+ // socket, an eviction frame that never arrived — would otherwise take// the connection straight back off whoever legitimately holds it. The// first open of this object's life sends nothing, which is how a// deliberate takeover still works: it has no incarnation to claim.if (this.generation !== null) params.set('expected_generation', String(this.generation));- const worker = this.deps.worker;- const resp = await fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/events?${params}`, {- headers: {- Authorization: `Bearer ${creds.token}`,- Accept: 'text/event-stream',- ...(this.cursor > 0 ? { 'Last-Event-ID': String(this.cursor) } : {}),- ...(worker- ? {- 'X-Switch-Worker-Capability': worker.capability,- 'X-Switch-Host-Boot-Id': worker.bootId,- 'X-Switch-Host-Instance-Id': worker.instanceId,- 'X-Switch-Worker-State-Version': String(worker.stateVersion),- }- : {}),- },- signal: AbortSignal.any([socketAbort.signal, signal, this.halt.signal]),- });-- if (!resp.ok || !resp.body) {- const body = await resp.text();- if (this.refuseWorker(resp.status, body)) return;+ for await (const frame of this.openFrames(params, socketAbort.signal, () => {+ openedAt = Date.now();+ this.restartingUntil = 0;+ log.debug('SwitchEventStream: connection open', {+ event: 'switch_stream_open',+ connectionId,+ cursor: this.cursor,+ rooms: this.rooms,+ });+ })) {+ await this.handleFrame(frame);+ if (frame.id) this.cursor = Math.max(this.cursor, Number(frame.id) || 0);+ }+ } catch (error) {+ if (error instanceof OpenRefused) {+ const { status, body } = error;+ if (this.refuseWorker(status, body)) return;// Reopening without the refused room is a different request from the// one that just failed, and the declared set strictly shrinks, so// this cannot spin: retry now rather than serving the backoff a// transport failure earned.- if (this.dropRefusedRooms(resp.status, body)) continue;- if (resp.status === 401 || resp.status === 403) {- this.rejectCredentials(resp.status, body);+ if (this.dropRefusedRooms(status, body)) continue;+ if (status === 401 || status === 403) {+ this.rejectCredentials(status, body);return;}- if (resp.status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {+ if (status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {// The reattach was refused because someone else holds the- // connection now, and nothing was disturbed in refusing it. This- // is the only place the loser can be told once its socket and its- // heartbeat have both stopped being able to reach it.+ // connection now, and nothing was disturbed in refusing it.this.standDown();return;}- throw new Error(`HTTP ${resp.status}: ${body}`);+ // The event stream we went to straight away is gone, so the server+ // has the socket now (`openFrames` has stopped preferring the+ // stream). A different request from the one refused, so retry at once.+ if (status === 410 && refusalCode(body) === TRANSPORT_REMOVED) continue;}-- openedAt = Date.now();- log.debug('SwitchEventStream: stream open', {- event: 'switch_stream_open',- connectionId,- cursor: this.cursor,- rooms: this.rooms,- });-- for await (const frame of readSse(resp.body, socketAbort.signal)) {- await this.handleFrame(frame);- if (frame.id) this.cursor = Math.max(this.cursor, Number(frame.id) || 0);- }- } catch (error) {failure = error;}@@ -939,16 +996,208 @@ export class SwitchEventStream {}}- private async handleFrame(frame: SseFrame): Promise<void> {+ /**+ * Open the connection and yield its frames, over the socket if the server+ * has one and over the event stream if it does not.+ *+ * Node's WebSocket does not say why a handshake failed, so a server with no+ * socket (it answers the upgrade with 404) looks exactly like one we cannot+ * reach. The event stream is what tells them apart: a socket that never+ * opened is followed at once by a request for the stream, and a server that+ * answers it is one we can talk to that way. Decided on each open rather+ * than once for good, so a network failure costs one request and decides+ * nothing, and a server upgraded under us is moved back onto the socket.+ */+ private async *openFrames(+ params: URLSearchParams,+ abort: AbortSignal,+ onOpen: () => void+ ): AsyncGenerator<SocketFrame> {+ const { creds } = this.deps;+ if (!this.preferStream) {+ this.openingOver = 'socket';+ const url = `${creds.apiEndpoint.replace(/^http/, 'ws')}/agents/${creds.agentId}/connection/ws?${params}`;+ try {+ yield* this.readSocket(url, abort, onOpen);+ return;+ } catch (error) {+ if (!(error instanceof SocketNeverOpened) || abort.aborted) throw error;+ this.openingOver = 'stream';+ yield* this.readEventStream(params, abort, onOpen, error);+ return;+ }+ }+ this.openingOver = 'stream';+ yield* this.readEventStream(params, abort, onOpen, null);+ }++ /** Who a hosted worker is, as the headers every open of its connection carries. */+ private workerHeaders(): Record<string, string> {+ const worker = this.deps.worker;+ if (!worker) return {};+ return {+ 'X-Switch-Worker-Capability': worker.capability,+ 'X-Switch-Host-Boot-Id': worker.bootId,+ 'X-Switch-Host-Instance-Id': worker.instanceId,+ 'X-Switch-Worker-State-Version': String(worker.stateVersion),+ };+ }++ /**+ * Open the connection as a Server-Sent Events stream, the way a server from+ * before the socket serves it, and yield its frames until it ends.+ *+ * `probing` is the socket's failure when this request is the one that tells+ * an old server from an unreachable one. Then anything short of a stream+ * means unreachable, and that failure is what is reported. A refusal is+ * still the server's answer, and handled as one.+ */+ private async *readEventStream(+ params: URLSearchParams,+ abort: AbortSignal,+ onOpen: () => void,+ probing: SocketNeverOpened | null+ ): AsyncGenerator<SocketFrame> {+ const { creds, signal } = this.deps;+ let resp: Response;+ try {+ resp = await fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/events?${params}`, {+ headers: {+ Authorization: `Bearer ${creds.token}`,+ Accept: 'text/event-stream',+ ...(this.cursor > 0 ? { 'Last-Event-ID': String(this.cursor) } : {}),+ ...this.workerHeaders(),+ },+ signal: AbortSignal.any([abort, signal, this.halt.signal]),+ });+ } catch (error) {+ throw probing ?? error;+ }+ if (!resp.ok || !resp.body) {+ const body = await resp.text();+ if (resp.status === 410 && refusalCode(body) === TRANSPORT_REMOVED) {+ // A server that has removed the stream has the socket.+ this.preferStream = false;+ throw probing ?? new OpenRefused(resp.status, body);+ }+ if (probing && resp.ok) throw probing;+ throw new OpenRefused(resp.status, body);+ }+ onOpen();+ yield* readSse(resp.body, abort);+ }++ /**+ * Open the connection's socket and yield its frames until it closes.+ *+ * The heartbeat is answered here, as each `ping` arrives, not queued behind+ * the frames: handling an event can take as long as the agent likes, and+ * the server must not decide we are gone while it does. The pong carries+ * the cursor as it stands, which is the last event fully handled.+ */+ private async *readSocket(+ url: string,+ abort: AbortSignal,+ onOpen: () => void+ ): AsyncGenerator<SocketFrame> {+ const Socket = (globalThis as { WebSocket?: HeaderedWebSocket }).WebSocket;+ if (!Socket) {+ throw new Error(+ `this runtime needs Node 22 or newer to connect to Switch (it has ${process.version}, with no WebSocket)`+ );+ }+ // A hosted worker proves who it is on the socket exactly as on the stream.+ const socket = new Socket(url, {+ headers: { Authorization: `Bearer ${this.deps.creds.token}`, ...this.workerHeaders() },+ });+ const pending: SocketFrame[] = [];+ let refused: unknown = null;+ let closed: { code: number; reason: string } | null = null;+ let opened = false;+ let wake: (() => void) | null = null;+ const notify = (): void => {+ const resolve = wake;+ wake = null;+ resolve?.();+ };+ socket.addEventListener('open', () => {+ opened = true;+ onOpen();+ });+ socket.addEventListener('message', (message: MessageEvent) => {+ let parsed: { event?: unknown; data?: unknown; id?: unknown };+ try {+ parsed = JSON.parse(String(message.data)) as typeof parsed;+ } catch {+ return;+ }+ if (parsed.event === 'ping') {+ socket.send(JSON.stringify({ type: 'pong', cursor: this.cursor }));+ return;+ }+ if (parsed.event === 'refused') {+ refused = (parsed.data as { detail?: unknown } | undefined)?.detail ?? null;+ return;+ }+ if (typeof parsed.event !== 'string') return;+ pending.push({+ event: parsed.event,+ data: (parsed.data as Record<string, unknown> | undefined) ?? {},+ ...(parsed.id !== undefined ? { id: String(parsed.id) } : {}),+ });+ notify();+ });+ socket.addEventListener('close', (event: CloseEvent) => {+ closed = { code: event.code, reason: event.reason };+ if (event.code === SERVICE_RESTART) this.restartingUntil = Date.now() + RESTART_WINDOW_MS;+ notify();+ });+ socket.addEventListener('error', () => notify());+ const stop = (): void => socket.close(1000);+ abort.addEventListener('abort', stop, { once: true });+ try {+ for (;;) {+ while (pending.length > 0) yield pending.shift() as SocketFrame;+ if (closed !== null || abort.aborted) break;+ await new Promise<void>((resolve) => {+ wake = resolve;+ });+ }+ const ending = closed as { code: number; reason: string } | null;+ if (ending !== null && ending.code >= 4000 && ending.code < 5000) {+ throw new OpenRefused(+ ending.code - 4000,+ JSON.stringify({ detail: refused ?? ending.reason })+ );+ }+ if (!opened && !abort.aborted) throw new SocketNeverOpened(ending?.code ?? null);+ } finally {+ abort.removeEventListener('abort', stop);+ if (socket.readyState === socket.CONNECTING || socket.readyState === socket.OPEN) {+ socket.close(1000);+ }+ }+ }++ private async handleFrame(frame: SocketFrame): Promise<void> {const { log, onGap, onEvicted, onEvent } = this.deps;switch (frame.event) {case 'connection_state':if (typeof frame.data.generation === 'number') this.generation = frame.data.generation;this.announces = frame.data.announce_session_starts === true;+ this.attachedOver = this.openingOver;+ // Reached over the stream: whether the server has a socket for next+ // time. One that names no revision is older than the declaration, and+ // so older than the socket.+ if (this.openingOver === 'stream') {+ const speaks = frame.data.protocol;+ this.preferStream = !(typeof speaks === 'number' && speaks >= SOCKET_PROTOCOL_REVISION);+ }log.debug('SwitchEventStream: connection established', {event: 'switch_stream_connected',rooms: frame.data.rooms,generation: this.generation,+ transport: this.attachedOver,// What the server says it is (CHOO-1865). Recorded, not acted on —// logging it is what makes "which versions are actually talking to// each other" answerable from a bug report rather than a guess.@@ -958,7 +1207,7 @@ export class SwitchEventStream {// A reopen asked for while this open was in flight, for something this// open was built too early to carry. Now that we know the incarnation// to claim, it is safe to replace the socket — and done before the- // gate opens, so no beat goes out on a socket about to be dropped.+ // gate opens, so nothing else is sent on a socket about to be dropped.if (this.reopensWanted > this.openCarries) {this.socketAbort?.abort();return;@@ -1030,8 +1279,8 @@ export class SwitchEventStream {roomId: frame.data.room_id ?? null,});// Halt before reporting: a takeover is the one ending that reopening- // cannot recover, because reopening is itself a takeover. Both loops- // end here, and nothing the callback does can restart them.+ // cannot recover, because reopening is itself a takeover. The+ // connection ends here, and nothing the callback does can restart them.if (code === EVICTION_TAKEN_OVER || code === EVICTION_LAUNCH_SUPERSEDED) this.halt.abort();onEvicted({code,@@ -1083,9 +1332,13 @@ export class SwitchEventStream {}/**- * The single heartbeat. Proves we are alive and reports the cursor.+ * The event stream's heartbeat. Proves we are alive and reports the cursor.*- * A 404 or 409 means we are not receiving — the connection expired, or it has+ * Only while the connection is attached over the event stream: the socket is+ * pinged by the server and answers from `readSocket`, and a beat beside it+ * would be a request every two seconds for nothing.+ *+ * A 404 or 409 means we are not receiving: the connection expired, or it has* no stream attached. Both are recovered by reopening, which resumes from the* cursor, and both count as a beat that did not land. The exception is the* 409 that says another client has taken the connection over, which no@@ -1096,11 +1349,11 @@ export class SwitchEventStream {* client that has stopped receiving while believing it is connected is* exactly the bug this transport exists to remove.*- * While beats succeed the cadence is fixed and short — the server declares the+ * While beats succeed the cadence is fixed and short: the server declares the* connection dead without them. While they fail it backs off, because a beat* that has already missed the TTL cannot save the connection: reopening the* stream is what re-establishes it, and that loop is doing its own retrying.- * Hammering a dead endpoint every two seconds forever is what this avoids —+ * Hammering a dead endpoint every two seconds forever is what this avoids:* an endpoint that is simply gone (a managed server's port after the stack was* destroyed) should cost a trickle of requests and a handful of log lines, not* a permanent stream of both.@@ -1117,7 +1370,7 @@ export class SwitchEventStream {/*** Wait `ms`, but stop waiting the moment the server names a new- * incarnation — and do not start waiting if it already has.+ * incarnation, and do not start waiting if it already has.** The back-off used to be served in full across a reattach. A beat that* had been failing left the loop sleeping for up to 30 seconds; the stream@@ -1163,7 +1416,7 @@ export class SwitchEventStream {* One beat, as a value rather than a throw.** A beat the fence disowns must have no outcome at all, and an exception- * is an outcome — it would slow the loop down over a request the reattach+ * is an outcome: it would slow the loop down over a request the reattach* itself invalidated. So the whole exchange, body included, happens inside* the flight the fence is timing, and comes back as something to ignore or* act on once it is known which.@@ -1196,6 +1449,13 @@ export class SwitchEventStream {// promise we were handed is the old one. Checked here, with nothing// awaited before the tick, so no beat leaves while admission is shut.if (!this.fence.admitting) continue;+ if (this.attachedOver !== 'stream') {+ // Attached over the socket, which carries its own heartbeat. Sleep+ // until the next attach, which may be over the stream.+ seenAttach = this.attaches;+ await pause(MAX_BACKOFF_MS);+ continue;+ }// A connection the server has just attached has never beaten, and has// 6 seconds to. Whatever an earlier connection's failures built up says// nothing about this one, so it starts at the base cadence.@@ -1211,7 +1471,7 @@ export class SwitchEventStream {// an incarnation we are no longer on and cannot be told apart from// one that arrived late about our own reopen. Inert: no stand-down,// no reopen, no cursor, and no mark against the beat rate either.- log.debug('SwitchEventStream: heartbeat answer discarded — reattached in flight', {+ log.debug('SwitchEventStream: heartbeat answer discarded: reattached in flight', {event: 'switch_beat_stale',connectionId,});@@ -1232,15 +1492,15 @@ export class SwitchEventStream {return;}if (status === 404 || status === 409) {- // The server answered, so this is not an outage — but it is not a+ // The server answered, so this is not an outage, but it is not a// beat that landed either: we are not attached, and only a reopen// fixes that. A reopen that keeps being refused is a client that// cannot currently succeed, and it must not spend the endpoint at// full rate while it fails.//// So the wait goes *before* the reopen, and grows with each refusal- // in a row. It used to go after — reopen at once, then sleep the- // back-off before beating — which starved the very connection the+ // in a row. It used to go after (reopen at once, then sleep the+ // back-off before beating), which starved the very connection the// reopen had just made. Pacing the reopen instead keeps the same// brake on a refusing server while letting each new connection beat// as soon as it is attached.@@ -1248,7 +1508,7 @@ export class SwitchEventStream {rejections += 1;const waitMs = Math.min(BEAT_INTERVAL_MS * 2 ** rejections, MAX_BACKOFF_MS);if ((failures & (failures - 1)) === 0) {- log.warn('SwitchEventStream: heartbeat rejected — reopening', {+ log.warn('SwitchEventStream: heartbeat rejected: reopening', {event: 'switch_beat_rejected',status,connectionId,@@ -1258,7 +1518,7 @@ export class SwitchEventStream {}await pause(waitMs);if (signal.aborted || this.halt.signal.aborted) return;- // The stream may have reattached on its own while we waited — the+ // The stream may have reattached on its own while we waited: the// server closes a lapsed connection's socket, and the stream loop// reopens it. Reopening again would throw that healthy connection// away; beat on it instead.diff --git a/console/packages/switch-agent-runtime/src/index.ts b/console/packages/switch-agent-runtime/src/index.tsindex ce12f58ca..5ee1940f4 100644--- a/console/packages/switch-agent-runtime/src/index.ts+++ b/console/packages/switch-agent-runtime/src/index.ts@@ -26,7 +26,7 @@ export {type ContractRange,} from './artifacts';export {- BEAT_INTERVAL_MS,+ OpenRefused,EVICTION_CLOSED,EVICTION_CREDENTIALS_REJECTED,EVICTION_HEARTBEAT_LAPSED,@@ -55,7 +55,7 @@ export {type RoomDelivery,type RoomReservation,} from './room-admission';-export { readSse, type SseFrame } from './sse';+export type { SocketFrame } from './event-stream';export type {AgentBridgeEvent,AttachmentRef,diff --git a/console/packages/switch-agent-runtime/src/sse.test.ts b/console/packages/switch-agent-runtime/src/sse.test.tsindex 7235ab9f9..da53e9838 100644--- a/console/packages/switch-agent-runtime/src/sse.test.ts+++ b/console/packages/switch-agent-runtime/src/sse.test.ts@@ -6,7 +6,7 @@ import { readSse, type SseFrame } from './sse';** These moved here with the parser. They used to sit beside a drift check that* compared two copies of this file; the copies are gone, so the check is too,- * but the behaviour it protected still needs testing — and now testing it once+ * but the behaviour it protected still needs testing, and now testing it once* genuinely covers every client, which was the point of the move.*/@@ -40,7 +40,7 @@ describe('SSE framing', () => {});it('leaves id undefined when the server sends none', async () => {- // Control frames carry no sequence — treating a missing id as 0 would drag+ // Control frames carry no sequence, treating a missing id as 0 would drag// the cursor backwards and replay the whole buffer on reconnect.const frames = await parse(['event: connection_state\ndata: {"rooms":[]}\n\n']);expect(frames[0].id).toBeUndefined();diff --git a/console/packages/switch-agent-runtime/src/sse.ts b/console/packages/switch-agent-runtime/src/sse.tsindex 667967ea2..13b8f2204 100644--- a/console/packages/switch-agent-runtime/src/sse.ts+++ b/console/packages/switch-agent-runtime/src/sse.ts@@ -2,8 +2,8 @@* The agent bridge's SSE framing.** One implementation, imported by everything that speaks the protocol: the- * local MCP runtime next to the agent, and Switch Console. It used to exist twice —- * separate deployables, no shared package — and the copies drifted within a day+ * local MCP runtime next to the agent, and Switch Console. It used to exist twice+ * (separate deployables, no shared package) and the copies drifted within a day* of being made. This package is what removed the excuse.** Deliberately free of imports and of anything specific to a host: it is thediff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.pyindex d52e3e94b..07eb88b9a 100644--- a/core/switch_core/artifacts.py+++ b/core/switch_core/artifacts.py@@ -33,8 +33,8 @@ ARTIFACT_VERSIONS: Final[dict[str, str]] = {CONTRACTS: Final[dict[str, dict[str, ContractRange]]] = {# The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel."agent-protocol": {- "switch-core": ContractRange(speaks=7, accepts=1),- "agent-runtime": ContractRange(speaks=7, accepts=1),+ "switch-core": ContractRange(speaks=8, accepts=1),+ "agent-runtime": ContractRange(speaks=8, accepts=1),},# The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong."gateway-api": {diff --git a/core/switch_core/bridges/agent/api/handlers.py b/core/switch_core/bridges/agent/api/handlers.pyindex 4a5c9f74d..1b29b2049 100644--- a/core/switch_core/bridges/agent/api/handlers.py+++ b/core/switch_core/bridges/agent/api/handlers.py@@ -1,7 +1,9 @@from __future__ import annotations+import asyncioimport hashlibimport logging+from collections.abc import AsyncGeneratorfrom contextvars import ContextVarfrom datetime import UTC, datetimefrom typing import Annotated, Any, cast@@ -14,6 +16,8 @@ from fastapi import (HTTPException,Query,UploadFile,+ WebSocket,+ WebSocketDisconnect,)from pydantic import ValidationErrorfrom sqlalchemy.ext.asyncio import AsyncSession@@ -86,6 +90,7 @@ from switch_core.bridges.agent.dependencies import ()from switch_core.bridges.agent.hosted_mailbox import deliver_on_attachfrom switch_core.bridges.agent.protocol.agent_connections import (+ HEARTBEAT_INTERVAL_SECONDS,TAKEN_OVER,AgentConnection,ClientDeclaration,@@ -99,6 +104,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (SupersededConnectionError,SupersededControlError,SupersededReattachError,+ Transport,UnfencedBeatError,UnfencedControlError,UnknownConnectionError,@@ -107,7 +113,13 @@ from switch_core.bridges.agent.protocol.agent_connections import (from switch_core.bridges.agent.protocol.agent_core import AgentCore, AgentExistsErrorfrom switch_core.bridges.agent.protocol.event_buffer import Readerfrom switch_core.bridges.agent.protocol.hosted_workers import hosted_launch_of-from switch_core.bridges.agent.protocol.stream import event_stream+from switch_core.bridges.agent.protocol.stream import (+ KEEPALIVE,+ Frame,+ encode_ws,+ event_frames,+ sse_stream,+)from switch_core.bridges.agent.registration_bootstrap import (BOOTSTRAP_KEY_TYPE,REGISTRATION_KEY_TYPES,@@ -747,16 +759,16 @@ async def poll_events() -> EventResponse | Response:"""Deliver the agent's events, as a push stream or a long poll.- `Accept: text/event-stream` opens a connection and streams: catch-up from- the client's cursor, then live delivery. Anything else falls back to the- long poll, which is served from the same buffer so the two cannot diverge- while both exist.+ The live connection is the WebSocket at `/connection/ws`. `Accept:+ text/event-stream` opens the same connection as a Server-Sent Events+ stream, with its heartbeat on `POST /connection/beat`: that is how an agent+ runtime built before the WebSocket connects (agent-protocol revision 7 and+ older), and it is kept for those clients for a compatibility window. It+ goes once no client still connects over it. Anything else falls back to+ the long poll, served from the same buffer.- The four declaration parameters are all optional and all default to None,- meaning *unknown* (CHOO-1865). `protocol` previously defaulted to the- server's own value, so a client that said nothing was read as having- agreed — and since no shipped client sent it, the check had never once- fired. Absent now records as unknown, and still connects.+ The declaration parameters are all optional and all default to None,+ meaning *unknown* (CHOO-1865): a client that says nothing still connects."""if accept and "text/event-stream" in accept:return await _open_event_stream(@@ -795,12 +807,12 @@ def _resolve_start_cursor(protocol: AgentCore,agent_id: str,start_from: str,- last_event_id: str | None,+ last_event_id: str | None = None,) -> int:- """Where a stream begins.+ """Where a connection's events begin: the head, or a sequence number.- `Last-Event-ID` wins when present: a reconnecting SSE client sends it- automatically and it is the most accurate statement of what it processed.+ `Last-Event-ID` wins when present: an event-stream client that reconnects+ sends it, and it is the most accurate statement of what it processed."""raw = last_event_id or start_fromif raw in ("", "head"):@@ -814,7 +826,7 @@ def _resolve_start_cursor() from exc-def _open_connection(+def _register_connection(*,protocol: AgentCore,agent: Agent,@@ -825,6 +837,7 @@ def _open_connection(cursor: int,declaration: ClientDeclaration,expected_generation: int | None,+ transport: Transport,) -> AgentConnection:try:conn = protocol.connections.open(@@ -836,6 +849,7 @@ def _open_connection(cursor=cursor,declaration=declaration,expected_generation=expected_generation,+ transport=transport,)except SupersededReattachError as exc:# Structured, like the refused heartbeat: this is the same ending, and@@ -869,7 +883,7 @@ def _open_connection(return conn-async def _open_event_stream(+async def _open_connection(*,agent: Agent,protocol: AgentCore,@@ -881,13 +895,20 @@ async def _open_event_stream(spawn_capable: bool,declaration: ClientDeclaration,rooms: str | None,- last_event_id: str | None,expected_generation: int | None,- worker_capability: str | None,- host_boot_id: str | None,- host_instance_id: str | None,- worker_state_version: int | None,-) -> StreamingResponse:+ worker_capability: str | None = None,+ host_boot_id: str | None = None,+ host_instance_id: str | None = None,+ worker_state_version: int | None = None,+ last_event_id: str | None = None,+ transport: Transport = "websocket",+) -> tuple[AgentConnection, AsyncGenerator[Frame]]:+ """Open or reattach the connection and claim its declared rooms.++ Shared by every transport, so the socket and the event stream fence,+ admit a hosted worker and claim rooms in exactly the same way. Returns+ the connection and its frames, for the transport to encode.+ """if not connection_id:raise HTTPException(status_code=400,@@ -907,8 +928,8 @@ async def _open_event_stream(cursor = _resolve_start_cursor(protocol, agent.id, start_from, last_event_id)- def open_connection() -> AgentConnection:- return _open_connection(+ def register() -> AgentConnection:+ return _register_connection(protocol=protocol,agent=agent,connection_id=connection_id,@@ -918,11 +939,12 @@ async def _open_event_stream(cursor=cursor,declaration=declaration,expected_generation=expected_generation,+ transport=transport,)launch_id = hosted_launch_of(agent.metadata_)if launch_id is None:- conn = open_connection()+ conn = register()else:# Admission, the open and the binding all happen under the launch# lock, so no revision bump lands between the check and the bind.@@ -940,7 +962,7 @@ async def _open_event_stream(instance_id=host_instance_id,state_version=worker_state_version,)- conn = open_connection()+ conn = register()if attach.takes_over is not None and attach.takes_over.id != conn.id:protocol.connections.close(attach.takes_over.id, TAKEN_OVER)protocol.connections.bind_worker(conn, attach.binding, attach.attached)@@ -951,7 +973,7 @@ async def _open_event_stream(# Built before anything below can yield, so it holds the generation this# open produced; a reconnect during the bookkeeping supersedes it rather# than being detached by it.- stream = event_stream(+ frames = event_frames(conn=conn,registry=protocol.connections,buffer=protocol.event_buffer,@@ -1013,9 +1035,55 @@ async def _open_event_stream(# session rather than starting another, and an open that failed after# registering its connection, retried on the same id, is still counted.await protocol.sessions.started(agent, conn)+ return conn, frames+++async def _open_event_stream(+ *,+ agent: Agent,+ protocol: AgentCore,+ config: SwitchConfig,+ connection_id: str | None,+ scope: str,+ event_filter: str,+ start_from: str,+ spawn_capable: bool,+ declaration: ClientDeclaration,+ rooms: str | None,+ last_event_id: str | None,+ expected_generation: int | None,+ worker_capability: str | None,+ host_boot_id: str | None,+ host_instance_id: str | None,+ worker_state_version: int | None,+) -> StreamingResponse:+ """The connection as a Server-Sent Events stream, for an old client.+ Opens exactly as the socket does and streams the same frames from the same+ loop, so the two cannot drift while both exist. A refusal is the HTTP+ error the socket would have sent as its `refused` frame.+ """+ _conn, frames = await _open_connection(+ agent=agent,+ protocol=protocol,+ config=config,+ connection_id=connection_id,+ scope=scope,+ event_filter=event_filter,+ start_from=start_from,+ spawn_capable=spawn_capable,+ declaration=declaration,+ rooms=rooms,+ expected_generation=expected_generation,+ worker_capability=worker_capability,+ host_boot_id=host_boot_id,+ host_instance_id=host_instance_id,+ worker_state_version=worker_state_version,+ last_event_id=last_event_id,+ transport="sse",+ )return StreamingResponse(- stream,+ sse_stream(frames),media_type="text/event-stream",headers={"Cache-Control": "no-cache",@@ -1026,6 +1094,32 @@ async def _open_event_stream()+def _record_beat(+ protocol: AgentCore,+ agent_id: str,+ connection_id: str,+ cursor: int,+ generation: int | None,+) -> AgentConnection:+ """Count a client's beat, and adopt its cursor: a pong on the socket, or a+ `POST /connection/beat` from an event-stream client.++ The cursor is clamped to the buffer head first. The buffer is in memory,+ so a restart resets the sequence while a client keeps reporting the+ number it had reached, and neither `beat()` nor `confirm()` will move a+ cursor backwards: adopting the stale number would skip every event up to+ it, silently, and mark them consumed on the way past.++ Raises the registry's refusal when the connection is no longer this+ client's to beat for: another socket took it over (`taken_over`), or it+ has no stream attached (`no_stream`), or it is gone.+ """+ cursor = min(cursor, protocol.event_buffer.head(agent_id))+ conn = protocol.connections.beat(agent_id, connection_id, cursor, generation)+ protocol.event_buffer.confirm(agent_id, conn.id, cursor)+ return conn++@router.post("/{agent_id}/connection/beat")async def connection_beat(agent_id: str,@@ -1033,49 +1127,233 @@ async def connection_beat(agent: Annotated[Agent, Depends(get_agent_from_scope)],protocol: Annotated[AgentCore, Depends(get_protocol)],) -> dict[str, Any]:- """The single per-connection heartbeat.-- Proves the client is alive and reports its cursor. Rejected when the- connection is unknown, dead, has no stream attached, or belongs to another- incarnation — an agent that can still make calls but is receiving nothing- must be told, not left believing it is connected. A refusal carries a code- beside its prose, because the remedies differ: `taken_over` is terminal for- the client that receives it, and the rest are recovered by reopening.+ """The heartbeat of a connection held over the event stream.++ The socket carries its own heartbeat; this is for a client built before it,+ kept with the event stream for the same compatibility window. Proves the+ client is alive and reports its cursor. Rejected when the connection is+ unknown, dead, has no stream attached, or belongs to another incarnation.+ A refusal carries a code beside its prose, because the remedies differ:+ `taken_over` is terminal for the client that receives it, and the rest are+ recovered by reopening."""- # A cursor above the buffer's head belongs to a previous life of this- # process: the buffer is in memory, so a restart resets the sequence while- # the client keeps beating the number it had reached. Both consumers below- # only ever move a cursor forward, so adopting it undoes the rewind the- # stream performs on resume — the connection then skips every event up to- # the stale value and confirms events it was never delivered. Clamp it here,- # where the untrusted value enters, rather than in either consumer.- head = protocol.event_buffer.head(agent.id)- cursor = min(req.cursor, head)-try:- conn = protocol.connections.beat(- agent.id, req.connection_id, cursor, req.generation+ conn = _record_beat(+ protocol, agent.id, req.connection_id, req.cursor, req.generation)except (NoStreamAttachedError,SupersededConnectionError,UnfencedBeatError,) as exc:- # All three refuse the tick, and one of them means something the others- # do not: a superseded tick is terminal for the client that sent it,- # because reopening is itself a takeover and would pull the connection- # back off the client that now holds it. Prose alone could not tell- # them apart, so every refusal was answered with a reopen.raise HTTPException(status_code=409, detail={"code": exc.code, "message": str(exc)}) from excexcept UnknownConnectionError as exc:raise HTTPException(status_code=404, detail=str(exc)) from exc-- protocol.event_buffer.confirm(agent.id, conn.id, cursor)return {"ok": True, "rooms": sorted(conn.rooms), "cursor": conn.cursor}+# A WebSocket close code for each refusal that would otherwise be an HTTP+# status: 4000 plus the status, so a client maps them onto one set of remedies.+def _ws_close_code(status: int) -> int:+ return 4000 + status++[email protected]("/{agent_id}/connection/ws")+async def connection_socket(+ websocket: WebSocket,+ agent_id: str,+ agent: Annotated[Agent, Depends(get_agent_from_scope)],+ protocol: Annotated[AgentCore, Depends(get_protocol)],+ config: Annotated[SwitchConfig, Depends(get_config)],+ connection_id: Annotated[str | None, Query()] = None,+ scope: Annotated[str, Query()] = "single",+ event_filter: Annotated[str, Query(alias="filter")] = "all",+ start_from: Annotated[str, Query()] = "head",+ spawn_capable: Annotated[bool, Query()] = False,+ protocol_version: Annotated[int | None, Query(alias="protocol")] = None,+ protocol_accepts: Annotated[int | None, Query()] = None,+ expected_generation: Annotated[int | None, Query()] = None,+ client: Annotated[str | None, Query()] = None,+ client_version: Annotated[str | None, Query()] = None,+ rooms: Annotated[str | None, Query()] = None,+ worker_capability: Annotated[+ str | None, Header(alias="x-switch-worker-capability")+ ] = None,+ host_boot_id: Annotated[str | None, Header(alias="x-switch-host-boot-id")] = None,+ host_instance_id: Annotated[+ str | None, Header(alias="x-switch-host-instance-id")+ ] = None,+ worker_state_version: Annotated[+ int | None, Header(alias="x-switch-worker-state-version")+ ] = None,+) -> None:+ """The agent's connection over one WebSocket: events down, heartbeats up.++ Opens exactly as the event stream does (same parameters, same fencing,+ same room claims) and carries the same frames, as JSON+ `{"event", "data", "id"}`. Authenticated once, when the socket opens.++ The heartbeat runs the other way round from `/connection/beat`: the server+ sends a `ping` frame every heartbeat interval and the client answers+ `{"type": "pong", "cursor": n}`, which counts as a beat. A client that+ stops answering lapses like one that stops beating. No request per beat,+ and no token lookup.++ A refusal on opening is sent as a `refused` frame carrying the status and+ detail an HTTP request would have been answered with, then the socket closes+ with code 4000 plus that status.++ A hosted agent's worker opens it as it opens the stream, with its+ capability and host identity in the `X-Switch-Worker-*` and+ `X-Switch-Host-*` headers, and is admitted and refused by the same rules+ (agent-protocol 7). Its frames come down the socket; its up-calls stay+ HTTP requests.+ """+ await websocket.accept()+ try:+ conn, frames = await _open_connection(+ agent=agent,+ protocol=protocol,+ config=config,+ connection_id=connection_id,+ scope=scope,+ event_filter=event_filter,+ start_from=start_from,+ spawn_capable=spawn_capable,+ declaration=ClientDeclaration(+ speaks=protocol_version,+ accepts=protocol_accepts,+ artifact=client,+ version=client_version,+ ),+ rooms=rooms,+ expected_generation=expected_generation,+ worker_capability=worker_capability,+ host_boot_id=host_boot_id,+ host_instance_id=host_instance_id,+ worker_state_version=worker_state_version,+ )+ except HTTPException as exc:+ await websocket.send_json(+ {+ "event": "refused",+ "data": {"status": exc.status_code, "detail": exc.detail},+ }+ )+ await websocket.close(code=_ws_close_code(exc.status_code))+ return++ # Bounded, so a client that reads slowly holds the stream back rather+ # than growing this queue.+ outbox: asyncio.Queue[Frame] = asyncio.Queue(maxsize=_SOCKET_OUTBOX)+ pump = asyncio.create_task(_pump_frames(frames, outbox))+ pongs = asyncio.create_task(+ _receive_pongs(websocket, protocol, agent.id, conn.id, conn.stream_generation)+ )+ loop = asyncio.get_running_loop()+ next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS+ try:+ # One writer: a WebSocket must not be sent to from two tasks at once,+ # so frames and pings both go out from this loop.+ while not pump.done() or not outbox.empty():+ if pongs.done():+ refusal = pongs.result()+ if refusal is not None:+ await websocket.send_json(+ {+ "event": "evicted",+ "data": {+ "code": getattr(refusal, "code", "closed"),+ "reason": str(refusal),+ "room_id": None,+ },+ }+ )+ return+ try:+ frame = await asyncio.wait_for(+ outbox.get(), timeout=max(0.0, next_ping - loop.time())+ )+ except TimeoutError:+ frame = None+ if frame is not None and frame is not KEEPALIVE:+ await websocket.send_json(encode_ws(frame))+ if loop.time() >= next_ping:+ await websocket.send_json({"event": "ping", "data": {}})+ next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS+ except WebSocketDisconnect:+ return+ except RuntimeError:+ # The server closed the socket under us (uvicorn does on shutdown), so+ # a send raises this rather than WebSocketDisconnect. Same ending.+ return+ finally:+ # The pump owns the stream: cancelling it runs the stream's own+ # cleanup (detaching it from the connection) inside that task.+ pump.cancel()+ pongs.cancel()+ failure = pump.exception() if pump.done() and not pump.cancelled() else None+ if failure is not None:+ logger.error(+ "[STREAM] agent=%s connection=%s delivery failed, closing its socket",+ agent.id,+ conn.id,+ exc_info=failure,+ )+ try:+ await websocket.close(code=1011 if failure is not None else 1000)+ except RuntimeError:+ pass # the client closed it first+++# Frames buffered between the stream and the socket.+_SOCKET_OUTBOX = 64+++async def _pump_frames(+ frames: AsyncGenerator[Frame], outbox: asyncio.Queue[Frame]+) -> None:+ # Closed here: cancelled while waiting on the outbox, the stream would+ # otherwise detach only when the generator is garbage collected.+ try:+ async for frame in frames:+ await outbox.put(frame)+ finally:+ await frames.aclose()+++async def _receive_pongs(+ websocket: WebSocket,+ protocol: AgentCore,+ agent_id: str,+ connection_id: str,+ generation: int,+) -> ConnectionError_ | None:+ """Turn each pong into a beat, until the client goes or a beat is refused.++ Returns the refusal, so the socket can say which one before it closes:+ `taken_over` is final for this client, the rest are recovered by+ reconnecting. Returns None when the client simply went away.+ """+ while True:+ try:+ message = await websocket.receive_json()+ except (WebSocketDisconnect, ValueError, RuntimeError):+ # Gone, or sent something that is not JSON: either way no more+ # beats are coming on this socket.+ return None+ if not isinstance(message, dict) or message.get("type") != "pong":+ continue+ raw = message.get("cursor", 0)+ cursor = raw if isinstance(raw, int) and raw >= 0 else 0+ try:+ _record_beat(protocol, agent_id, connection_id, cursor, generation)+ except ConnectionError_ as refusal:+ return refusal++def _current_connection(protocol: AgentCore,agent: Agent,diff --git a/core/switch_core/bridges/agent/api/schemas.py b/core/switch_core/bridges/agent/api/schemas.pyindex 1b1573453..3040f1815 100644--- a/core/switch_core/bridges/agent/api/schemas.py+++ b/core/switch_core/bridges/agent/api/schemas.py@@ -139,11 +139,12 @@ class ConnectionPlacementsRequest(BaseModel):class ConnectionBeatRequest(BaseModel):- """The single client tick that keeps a connection alive (CHOO-1857).+ """The heartbeat of a connection held over the event stream (CHOO-1857).- Replaces /connection/renew, /watch/heartbeat and /leases/renew: it proves- the client is alive *and* consuming, and reports how far it has read so the- event buffer knows what has been seen.+ Kept for clients built before the WebSocket, whose socket carries the+ heartbeat itself, for a compatibility window. Proves the client is alive+ *and* consuming, and reports how far it has read so the event buffer knows+ what has been seen."""connection_id: str@@ -153,7 +154,7 @@ class ConnectionBeatRequest(BaseModel):#: been displaced still holds the id and the token, and is otherwise#: indistinguishable from the one that replaced it. Null is accepted only#: while the connection's holder is a client built before the fence existed- #: — unknown, not current; from a holder that declares the revision which+ #: (unknown, not current); from a holder that declares the revision which#: carries it, a tick without one is refused.generation: int | None = Nonediff --git a/core/switch_core/bridges/agent/auth.py b/core/switch_core/bridges/agent/auth.pyindex f670b0d8f..edf161bbe 100644--- a/core/switch_core/bridges/agent/auth.py+++ b/core/switch_core/bridges/agent/auth.py@@ -1,15 +1,17 @@from __future__ import annotationsimport hashlib+import jsonimport loggingimport refrom dataclasses import dataclassfrom typing import Protocolimport jwt-from fastapi import HTTPException, Request+from fastapi import HTTPExceptionfrom jwt import PyJWKClientfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from starlette.requests import HTTPConnectionfrom starlette.responses import JSONResponse, Responsefrom starlette.types import ASGIApp, Receive, Scope, Send@@ -248,10 +250,9 @@ class BearerAuthMiddleware:returnif not auth_header.startswith("Bearer "):- response = Response(- "Missing or invalid Authorization header", status_code=401+ await _unauthorized(+ scope, receive, send, "Missing or invalid Authorization header")- await response(scope, receive, send)returntoken = auth_header[7:]@@ -325,8 +326,7 @@ class BearerAuthMiddleware:await self.app(scope, receive, send)return- response = Response("Invalid credentials", status_code=401)- await response(scope, receive, send)+ await _unauthorized(scope, receive, send, "Invalid credentials")async def _serve_controller(self,@@ -531,12 +531,36 @@ class BearerAuthMiddleware:return await self._agent_store.get_by_oauth_client_id(session, client_id)-def _controller_refusal(code: str, message: str, status_code: int) -> Response:- return JSONResponse(+def _controller_refusal(code: str, message: str, status_code: int) -> ASGIApp:+ """A controller refusal, in the contract's error envelope.++ On a WebSocket, which cannot read a refused handshake (see+ `_unauthorized`), the same refusal the agent connection sends: a `refused`+ frame carrying the code, then a close with 4000 plus the status.+ """+ response = JSONResponse({"error": {"code": code, "message": message, "retryable": False}},status_code=status_code,)+ async def refuse(scope: Scope, receive: Receive, send: Send) -> None:+ if scope["type"] != "websocket":+ await response(scope, receive, send)+ return+ await receive() # the client's websocket.connect+ await send({"type": "websocket.accept"})+ frame = {+ "event": "refused",+ "data": {+ "status": status_code,+ "detail": {"code": code, "message": message},+ },+ }+ await send({"type": "websocket.send", "text": json.dumps(frame)})+ await send({"type": "websocket.close", "code": 4000 + status_code})++ return refuse+def _is_agent_route(path: str) -> bool:if path.startswith(_AGENT_SESSIONS_PREFIX):@@ -553,13 +577,35 @@ def _path_agent(path: str) -> tuple[str | None, str]:return match["segment"], match["rest"] or ""+async def _unauthorized(+ scope: Scope, receive: Receive, send: Send, reason: str+) -> None:+ """Refuse an unauthenticated request in a form its client can read.++ An HTTP 401. A WebSocket client cannot read the status of a handshake that+ was refused (the browser-style API hides it), so it would see only a+ failed connection and retry for ever. A WebSocket is accepted and closed at+ once with 4401 instead: 4000 plus the status, the same mapping the agent+ connection uses for every refusal.+ """+ if scope["type"] == "websocket":+ await receive() # the client's websocket.connect+ await send({"type": "websocket.accept"})+ await send({"type": "websocket.close", "code": 4401, "reason": reason})+ return+ await Response(reason, status_code=401)(scope, receive, send)++def _is_public_path(path: str) -> bool:return any(path == p or path.startswith(p + "/") for p in PUBLIC_PATH_PREFIXES)-def get_agent_from_scope(request: Request) -> Agent:- """Get authenticated agent from request scope (set by middleware)."""- agent: object = request.scope.get("agent")+def get_agent_from_scope(connection: HTTPConnection) -> Agent:+ """Get authenticated agent from the connection's scope (set by middleware).++ Any HTTP connection, so a WebSocket route can use it too.+ """+ agent: object = connection.scope.get("agent")if not isinstance(agent, Agent):raise HTTPException(status_code=401, detail="Not authenticated")return agentdiff --git a/core/switch_core/bridges/agent/protocol/agent_connections.py b/core/switch_core/bridges/agent/protocol/agent_connections.pyindex e11ce9d4b..182535d25 100644--- a/core/switch_core/bridges/agent/protocol/agent_connections.py+++ b/core/switch_core/bridges/agent/protocol/agent_connections.py@@ -62,6 +62,10 @@ logger = logging.getLogger(__name__)Scope = Literal["single", "all"]DeliveryFilter = Literal["all", "addressed"]+# How a connection's stream reaches its client. `sse` is the event stream an+# agent runtime built before the WebSocket still asks for, served for a+# compatibility window and told apart so we can see those clients drain.+Transport = Literal["websocket", "sse"]# Refuse a client that cannot meet this server's protocol rather than degrading# in ways neither side can see. The runtime lives on the user's machine and@@ -405,6 +409,8 @@ class AgentConnection:# agent belongs to that no sibling has claimed.rooms: set[str] = field(default_factory=set)stream_attached: bool = False+ # What the attached stream travels over, as of the last attach.+ stream_transport: Transport = "websocket"# How many heartbeats this connection has received. Diagnostic: it is the# difference between a client that never started beating and one that beat# and then stopped, which the timestamp alone cannot tell you.@@ -505,6 +511,7 @@ class AgentConnectionRegistry:cursor: int,declaration: ClientDeclaration,expected_generation: int | None,+ transport: Transport = "websocket",) -> AgentConnection:"""Open a connection, or reattach to one the client already owns.@@ -563,18 +570,24 @@ class AgentConnectionRegistry:existing.last_beat = time.monotonic()existing.closure = Noneexisting.stream_attached = True+ existing.stream_transport = transportexisting.stream_generation = self._new_incarnation()+ # Wake the stream this one replaces, so its client hears it was+ # taken over now rather than on the stream's next idle tick.+ existing.wake.set()# A reattach can come from an upgraded client, so the declaration# is replaced rather than kept. The connection outlives the socket;# what is on the other end of it need not.existing.declaration = declarationself._reset_worker(existing)logger.info(- "[CONN] reattached agent=%s connection=%s scope=%s generation=%s",+ "[CONN] reattached agent=%s connection=%s scope=%s generation=%s "+ "transport=%s",agent_id,connection_id,scope,existing.stream_generation,+ transport,)return existing@@ -593,6 +606,7 @@ class AgentConnectionRegistry:last_beat=now,opened_at=now,stream_attached=True,+ stream_transport=transport,stream_generation=self._new_incarnation(),declaration=declaration,)@@ -600,7 +614,7 @@ class AgentConnectionRegistry:owned.add(connection_id)logger.info("[CONN] opened agent=%s connection=%s scope=%s filter=%s spawn=%s "- "client=%s version=%s protocol=%s",+ "client=%s version=%s protocol=%s transport=%s",agent_id,connection_id,scope,@@ -611,6 +625,7 @@ class AgentConnectionRegistry:f"{floor}-{declaration.speaks}"if declaration.declares_protocolelse "unknown",+ transport,)return conndiff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.pyindex 75b68192c..a872e768c 100644--- a/core/switch_core/bridges/agent/protocol/stream.py+++ b/core/switch_core/bridges/agent/protocol/stream.py@@ -1,13 +1,15 @@-"""Server-sent event stream for agent connections (CHOO-1857).+"""The frames of an agent connection (CHOO-1857).-Turns a connection plus the event buffer into a `text/event-stream`: catch-up+Turns a connection plus the event buffer into a sequence of frames: catch-upfrom the client's cursor, then live delivery as events are appended. The client-never asks again — it opens once and reads.--Every event carries its sequence number as the SSE `id`, so a client that-reconnects sends `Last-Event-ID` and resumes exactly where it stopped. Gaps are-reported as their own event rather than skipped: a client that has missed-events must never see a stream that looks complete.+never asks again; it opens once and reads. `api/handlers.py` sends them over+the WebSocket, or, for a client built before it, as a Server-Sent Events+stream: both encode the same frames from the same loop, so they cannot drift.++Every event carries its sequence number, so a client that reconnects names the+last one it processed and resumes exactly where it stopped. Gaps are reported+as their own event rather than skipped: a client that has missed events must+never see a stream that looks complete."""from __future__ import annotations@@ -16,7 +18,8 @@ import asyncioimport jsonimport loggingimport time-from collections.abc import AsyncIterator+from collections.abc import AsyncGenerator, AsyncIterator+from dataclasses import dataclassfrom typing import TYPE_CHECKING, Anyfrom switch_core.bridges.agent.protocol.agent_connections import (@@ -42,9 +45,8 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-# How long to wait for an event before writing a keepalive comment. This exists-# only to stop proxies dropping an idle connection — liveness comes from the-# client's heartbeat, never from this.+# How long the loop waits for work before yielding an idle tick, so whoever+# reads it gets a chance to do its own periodic work.KEEPALIVE_INTERVAL_SECONDS = 15.0# Cap on how many buffered events are written in one batch, so a large catch-up@@ -52,15 +54,67 @@ KEEPALIVE_INTERVAL_SECONDS = 15.0CATCH_UP_BATCH = 200-def _frame(event: str, data: dict[str, Any], *, seq: int | None = None) -> bytes:+@dataclass(frozen=True)+class Frame:+ """One message on an agent's event stream, before it is encoded.++ The loop decides what to send; the transport decides how it is encoded.+ """++ event: str+ data: dict[str, Any]+ seq: int | None = None+++# Not a message: the loop's signal that it waited a keepalive interval with+# nothing to send. The WebSocket has its own ping and drops it; the event+# stream writes a comment, which only stops proxies dropping an idle+# connection. Liveness comes from the client's beat, never from this.+KEEPALIVE = Frame("keepalive", {})+++def _frame(event: str, data: dict[str, Any], *, seq: int | None = None) -> Frame:+ return Frame(event, data, seq)+++def encode_ws(frame: Frame) -> dict[str, Any]:+ """A frame as a WebSocket message. `id` is the event's sequence number,+ which a reconnecting client passes back as `start_from`."""+ message: dict[str, Any] = {"event": frame.event, "data": frame.data}+ if frame.seq is not None:+ message["id"] = frame.seq+ return message+++def encode_sse(frame: Frame) -> bytes:+ """A frame as a Server-Sent Event. `id` is the event's sequence number,+ which a reconnecting client sends back as `Last-Event-ID`."""+ if frame is KEEPALIVE:+ return b": keepalive\n\n"lines = []- if seq is not None:- lines.append(f"id: {seq}")- lines.append(f"event: {event}")- lines.append(f"data: {json.dumps(data, separators=(',', ':'))}")+ if frame.seq is not None:+ lines.append(f"id: {frame.seq}")+ lines.append(f"event: {frame.event}")+ lines.append(f"data: {json.dumps(frame.data, separators=(',', ':'))}")return ("\n".join(lines) + "\n\n").encode()+async def sse_stream(frames: AsyncGenerator[Frame]) -> AsyncIterator[bytes]:+ """The connection's frames as a `text/event-stream` body.++ Kept for clients built before the WebSocket (agent-protocol revision 7 and+ older), for a compatibility window: it goes once no client still connects+ over it.+ """+ try:+ async for frame in frames:+ yield encode_sse(frame)+ finally:+ # Closed here, so the stream detaches from its connection when the+ # client goes rather than when the generator is garbage collected.+ await frames.aclose()++def _connection_state(conn: AgentConnection) -> dict[str, Any]:"""The first frame of every stream, and where the server declares itself.@@ -104,13 +158,14 @@ def _eviction(closure: Closure) -> dict[str, Any]:}-def event_stream(+def event_frames(*,conn: AgentConnection,registry: AgentConnectionRegistry,buffer: EventBuffer,approvals: ApprovalOutcomes | None,-) -> AsyncIterator[bytes]:+) -> AsyncGenerator[Frame]:+ """The connection's frames, for a transport to encode."""return _event_stream(conn=conn,registry=registry,@@ -127,8 +182,8 @@ async def _event_stream(buffer: EventBuffer,approvals: ApprovalOutcomes | None,generation: int,-) -> AsyncIterator[bytes]:- """Yield SSE frames for a connection until its stream is superseded or it dies."""+) -> AsyncGenerator[Frame]:+ """Yield frames for a connection until its stream is superseded or it dies."""if conn.stream_generation != generation:returnagent_id = conn.agent_id@@ -348,7 +403,7 @@ async def _event_stream(# cursor still where it started.conn.wake.clear()if not conn.rooms and not await _wait_for_wake(conn):- yield b": keepalive\n\n"+ yield KEEPALIVEcontinue# Where counting starts for a room nothing is counting yet. It is@@ -454,7 +509,7 @@ async def _event_stream(continueif not await _wait_for_work(bell, conn):- yield b": keepalive\n\n"+ yield KEEPALIVEfinally:if unsubscribe_outcomes is not None:unsubscribe_outcomes()diff --git a/core/tests/benchmarks/client.py b/core/tests/benchmarks/client.pyindex a0593b9c3..2d90fa251 100644--- a/core/tests/benchmarks/client.py+++ b/core/tests/benchmarks/client.py@@ -7,8 +7,8 @@ is proving the harness itself works — that the server serves, that a streamdelivers, and that the instrumentation points fire — without needing the wholeNode side up first.-It speaks only the part of the protocol those checks need: open the stream,-heartbeat, read frames.+It speaks only the part of the protocol those checks need: open the+connection's WebSocket, answer the server's pings, read frames."""from __future__ import annotations@@ -17,12 +17,9 @@ import asyncioimport jsonfrom dataclasses import dataclassfrom typing import Any+from urllib.parse import urlencode-import httpx--from switch_core.bridges.agent.protocol.agent_connections import (- HEARTBEAT_INTERVAL_SECONDS,-)+import websockets@dataclass(frozen=True, slots=True)@@ -33,7 +30,7 @@ class StreamFrame:class AgentConnection:- """One SSE connection, with the heartbeat the server requires to keep it."""+ """One agent connection over its WebSocket, answering the server's pings."""def __init__(self,@@ -55,9 +52,7 @@ class AgentConnection:self._spawn_capable = spawn_capableself._frames: asyncio.Queue[StreamFrame] = asyncio.Queue()self._cursor = 0- self._client: httpx.AsyncClient | None = Noneself._reader: asyncio.Task[None] | None = None- self._beater: asyncio.Task[None] | None = Noneself._opened = asyncio.Event()self._failure: BaseException | None = None@@ -66,11 +61,6 @@ class AgentConnection:return self._cursorasync def open(self, timeout: float) -> None:- self._client = httpx.AsyncClient(- base_url=self._base_url,- headers={"Authorization": f"Bearer {self._api_key}"},- timeout=httpx.Timeout(None),- )self._reader = asyncio.create_task(self._read())try:await asyncio.wait_for(self._opened.wait(), timeout)@@ -79,23 +69,17 @@ class AgentConnection:raiseif self._failure is not None:raise self._failure- self._beater = asyncio.create_task(self._beat())async def close(self) -> None:- for task in (self._beater, self._reader):- if task is not None:- task.cancel()- for task in (self._beater, self._reader):- if task is not None:- try:- await task- except asyncio.CancelledError:- pass- if self._client is not None:- await self._client.aclose()+ if self._reader is not None:+ self._reader.cancel()+ try:+ await self._reader+ except asyncio.CancelledError:+ passasync def next_frame(self, event: str, timeout: float) -> StreamFrame:- """The next frame of a given type, discarding keepalives and others.+ """The next frame of a given type, discarding pings and others.Raises on timeout rather than returning None: every caller here isasserting that something was delivered, and a None would turn a@@ -114,38 +98,44 @@ class AgentConnection:return frameasync def _read(self) -> None:- assert self._client is not None- params = {- "connection_id": self._connection_id,- "scope": self._scope,- "filter": self._filter,- "spawn_capable": str(self._spawn_capable).lower(),- }+ query = urlencode(+ {+ "connection_id": self._connection_id,+ "scope": self._scope,+ "filter": self._filter,+ "spawn_capable": str(self._spawn_capable).lower(),+ }+ )+ url = (+ self._base_url.replace("http", "ws", 1)+ + f"/agents/{self._agent_id}/connection/ws?{query}"+ )try:- async with self._client.stream(- "GET",- f"/agents/{self._agent_id}/events",- params=params,- headers={"Accept": "text/event-stream"},- ) as response:- if response.status_code != 200:- body = await response.aread()- self._failure = RuntimeError(- f"event stream refused with {response.status_code}: "- f"{body.decode(errors='replace')}"- )+ async with websockets.connect(+ url, additional_headers={"Authorization": f"Bearer {self._api_key}"}+ ) as socket:+ async for raw in socket:+ message = json.loads(raw)+ event = message.get("event")+ if event == "refused":+ self._failure = RuntimeError(+ f"connection refused: {message.get('data')}"+ )+ self._opened.set()+ returnself._opened.set()- return- self._opened.set()- block: list[str] = []- async for line in response.aiter_lines():- if line:- block.append(line)- continue- frame = _parse(block)- block = []- if frame is None:+ if event == "ping":+ # The heartbeat: the answer is what keeps the+ # connection alive.+ await socket.send(+ json.dumps({"type": "pong", "cursor": self._cursor})+ )continue+ frame = StreamFrame(+ event=event,+ sequence=message.get("id"),+ data=message.get("data") or {},+ )if frame.sequence is not None:self._cursor = frame.sequenceawait self._frames.put(frame)@@ -154,35 +144,3 @@ class AgentConnection:except Exception as exc: # noqa: BLE001 - surfaced through open()/next_frameself._failure = excself._opened.set()-- async def _beat(self) -> None:- assert self._client is not None- while True:- await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS)- response = await self._client.post(- f"/agents/{self._agent_id}/connection/beat",- json={"connection_id": self._connection_id, "cursor": self._cursor},- )- if response.status_code != 200:- raise RuntimeError(- f"heartbeat for {self._connection_id} rejected with "- f"{response.status_code}: {response.text}"- )---def _parse(block: list[str]) -> StreamFrame | None:- event = None- data = None- sequence = None- for line in block:- if line.startswith(":"):- continue- if line.startswith("id: "):- sequence = int(line[4:])- elif line.startswith("event: "):- event = line[7:]- elif line.startswith("data: "):- data = json.loads(line[6:])- if event is None or not isinstance(data, dict):- return None- return StreamFrame(event=event, sequence=sequence, data=data)diff --git a/core/tests/benchmarks/test_baseline.py b/core/tests/benchmarks/test_baseline.pyindex bd4ef1b0d..05d43493c 100644--- a/core/tests/benchmarks/test_baseline.py+++ b/core/tests/benchmarks/test_baseline.py@@ -380,7 +380,9 @@ async def test_baseline_idles_at_session_count()lines.append(f"{rooms} idle session(s) over {watched:.1f}s: "- f"watcher heartbeats {requests['agents/connection/beat']}, "+ # Heartbeats ride the connection's WebSocket now, so they are no+ # longer requests; anything left on this route is an old client.+ f"heartbeat requests {requests.get('agents/connection/beat', 0)}, "f"session host requests {reports}, "f"all requests {sum(requests.values())} ({routes}); "f"{statements} database statements; CPU driver "diff --git a/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py b/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.pyindex 501f01d20..bb5194604 100644--- a/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py+++ b/core/tests/switch_core/bridges/agent/api/test_beat_cursor_clamp.py@@ -1,10 +1,9 @@-"""A heartbeat cursor is clamped to the buffer head before anything adopts it.+"""A heartbeat's cursor is clamped to the buffer head before anything adopts it.The event buffer is in memory, so restarting switch-core resets the sequence tozero while clients keep beating the number they had reached. Nothing downstreamcan undo that: `beat()` and `confirm()` both refuse to move a cursor backwards,-and the stream's own rewind on resume is re-poisoned by the next heartbeat two-seconds later.+and the stream's own rewind on resume is re-poisoned by the next heartbeat.The result is silent. The connection is claimed, heartbeats succeed and postingworks, so the session looks healthy while every event up to the stale cursor is@@ -13,11 +12,9 @@ skipped — and `confirm()` marks those events consumed on the way past.from __future__ import annotations-from types import SimpleNamespacefrom typing import Any-from switch_core.bridges.agent.api.handlers import connection_beat-from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest+from switch_core.bridges.agent.api.handlers import _record_beatfrom switch_core.bridges.agent.protocol.agent_connections import (PROTOCOL_VERSION,AgentConnectionRegistry,@@ -67,17 +64,11 @@ def _connect(protocol: _Protocol, cursor: int = 0) -> Any:return conn-async def _beat(protocol: _Protocol, cursor: int) -> Any:- # Fenced with the only incarnation these tests ever open. A client that- # declares the current revision and then ticks without one is refused, and- # nothing here is about that.+def _beat(protocol: _Protocol, cursor: int) -> Any:+ # Fenced with the only incarnation these tests ever open, as the socket+ # does with the incarnation it opened.current = protocol.connections.require(AGENT_ID, CONN_ID).stream_generation- return await connection_beat(- AGENT_ID,- ConnectionBeatRequest(connection_id=CONN_ID, cursor=cursor, generation=current),- SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]- protocol, # type: ignore[arg-type]- )+ return _record_beat(protocol, AGENT_ID, CONN_ID, cursor, current) # type: ignore[arg-type]async def test_a_cursor_past_the_head_is_pulled_back_to_it() -> None:@@ -86,11 +77,11 @@ async def test_a_cursor_past_the_head_is_pulled_back_to_it() -> None:protocol = _Protocol()conn = _connect(protocol)- result = await _beat(protocol, cursor=9)+ beaten = _beat(protocol, cursor=9)assert protocol.event_buffer.head(AGENT_ID) == 0assert conn.cursor == 0- assert result["cursor"] == 0+ assert beaten.cursor == 0async def test_the_first_event_after_a_restart_is_still_delivered() -> None:@@ -101,7 +92,7 @@ async def test_the_first_event_after_a_restart_is_still_delivered() -> None:conn = _connect(protocol)protocol.connections.claim_room(conn, ROOM_ID, takeover=True)- await _beat(protocol, cursor=9)+ _beat(protocol, cursor=9)protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())delivered = protocol.event_buffer.read_from(AGENT_ID, conn.cursor, rooms={ROOM_ID})@@ -114,7 +105,7 @@ async def test_a_cursor_within_the_buffer_is_left_alone() -> None:for _ in range(3):protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())- await _beat(protocol, cursor=2)+ _beat(protocol, cursor=2)assert conn.cursor == 2@@ -129,7 +120,7 @@ async def test_the_clamped_value_is_what_gets_confirmed() -> None:lambda agent_id, connection_id, cursor: confirmed.append(cursor))- await _beat(protocol, cursor=9)+ _beat(protocol, cursor=9)assert confirmed == [0]@@ -139,6 +130,6 @@ async def test_a_heartbeat_from_the_old_boot_cannot_undo_the_gap_position():protocol.event_buffer = EventBuffer(sequence_base=2 << 32)sequence = protocol.event_buffer.enqueue(AGENT_ID, ROOM_ID, _message())conn = _connect(protocol, cursor=2 << 32)- await _beat(protocol, cursor=(1 << 32) + 2)+ _beat(protocol, cursor=(1 << 32) + 2)assert conn.cursor == 2 << 32assert protocol.event_buffer.read_from(AGENT_ID, conn.cursor)[0].seq == sequencediff --git a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.pyindex 48e98e3e1..ddf493796 100644--- a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py+++ b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py@@ -17,12 +17,14 @@ from typing import Anyimport pytestfrom fastapi import HTTPException-from switch_core.bridges.agent.api.handlers import connection_beat+from switch_core.bridges.agent.api.handlers import _record_beat, connection_beatfrom switch_core.bridges.agent.api.schemas import ConnectionBeatRequestfrom switch_core.bridges.agent.protocol.agent_connections import (PROTOCOL_VERSION,AgentConnectionRegistry,ClientDeclaration,+ NoStreamAttachedError,+ SupersededConnectionError,)from switch_core.bridges.agent.protocol.event_buffer import EventBuffer@@ -49,7 +51,47 @@ def _open(protocol: _Protocol, *, speaks: int | None = PROTOCOL_VERSION) -> Any:)-async def _beat(protocol: _Protocol, generation: int | None) -> Any:+def _beat(protocol: _Protocol, generation: int | None) -> Any:+ return _record_beat(protocol, AGENT_ID, CONN_ID, 0, generation) # type: ignore[arg-type]+++def test_a_superseded_tick_is_refused_as_a_takeover() -> None:+ protocol = _Protocol()+ displaced = _open(protocol).stream_generation+ _open(protocol) # the winner attaches; the incarnation bumps++ with pytest.raises(SupersededConnectionError) as caught:+ _beat(protocol, displaced)++ # The same code the displaced stream is evicted with: one ending reaching+ # the client by whichever door is still open to it.+ assert caught.value.code == "taken_over"+ # The prose still names both incarnations for whoever reads it.+ assert str(displaced) in str(caught.value)+ assert str(displaced + 1) in str(caught.value)+++def test_a_tick_with_no_stream_is_refused_as_something_to_reopen() -> None:+ protocol = _Protocol()+ conn = _open(protocol)+ protocol.connections.detach_stream(conn, conn.stream_generation)++ with pytest.raises(NoStreamAttachedError) as caught:+ _beat(protocol, conn.stream_generation)++ # Recoverable, and distinct from the takeover: this client still holds the+ # connection and only has to reconnect.+ assert caught.value.code == "no_stream"+++# ── Over HTTP, for a client on the event stream ─────────────────────────────+#+# A client built before the WebSocket beats with `POST /connection/beat`, kept+# with the event stream for a compatibility window. Its refusals are the same+# ones, answered as HTTP with the code beside the prose.+++async def _post_beat(protocol: _Protocol, generation: int | None) -> Any:return await connection_beat(AGENT_ID,ConnectionBeatRequest(connection_id=CONN_ID, cursor=0, generation=generation),@@ -58,52 +100,56 @@ async def _beat(protocol: _Protocol, generation: int | None) -> Any:)-async def test_a_superseded_tick_is_refused_as_a_takeover() -> None:+async def test_a_superseded_beat_is_answered_409_taken_over() -> None:protocol = _Protocol()displaced = _open(protocol).stream_generation- _open(protocol) # the winner attaches; the incarnation bumps+ _open(protocol)with pytest.raises(HTTPException) as caught:- await _beat(protocol, displaced)+ await _post_beat(protocol, displaced)- # The same code the displaced stream is evicted with: one ending reaching- # the client by whichever door is still open to it.assert caught.value.status_code == 409assert caught.value.detail["code"] == "taken_over" # type: ignore[index]- # The prose stays, and still names both incarnations for whoever reads it.message = caught.value.detail["message"] # type: ignore[index]assert str(displaced) in message and str(displaced + 1) in message-async def test_a_tick_with_no_stream_is_refused_as_something_to_reopen() -> None:+async def test_a_beat_with_no_stream_is_answered_409_no_stream() -> None:protocol = _Protocol()conn = _open(protocol)protocol.connections.detach_stream(conn, conn.stream_generation)with pytest.raises(HTTPException) as caught:- await _beat(protocol, conn.stream_generation)+ await _post_beat(protocol, conn.stream_generation)- # Recoverable, and distinct from the takeover: this client still holds the- # connection and only has to reopen the stream.assert caught.value.status_code == 409assert caught.value.detail["code"] == "no_stream" # type: ignore[index]-async def test_an_unfenced_tick_from_a_fenced_holder_is_refused_on_its_own_code() -> (+async def test_an_unfenced_beat_from_a_fenced_holder_is_refused_on_its_own_code() -> (None):protocol = _Protocol()_open(protocol)with pytest.raises(HTTPException) as caught:- await _beat(protocol, None)+ await _post_beat(protocol, None)assert caught.value.status_code == 409assert caught.value.detail["code"] == "unfenced" # type: ignore[index]+async def test_a_beat_for_an_unknown_connection_is_answered_404() -> None:+ protocol = _Protocol()++ with pytest.raises(HTTPException) as caught:+ await _post_beat(protocol, None)++ assert caught.value.status_code == 404++async def test_a_client_that_cannot_be_fenced_still_beats() -> None:protocol = _Protocol()_open(protocol, speaks=None)- assert (await _beat(protocol, None))["ok"] is True+ assert (await _post_beat(protocol, None))["ok"] is Truediff --git a/core/tests/switch_core/bridges/agent/api/test_connection_placements.py b/core/tests/switch_core/bridges/agent/api/test_connection_placements.pyindex 9ca4eade0..246c9d8b8 100644--- a/core/tests/switch_core/bridges/agent/api/test_connection_placements.py+++ b/core/tests/switch_core/bridges/agent/api/test_connection_placements.py@@ -10,7 +10,7 @@ from fastapi import HTTPExceptionfrom switch_core.bridges.agent.api.handlers import connection_placementsfrom switch_core.bridges.agent.api.schemas import ConnectionPlacementsRequestfrom switch_core.bridges.agent.protocol.agent_connections import (- ROOM_RELEASED_PROTOCOL_REVISION,+ PROTOCOL_VERSION,AgentConnection,AgentConnectionRegistry,ClientDeclaration,@@ -50,7 +50,7 @@ def _open(delivery_filter="all",spawn_capable=True,cursor=0,- declaration=ClientDeclaration(speaks=ROOM_RELEASED_PROTOCOL_REVISION),+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),expected_generation=None,)diff --git a/core/tests/switch_core/bridges/agent/api/test_connection_socket.py b/core/tests/switch_core/bridges/agent/api/test_connection_socket.pynew file mode 100644index 000000000..d15ff1f09--- /dev/null+++ b/core/tests/switch_core/bridges/agent/api/test_connection_socket.py@@ -0,0 +1,260 @@+"""The agent connection over one WebSocket: same frames as the event stream,+authenticated once, and a heartbeat the server drives.++Opening, fencing and room claims are shared with the SSE stream and tested+there; these cover what is particular to the socket.+"""++from __future__ import annotations++import asyncio+from collections.abc import AsyncGenerator, Iterator+from typing import Any++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient+from starlette.types import ASGIApp, Receive, Scope, Send+from starlette.websockets import WebSocket, WebSocketDisconnect++from switch_core.bridges.agent.api import handlers+from switch_core.bridges.agent.api.session_reporter import SessionReporter+from switch_core.bridges.agent.dependencies import get_config, get_protocol+from switch_core.bridges.agent.protocol.agent_connections import (+ PROTOCOL_VERSION,+ AgentConnectionRegistry,+ ClientDeclaration,+)+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload+from switch_core.db.models import Agent++AGENT_ID = "agent-1"+ROOM = "room-a"+++class _Protocol:+ def __init__(self) -> None:+ self.event_buffer = EventBuffer(sequence_base=0)+ self.connections = AgentConnectionRegistry()+ self.approval_outcomes = None+ self.sessions = SessionReporter(None)++ async def record_client_declaration(+ self, agent_id: str, connection_id: str, declaration: ClientDeclaration+ ) -> None:+ return None++ async def require_room_member(self, agent_id: str, room_id: str) -> None:+ return None+++class _AuthenticatedAs:+ """What the bearer-auth middleware leaves in scope for a valid token."""++ def __init__(self, app: ASGIApp, agent: Agent) -> None:+ self.app = app+ self.agent = agent++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ scope["agent"] = self.agent+ await self.app(scope, receive, send)+++def protocol() -> _Protocol:+ return _Protocol()+++def client(+ protocol: _Protocol, monkeypatch: pytest.MonkeyPatch+) -> Iterator[TestClient]:+ # Pings every 50 ms rather than every heartbeat interval.+ monkeypatch.setattr(handlers, "HEARTBEAT_INTERVAL_SECONDS", 0.05)+ app = FastAPI()+ app.include_router(handlers.router, prefix="/agents")+ app.dependency_overrides[get_protocol] = lambda: protocol+ app.dependency_overrides[get_config] = lambda: None+ agent = Agent(+ id=AGENT_ID, name="agent", metadata_={"known_agent_type": "claude-code"}+ )+ with TestClient(_AuthenticatedAs(app, agent)) as client: # type: ignore[arg-type]+ yield client+++def _url(**params: Any) -> str:+ query = {"connection_id": "c1", "protocol": PROTOCOL_VERSION, **params}+ return f"/agents/{AGENT_ID}/connection/ws?" + "&".join(+ f"{k}={v}" for k, v in query.items()+ )+++def _next(ws: Any, event: str) -> dict[str, Any]:+ """The next message with this event name, skipping pings."""+ for _ in range(50):+ message = ws.receive_json()+ if message["event"] == event:+ return message+ raise AssertionError(f"no {event} frame")+++def test_it_opens_like_the_event_stream(+ client: TestClient, protocol: _Protocol+) -> None:+ with client.websocket_connect(_url(scope="all")) as ws:+ state = ws.receive_json()++ assert state["event"] == "connection_state"+ assert state["data"]["connection_id"] == "c1"+ conn = protocol.connections.get("c1")+ assert conn is not None+ assert conn.stream_attached+ assert state["data"]["generation"] == conn.stream_generation+++def test_events_arrive_with_their_sequence_number(+ client: TestClient, protocol: _Protocol+) -> None:+ with client.websocket_connect(_url(rooms=ROOM)) as ws:+ ws.receive_json()+ seq = protocol.event_buffer.enqueue(+ AGENT_ID,+ ROOM,+ AgentEvent(+ type="message",+ room_id=ROOM,+ payload=MessagePayload(+ addressed=True,+ sender="@u:s",+ sender_name="u",+ message_id="$evt-1",+ body="hello",+ timestamp=0,+ ),+ ),+ )++ message = _next(ws, "message")++ assert message["id"] == seq+ assert message["data"]["payload"]["body"] == "hello"+++def test_the_server_pings_and_a_pong_counts_as_a_beat(+ client: TestClient, protocol: _Protocol+) -> None:+ with client.websocket_connect(_url()) as ws:+ ws.receive_json()+ conn = protocol.connections.get("c1")+ assert conn is not None+ beats_before = conn.beats++ _next(ws, "ping")+ ws.send_json({"type": "pong", "cursor": 0})+ _next(ws, "ping") # the pong has been handled by the next ping++ assert conn.beats > beats_before+++def test_a_refusal_is_sent_then_the_socket_closes_with_its_status(+ client: TestClient,+) -> None:+ with client.websocket_connect(_url(scope="everywhere")) as ws:+ refused = ws.receive_json()++ assert refused["event"] == "refused"+ assert refused["data"]["status"] == 400+ with pytest.raises(WebSocketDisconnect) as closed:+ ws.receive_json()+ assert closed.value.code == 4400+++def test_closing_the_socket_detaches_the_stream(+ client: TestClient, protocol: _Protocol+) -> None:+ with client.websocket_connect(_url()) as ws:+ ws.receive_json()++ conn = protocol.connections.get("c1")+ assert conn is not None+ assert not conn.stream_attached+++def test_a_socket_whose_connection_is_taken_over_is_told_so(+ client: TestClient, protocol: _Protocol+) -> None:+ """Final for the client that lost it: reconnecting would take the+ connection straight back off the winner."""+ with client.websocket_connect(_url()) as loser:+ loser.receive_json()+ with client.websocket_connect(_url()) as winner:+ winner.receive_json()++ evicted = _next(loser, "evicted")++ assert evicted["data"]["code"] == "taken_over"+++def test_a_socket_the_server_closed_under_it_ends_cleanly(+ client: TestClient, protocol: _Protocol, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Uvicorn closes sockets itself on shutdown, and the next send raises+ RuntimeError rather than WebSocketDisconnect."""+ send_json = WebSocket.send_json++ async def closed_on_ping(self: WebSocket, data: Any, mode: str = "text") -> None:+ if data.get("event") == "ping":+ raise RuntimeError("Unexpected ASGI message 'websocket.send'")+ await send_json(self, data, mode) # type: ignore[arg-type]++ monkeypatch.setattr(WebSocket, "send_json", closed_on_ping)+ with client.websocket_connect(_url()) as ws:+ ws.receive_json()+ with pytest.raises(WebSocketDisconnect):+ ws.receive_json()++ conn = protocol.connections.get("c1")+ assert conn is not None+ assert not conn.stream_attached+++def test_a_stream_that_fails_closes_the_socket_as_an_error(+ client: TestClient, monkeypatch: pytest.MonkeyPatch, caplog: Any+) -> None:+ async def failing(**kwargs: Any) -> AsyncGenerator[Any]:+ raise RuntimeError("approval resync failed")+ yield++ monkeypatch.setattr(handlers, "event_frames", failing)+ with client.websocket_connect(_url()) as ws:+ with pytest.raises(WebSocketDisconnect) as closed:+ _next(ws, "connection_state")++ assert closed.value.code == 1011+ assert "delivery failed" in caplog.text+++async def test_a_cancelled_pump_closes_its_stream() -> None:+ """Cancelled while the outbox is full, the pump is not inside the stream,+ so only closing it runs the stream's cleanup there and then."""+ closed = asyncio.Event()++ async def frames() -> AsyncGenerator[Any]:+ try:+ while True:+ yield "frame"+ finally:+ closed.set()++ stream = frames()+ outbox: asyncio.Queue[Any] = asyncio.Queue(maxsize=1)+ pump = asyncio.create_task(handlers._pump_frames(stream, outbox))+ while not outbox.full():+ await asyncio.sleep(0)++ pump.cancel()+ with pytest.raises(asyncio.CancelledError):+ await pump++ assert closed.is_set()diff --git a/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py b/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.pynew file mode 100644index 000000000..8e7ed15e1--- /dev/null+++ b/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py@@ -0,0 +1,256 @@+"""An old client keeps working against this server.++Agent runtimes up to 0.8.x connect over the Server-Sent Events stream and beat+with `POST /connection/beat`, declaring agent-protocol 6 (0.7.x) or 7 (0.8.0,+which added the hosted worker).+The WebSocket replaces both, and both are kept beside it for a compatibility+window (the expand half of expand/contract), so these drive the server the way+such a client does: open the stream, read events, beat, and lapse when it stops+beating. The stream encodes the same frames the socket sends, from the same+loop, so the rules themselves are tested once, on the frames.+"""++from __future__ import annotations++import asyncio+import json+import time+from collections.abc import AsyncIterator+from types import SimpleNamespace+from typing import Any++import pytest+from fastapi import HTTPException+from starlette.responses import StreamingResponse++from switch_core.artifacts import contract_range+from switch_core.bridges.agent.api.handlers import connection_beat, poll_events+from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest+from switch_core.bridges.agent.api.session_reporter import SessionReporter+from switch_core.bridges.agent.protocol.agent_connections import (+ HEARTBEAT_TTL_SECONDS,+ AgentConnectionRegistry,+ ClientDeclaration,+)+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload++AGENT_ID = "agent-1"+ROOM = "room-a"+# What agent-runtime 0.7.x declares: agent-protocol 6, accepting 1.+OLD_RUNTIME = {+ "protocol_version": 6,+ "protocol_accepts": 1,+ "client": "agent-runtime",+ "client_version": "0.7.2",+}+++class _Protocol:+ def __init__(self) -> None:+ self.event_buffer = EventBuffer(sequence_base=0)+ self.connections = AgentConnectionRegistry()+ self.approval_outcomes = None+ self.sessions = SessionReporter(None)++ async def record_client_declaration(+ self, agent_id: str, connection_id: str, declaration: ClientDeclaration+ ) -> None:+ return None++ async def require_room_member(self, agent_id: str, room_id: str) -> None:+ return None+++def _agent() -> Any:+ return SimpleNamespace(id=AGENT_ID, metadata_={"known_agent_type": "claude-code"})+++def _message(body: str) -> AgentEvent:+ return AgentEvent(+ type="message",+ room_id=ROOM,+ payload=MessagePayload(+ addressed=True,+ sender="@u:s",+ sender_name="u",+ message_id=f"${body}",+ body=body,+ timestamp=0,+ ),+ )+++async def _open(protocol: _Protocol, **kw: Any) -> StreamingResponse:+ params: dict[str, Any] = {+ "accept": "text/event-stream",+ "connection_id": "c1",+ "rooms": ROOM,+ **OLD_RUNTIME,+ **kw,+ }+ resp = await poll_events(+ agent_id=AGENT_ID,+ agent=_agent(),+ protocol=protocol, # type: ignore[arg-type]+ config=None, # type: ignore[arg-type]+ **params,+ )+ assert isinstance(resp, StreamingResponse)+ return resp+++class _Reader:+ """Reads a response body the way the old runtime's `readSse` did."""++ def __init__(self, body: AsyncIterator[Any]) -> None:+ self.body = body+ self.buffered = ""++ async def frame(self, timeout: float = 2.0) -> tuple[str, str | None, dict]:+ while "\n\n" not in self.buffered:+ chunk = await asyncio.wait_for(anext(self.body), timeout)+ self.buffered += chunk.decode() if isinstance(chunk, bytes) else chunk+ raw, self.buffered = self.buffered.split("\n\n", 1)+ event, seq, data = "message", None, None+ for line in raw.split("\n"):+ if line.startswith(":"):+ continue+ if line.startswith("event: "):+ event = line[len("event: ") :]+ elif line.startswith("id: "):+ seq = line[len("id: ") :]+ elif line.startswith("data: "):+ data = line[len("data: ") :]+ if data is None:+ return await self.frame(timeout)+ return event, seq, json.loads(data)++ async def close(self) -> None:+ await self.body.aclose() # type: ignore[attr-defined]+++async def _beat(protocol: _Protocol, cursor: int, generation: int | None) -> Any:+ return await connection_beat(+ AGENT_ID,+ ConnectionBeatRequest(connection_id="c1", cursor=cursor, generation=generation),+ SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]+ protocol, # type: ignore[arg-type]+ )+++async def test_an_old_client_connects_receives_beats_and_lapses() -> None:+ protocol = _Protocol()+ stream = _Reader((await _open(protocol)).body_iterator)++ event, _, state = await stream.frame()+ assert event == "connection_state"+ assert state["rooms"] == [ROOM]+ generation = state["generation"]++ seq = protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("hello"))+ event, sent_seq, data = await stream.frame()+ assert event == "message"+ assert sent_seq == str(seq)+ assert data["payload"]["body"] == "hello"++ beaten = await _beat(protocol, seq, generation)+ assert beaten == {"ok": True, "rooms": [ROOM], "cursor": seq}++ conn = protocol.connections.get("c1")+ assert conn is not None+ conn.last_beat = time.monotonic() - (HEARTBEAT_TTL_SECONDS + 1)+ protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("not after a lapse"))++ event, _, data = await stream.frame()+ assert event == "evicted"+ assert data["code"] == "heartbeat_lapsed"+ await stream.close()++ with pytest.raises(HTTPException) as caught:+ await _beat(protocol, seq, generation)+ assert caught.value.status_code == 404+++async def test_an_old_client_resumes_from_last_event_id() -> None:+ protocol = _Protocol()+ first = protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("seen"))+ protocol.event_buffer.enqueue(AGENT_ID, ROOM, _message("missed"))++ # The old runtime sent the cursor both ways; the header wins.+ stream = _Reader(+ (+ await _open(protocol, start_from="head", last_event_id=str(first))+ ).body_iterator+ )+ await stream.frame()+ event, _, data = await stream.frame()+ await stream.close()++ assert event == "message"+ assert data["payload"]["body"] == "missed"+++async def test_closing_the_stream_detaches_it_and_keeps_the_connection() -> None:+ protocol = _Protocol()+ stream = _Reader((await _open(protocol)).body_iterator)+ await stream.frame()++ await stream.close()++ conn = protocol.connections.get("c1")+ assert conn is not None+ assert not conn.stream_attached+++# ── The declared ranges still meet ──────────────────────────────────────────+++def _overlap(a: tuple[int, int], b: tuple[int, int]) -> bool:+ """Whether two (speaks, accepts) ranges share a revision."""+ return a[1] <= b[0] and b[1] <= a[0]+++# What the releases before the socket declare, on either side: agent-protocol+# 6 (agent-runtime 0.7.x, and switch-core before the hosted worker) and 7+# (agent-runtime 0.8.0, and switch-core with it).+BEFORE_THE_SOCKET = [(6, 1), (7, 1)]++[email protected]("old", BEFORE_THE_SOCKET)+def test_an_old_runtime_and_this_server_share_a_revision(old: tuple[int, int]) -> None:+ server = contract_range("agent-protocol", "switch-core")++ assert _overlap(old, (server.speaks, server.accepts))++[email protected]("old", BEFORE_THE_SOCKET)+def test_this_runtime_and_an_old_server_share_a_revision(old: tuple[int, int]) -> None:+ runtime = contract_range("agent-protocol", "agent-runtime")++ assert _overlap((runtime.speaks, runtime.accepts), old)+++def test_this_runtime_and_this_server_share_the_newest_revision() -> None:+ server = contract_range("agent-protocol", "switch-core")+ runtime = contract_range("agent-protocol", "agent-runtime")++ assert runtime.speaks == server.speaks++[email protected]("speaks", [6, 7])+def test_this_server_admits_an_old_runtime(speaks: int) -> None:+ registry = AgentConnectionRegistry()+ conn = registry.open(+ agent_id=AGENT_ID,+ connection_id="c1",+ scope="single",+ delivery_filter="all",+ spawn_capable=False,+ cursor=0,+ declaration=ClientDeclaration(speaks=speaks, accepts=1),+ expected_generation=None,+ transport="sse",+ )++ assert conn.stream_transport == "sse"diff --git a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.pyindex 4744a0332..dfedf3c7b 100644--- a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py+++ b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py@@ -1,8 +1,9 @@-"""The /events endpoint serves both transports off one path (CHOO-1857).+"""Opening an agent connection, and the /events endpoint.-`Accept: text/event-stream` opens a connection and streams; anything else gets-the long poll. Both read the same buffer, so the two cannot diverge while they-coexist — but the dispatch itself is easy to break silently, hence these.+The WebSocket and the event stream both open through `_open_connection`,+which these drive directly: validation, protocol ranges, declarations, room+claims and session reporting. `/events` with `Accept: text/event-stream` opens+the event stream an old client still asks for; anything else long polls."""from __future__ import annotations@@ -16,7 +17,11 @@ from fastapi import HTTPExceptionfrom starlette.responses import StreamingResponsefrom switch_core.bridges.agent.api import session_reporter-from switch_core.bridges.agent.api.handlers import _resolve_start_cursor, poll_events+from switch_core.bridges.agent.api.handlers import (+ _open_connection,+ _resolve_start_cursor,+ poll_events,+)from switch_core.bridges.agent.api.session_reporter import SessionReporterfrom switch_core.bridges.agent.protocol.agent_connections import (HEARTBEAT_LAPSED,@@ -90,10 +95,31 @@ async def _call(protocol: _Protocol, **kw: Any) -> Any:"client": None,"client_version": None,"rooms": None,- "last_event_id": None,}params.update(kw)- return await poll_events(**params)+ if params.pop("accept") != "text/event-stream":+ return await poll_events(**params, accept=None)+ # What used to open the stream now opens the connection the WebSocket+ # carries; the rules are the same ones.+ conn, _frames = await _open_connection(+ config=None, # type: ignore[arg-type]+ agent=params["agent"],+ protocol=params["protocol"],+ connection_id=params["connection_id"],+ scope=params["scope"],+ event_filter=params["event_filter"],+ start_from=params["start_from"],+ spawn_capable=params["spawn_capable"],+ declaration=ClientDeclaration(+ speaks=params["protocol_version"],+ accepts=params["protocol_accepts"],+ artifact=params["client"],+ version=params["client_version"],+ ),+ rooms=params["rooms"],+ expected_generation=params.get("expected_generation"),+ )+ return connasync def test_without_the_sse_accept_header_it_long_polls() -> None:@@ -104,10 +130,18 @@ async def test_without_the_sse_accept_header_it_long_polls() -> None:assert resp.status_code == 204-async def test_with_the_sse_accept_header_it_opens_a_connection() -> None:+async def test_with_the_sse_accept_header_it_opens_the_event_stream() -> None:+ """How a runtime built before the WebSocket connects, kept for a+ compatibility window."""protocol = _Protocol()- resp = await _call(- protocol, accept="text/event-stream", connection_id="c1", scope="all"+ resp = await poll_events(+ agent_id=AGENT_ID,+ agent=_agent(),+ protocol=protocol, # type: ignore[arg-type]+ config=None, # type: ignore[arg-type]+ accept="text/event-stream",+ connection_id="c1",+ scope="all",)assert isinstance(resp, StreamingResponse)@@ -120,6 +154,19 @@ async def test_with_the_sse_accept_header_it_opens_a_connection() -> None:conn = protocol.connections.get("c1")assert conn is not Noneassert conn.scope == "all"+ assert conn.stream_transport == "sse"+++async def test_opening_registers_the_connection() -> None:+ protocol = _Protocol()+ conn = await _call(+ protocol, accept="text/event-stream", connection_id="c1", scope="all"+ )++ assert protocol.connections.get("c1") is conn+ assert conn.scope == "all"+ assert conn.stream_attached+ assert conn.stream_transport == "websocket"async def test_streaming_without_a_connection_id_is_refused() -> None:@@ -278,7 +325,7 @@ async def test_an_older_client_inside_the_server_range_is_accepted() -> None:def test_head_means_only_what_happens_next() -> None:protocol = _Protocol()protocol.event_buffer.enqueue(AGENT_ID, "room", _event())- assert _resolve_start_cursor(protocol, AGENT_ID, "head", None) == 1+ assert _resolve_start_cursor(protocol, AGENT_ID, "head") == 1def test_last_event_id_wins_over_start_from() -> None:@@ -288,13 +335,13 @@ def test_last_event_id_wins_over_start_from() -> None:def test_explicit_start_from_is_honoured() -> None:protocol = _Protocol()- assert _resolve_start_cursor(protocol, AGENT_ID, "17", None) == 17+ assert _resolve_start_cursor(protocol, AGENT_ID, "17") == 17def test_a_nonsense_cursor_is_refused_rather_than_guessed() -> None:protocol = _Protocol()with pytest.raises(HTTPException) as excinfo:- _resolve_start_cursor(protocol, AGENT_ID, "banana", None)+ _resolve_start_cursor(protocol, AGENT_ID, "banana")assert excinfo.value.status_code == 400@@ -341,14 +388,13 @@ class TestDeclaringARoomAtOpenTakesOver:)protocol.connections.claim_room(incumbent, "room-1")- resp = await _call(+ await _call(protocol,accept="text/event-stream",connection_id="supervisor",rooms="room-1",)- assert isinstance(resp, StreamingResponse)claimant = protocol.connections.claimant_of(AGENT_ID, "room-1")assert claimant is not Noneassert claimant.id == "supervisor"@@ -367,12 +413,20 @@ async def test_reconnect_during_bookkeeping_cannot_detach_the_new_stream(entered.set()await release.wait()+ async def open_stream() -> Any:+ return await poll_events(+ agent_id=AGENT_ID,+ agent=_agent(),+ protocol=protocol, # type: ignore[arg-type]+ config=None, # type: ignore[arg-type]+ accept="text/event-stream",+ connection_id="c1",+ )+monkeypatch.setattr(protocol, "record_client_declaration", record)- first = asyncio.create_task(- _call(protocol, accept="text/event-stream", connection_id="c1")- )+ first = asyncio.create_task(open_stream())await entered.wait()- newer = await _call(protocol, accept="text/event-stream", connection_id="c1")+ newer = await open_stream()await anext(newer.body_iterator)release.set()older = await firstdiff --git a/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py b/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.pyindex 8759b71d7..356332f42 100644--- a/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py+++ b/core/tests/switch_core/bridges/agent/api/test_reattach_refusal.py@@ -17,7 +17,7 @@ from typing import Anyimport pytestfrom fastapi import HTTPException-from switch_core.bridges.agent.api.handlers import poll_events+from switch_core.bridges.agent.api.handlers import _open_connectionfrom switch_core.bridges.agent.api.session_reporter import SessionReporterfrom switch_core.bridges.agent.protocol.agent_connections import (PROTOCOL_VERSION,@@ -62,16 +62,20 @@ def _attach(protocol: _Protocol) -> Any:async def _reopen(protocol: _Protocol, expected_generation: int | None) -> Any:- return await poll_events(- AGENT_ID,- SimpleNamespace(id=AGENT_ID, metadata_={}), # type: ignore[arg-type]- protocol, # type: ignore[arg-type]- None, # type: ignore[arg-type]- accept="text/event-stream",+ conn, _frames = await _open_connection(+ config=None, # type: ignore[arg-type]+ agent=SimpleNamespace(id=AGENT_ID, metadata_={}), # type: ignore[arg-type]+ protocol=protocol, # type: ignore[arg-type]connection_id=CONN_ID,- protocol_version=PROTOCOL_VERSION,+ scope="single",+ event_filter="all",+ start_from="head",+ spawn_capable=False,+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),+ rooms=None,expected_generation=expected_generation,)+ return connasync def test_a_reattach_claiming_a_superseded_incarnation_is_refused() -> None:diff --git a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py b/core/tests/switch_core/bridges/agent/protocol/test_room_released.pyindex 18a2d749b..3c98bd96d 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_room_released.py@@ -3,7 +3,6 @@from __future__ import annotationsimport asyncio-import jsonfrom typing import Anyimport pytest@@ -11,6 +10,7 @@ import pytestfrom switch_core.bridges.agent.protocol.agent_connections import (HEARTBEAT_LAPSED,LAUNCH_SUPERSEDED,+ PROTOCOL_VERSION,ROOM_RELEASED_PROTOCOL_REVISION,AgentConnection,AgentConnectionRegistry,@@ -20,7 +20,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (from switch_core.bridges.agent.protocol.event_buffer import EventBufferfrom switch_core.bridges.agent.protocol.hosted_workers import WorkerBindingfrom switch_core.bridges.agent.protocol.presence import agents_present_in-from switch_core.bridges.agent.protocol.stream import event_stream+from switch_core.bridges.agent.protocol.stream import KEEPALIVE, event_framesAGENT = "agent-1"ROOM_A = "room-a"@@ -54,15 +54,9 @@ async def _frames_until_quiet(async def pump() -> None:async for frame in stream:- if frame.startswith(b":"):+ if frame is KEEPALIVE:continue- name, data = "", "{}"- for line in frame.decode().strip().splitlines():- if line.startswith("event: "):- name = line[len("event: ") :]- elif line.startswith("data: "):- data = line[len("data: ") :]- out.append((name, json.loads(data)))+ out.append((frame.event, frame.data))try:await asyncio.wait_for(pump(), timeout=timeout)@@ -73,12 +67,8 @@ async def _frames_until_quiet(def test_a_claim_taken_over_is_released_to_its_holder_with_its_session() -> None:registry = AgentConnectionRegistry()- loser = _open(- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )- winner = _open(- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")registry.claim_room(loser, ROOM_A)registry.place_session(AGENT, "session-1", ROOM_A, loser.id)@@ -91,12 +81,8 @@ def test_a_claim_taken_over_is_released_to_its_holder_with_its_session() -> Nonedef test_connect_to_room_placement_releases_the_other_connections_session() -> None:"""`connect_to_room` places before it claims; the frame names the session once."""registry = AgentConnectionRegistry()- loser = _open(- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )- winner = _open(- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")registry.claim_room(loser, ROOM_A)registry.place_session(AGENT, "session-old", ROOM_A, loser.id)@@ -108,9 +94,7 @@ def test_connect_to_room_placement_releases_the_other_connections_session() -> Ndef test_displacing_a_sibling_on_the_same_connection_releases_nothing() -> None:registry = AgentConnectionRegistry()- conn = _open(- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")registry.place_session(AGENT, "session-old", ROOM_A, conn.id)registry.place_session(AGENT, "session-new", ROOM_A, conn.id)@@ -120,11 +104,11 @@ def test_displacing_a_sibling_on_the_same_connection_releases_nothing() -> None:@pytest.mark.parametrize("speaks", [ROOM_RELEASED_PROTOCOL_REVISION - 1, None])def test_a_client_that_cannot_take_the_frame_is_not_sent_it(speaks: int | None) -> None:+ """A client from before the frame, or one that declared no revision, gets+ only what every client takes."""registry = AgentConnectionRegistry()loser = _open(registry, "loser", speaks=speaks, scope="single")- winner = _open(- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")registry.claim_room(loser, ROOM_A)registry.claim_room(winner, ROOM_A, takeover=True)@@ -135,12 +119,8 @@ def test_a_client_that_cannot_take_the_frame_is_not_sent_it(speaks: int | None)def test_taking_the_room_back_withdraws_the_unsent_release() -> None:registry = AgentConnectionRegistry()- first = _open(- registry, "first", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )- second = _open(- registry, "second", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ first = _open(registry, "first", speaks=PROTOCOL_VERSION, scope="all")+ second = _open(registry, "second", speaks=PROTOCOL_VERSION, scope="all")registry.claim_room(first, ROOM_A)registry.claim_room(second, ROOM_A, takeover=True)@@ -152,9 +132,7 @@ def test_taking_the_room_back_withdraws_the_unsent_release() -> None:def test_replacing_placements_is_a_full_replacement() -> None:registry = AgentConnectionRegistry()- conn = _open(- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(conn, {"session-1": ROOM_A, "session-2": ROOM_B})released = registry.replace_placements(conn, {"session-2": ROOM_C})@@ -167,10 +145,8 @@ def test_replacing_placements_is_a_full_replacement() -> None:def test_replacing_placements_leaves_other_connections_alone() -> None:registry = AgentConnectionRegistry()- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")- theirs = _open(- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(theirs, {"session-t": ROOM_B})registry.replace_placements(mine, {"session-m": ROOM_A})@@ -184,10 +160,8 @@ def test_replacing_placements_leaves_other_connections_alone() -> None:def test_replacing_placements_takes_a_room_over_and_releases_it() -> None:registry = AgentConnectionRegistry()- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")- theirs = _open(- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(theirs, {"session-t": ROOM_A, "session-u": ROOM_B})released = registry.replace_placements(mine, {"session-m": ROOM_A})@@ -203,10 +177,8 @@ def test_replacing_placements_takes_a_room_over_and_releases_it() -> None:def test_a_room_claimed_without_a_placement_is_released_with_no_session() -> None:registry = AgentConnectionRegistry()- mine = _open(registry, "mine", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")- theirs = _open(- registry, "theirs", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="single"- )+ mine = _open(registry, "mine", speaks=PROTOCOL_VERSION, scope="all")+ theirs = _open(registry, "theirs", speaks=PROTOCOL_VERSION, scope="single")registry.claim_room(theirs, ROOM_A)released = registry.replace_placements(mine, {"session-m": ROOM_A})@@ -219,9 +191,7 @@ def test_a_room_claimed_without_a_placement_is_released_with_no_session() -> Nondef test_two_sessions_in_one_room_are_refused_without_change() -> None:registry = AgentConnectionRegistry()- conn = _open(- registry, "watcher", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ conn = _open(registry, "watcher", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(conn, {"session-1": ROOM_B})with pytest.raises(ValueError, match=ROOM_A):@@ -234,14 +204,10 @@ def test_two_sessions_in_one_room_are_refused_without_change() -> None:async def test_the_displaced_stream_is_sent_room_released() -> None:registry = AgentConnectionRegistry()buffer = EventBuffer(sequence_base=0)- loser = _open(- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )- winner = _open(- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ loser = _open(registry, "loser", speaks=PROTOCOL_VERSION, scope="all")+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(loser, {"session-1": ROOM_A})- stream = event_stream(conn=loser, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=loser, registry=registry, buffer=buffer, approvals=None)try:await anext(stream) # connection_statewaiting = asyncio.ensure_future(_frames_until_quiet(stream))@@ -255,17 +221,13 @@ async def test_the_displaced_stream_is_sent_room_released() -> None:assert [name for name, _ in frames].count("room_released") == 1-async def test_an_older_client_stream_is_not_sent_room_released() -> None:+async def test_a_client_that_declared_nothing_is_not_sent_room_released() -> None:registry = AgentConnectionRegistry()buffer = EventBuffer(sequence_base=0)- loser = _open(- registry, "loser", speaks=ROOM_RELEASED_PROTOCOL_REVISION - 1, scope="all"- )- winner = _open(- registry, "winner", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all"- )+ loser = _open(registry, "loser", speaks=None, scope="all")+ winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")registry.replace_placements(loser, {"session-1": ROOM_A})- stream = event_stream(conn=loser, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=loser, registry=registry, buffer=buffer, approvals=None)try:await anext(stream) # connection_statewaiting = asyncio.ensure_future(_frames_until_quiet(stream))diff --git a/core/tests/switch_core/bridges/agent/protocol/test_stream.py b/core/tests/switch_core/bridges/agent/protocol/test_stream.pyindex 03b2a3422..79a4eb78e 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_stream.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_stream.py@@ -3,7 +3,6 @@from __future__ import annotationsimport asyncio-import jsonfrom typing import Anyimport pytest@@ -20,7 +19,14 @@ from switch_core.bridges.agent.protocol.agent_connections import (Closure,)from switch_core.bridges.agent.protocol.event_buffer import EventBuffer-from switch_core.bridges.agent.protocol.stream import event_stream+from switch_core.bridges.agent.protocol.stream import (+ KEEPALIVE,+ Frame,+ encode_sse,+ encode_ws,+ event_frames,+ sse_stream,+)from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayloadfrom switch_core.observability.metrics import MetricsRegistry, install, uninstall@@ -44,17 +50,9 @@ def _message(body: str, *, addressed: bool = False, room: str = ROOM_A) -> Agent)-def _parse(frame: bytes) -> tuple[str, dict[str, Any]]:- """Split one SSE frame into (event name, data)."""- text = frame.decode()- name = ""- data = "{}"- for line in text.strip().splitlines():- if line.startswith("event: "):- name = line[len("event: ") :]- elif line.startswith("data: "):- data = line[len("data: ") :]- return name, json.loads(data)+def _parse(frame: Frame) -> tuple[str, dict[str, Any]]:+ """A frame as (event name, data)."""+ return frame.event, frame.dataasync def _take(stream, count: int, timeout: float = 2.0) -> list[tuple[str, dict]]:@@ -63,7 +61,7 @@ async def _take(stream, count: int, timeout: float = 2.0) -> list[tuple[str, dicasync def pump() -> None:async for frame in stream:- if frame.startswith(b":"):+ if frame is KEEPALIVE:continueout.append(_parse(frame))if len(out) >= count:@@ -94,7 +92,7 @@ async def test_first_frame_is_the_connection_state() -> None:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)((name, data),) = await _take(stream, 1)assert name == "connection_state"@@ -118,7 +116,7 @@ async def test_the_first_frame_declares_the_server(monkeypatch) -> None:buffer = EventBuffer(sequence_base=0)conn = _open(registry)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)((_, data),) = await _take(stream, 1)assert data["server"] == {@@ -141,7 +139,7 @@ async def test_an_unreadable_server_version_is_null_not_a_placeholder(buffer = EventBuffer(sequence_base=0)conn = _open(registry)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)((_, data),) = await _take(stream, 1)assert data["server"]["version"] is None@@ -165,7 +163,7 @@ async def test_the_first_frame_echoes_what_the_client_declared() -> None:),)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)((_, data),) = await _take(stream, 1)assert data["client"] == {@@ -181,7 +179,7 @@ async def test_an_undeclared_client_is_echoed_as_all_null() -> None:buffer = EventBuffer(sequence_base=0)conn = _open(registry, declaration=ClientDeclaration())- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)((_, data),) = await _take(stream, 1)assert data["client"] == {@@ -201,7 +199,7 @@ async def test_catch_up_then_live_delivery() -> None:# Queued before the stream opened: must be caught up, not skipped.buffer.enqueue(AGENT, ROOM_A, _message("before"))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "message"assert frames[1][1]["payload"]["body"] == "before"@@ -225,7 +223,7 @@ async def test_sequence_number_is_carried_for_resume() -> None:registry.claim_room(conn, ROOM_A)seq = buffer.enqueue(AGENT, ROOM_A, _message("one"))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][1]["sequence"] == seq@@ -241,7 +239,7 @@ async def test_resuming_from_a_cursor_skips_what_was_already_seen() -> None:conn = _open(registry, cursor=first)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][1]["payload"]["body"] == "missed"@@ -256,7 +254,7 @@ async def test_expired_cursor_produces_a_gap_event_rather_than_silence() -> Noneconn = _open(registry, cursor=0)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "gap"@@ -272,7 +270,7 @@ async def test_events_for_rooms_the_connection_does_not_cover_are_skipped() -> Nbuffer.enqueue(AGENT, ROOM_B, _message("elsewhere", room=ROOM_B))buffer.enqueue(AGENT, ROOM_A, _message("mine"))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][1]["payload"]["body"] == "mine"@@ -286,7 +284,7 @@ async def test_addressed_filter_drops_ambient_chatter() -> None:buffer.enqueue(AGENT, ROOM_A, _message("chatter"))buffer.enqueue(AGENT, ROOM_B, _message("wanted", addressed=True, room=ROOM_B))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][1]["payload"]["body"] == "wanted"@@ -298,7 +296,7 @@ async def test_a_superseded_stream_is_told_it_was_evicted() -> None:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)# A second stream attaches to the same connection id.@@ -318,7 +316,7 @@ async def test_closing_the_connection_ends_the_stream_with_a_reason() -> None:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)registry.close(conn.id, HEARTBEAT_LAPSED)@@ -337,7 +335,7 @@ async def test_a_close_about_a_room_names_the_room_it_was_about() -> None:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)registry.close(@@ -357,7 +355,7 @@ async def test_subscription_change_is_announced() -> None:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)registry.claim_room(conn, ROOM_B)@@ -378,7 +376,7 @@ async def test_stream_detaches_on_exit_without_killing_the_connection() -> None:buffer = EventBuffer(sequence_base=0)conn = _open(registry)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)await stream.aclose()@@ -419,8 +417,8 @@ async def test_two_connections_receive_the_same_event(scope: str) -> None:buffer.enqueue(AGENT, ROOM_A, _message("shared"))buffer.enqueue("agent-2", ROOM_A, _message("shared"))- a = event_stream(conn=session, registry=registry, buffer=buffer, approvals=None)- b = event_stream(conn=other_agent, registry=registry, buffer=buffer, approvals=None)+ a = event_frames(conn=session, registry=registry, buffer=buffer, approvals=None)+ b = event_frames(conn=other_agent, registry=registry, buffer=buffer, approvals=None)assert (await _take(a, 2))[1][1]["payload"]["body"] == "shared"assert (await _take(b, 2))[1][1]["payload"]["body"] == "shared"@@ -444,7 +442,7 @@ async def test_resume_replays_buffered_events_for_a_room_claimed_at_open() -> No# Claimed before the stream is created, as the endpoint does.registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "message"@@ -461,7 +459,7 @@ async def test_events_for_uncovered_rooms_do_not_block_the_cursor() -> None:buffer.enqueue(AGENT, ROOM_B, _message("someone else's room", room=ROOM_B))last = buffer.enqueue(AGENT, ROOM_A, _message("mine"))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][1]["payload"]["body"] == "mine"@@ -482,7 +480,7 @@ async def test_a_cursor_from_before_a_restart_is_reported_not_ignored() -> None:conn = _open(registry, cursor=4812) # from a previous liferegistry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "gap"@@ -501,7 +499,7 @@ async def test_stream_reports_restart_below_floor() -> None:conn = _open(registry, cursor=old_cursor)registry.claim_room(conn, ROOM_A)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 3)kind, gap = frames[1]@@ -535,7 +533,7 @@ class TestALapsedHeartbeatStopsDelivery:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)# Consume the opening frame while still healthy.@@ -559,7 +557,7 @@ class TestALapsedHeartbeatStopsDelivery:conn = _open(registry)registry.claim_room(conn, ROOM_A)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)@@ -600,7 +598,7 @@ class TestFilteredEventsDoNotSpinTheLoop:# milliseconds. A spinning generator never yields one at any interval.monkeypatch.setattr(stream_module, "KEEPALIVE_INTERVAL_SECONDS", 0.05)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1) # connection_state@@ -621,7 +619,7 @@ class TestFilteredEventsDoNotSpinTheLoop:buffer.enqueue(AGENT, ROOM_A, _message("chatter", addressed=False))buffer.enqueue(AGENT, ROOM_A, _message("for-you", addressed=True))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -656,7 +654,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:buffer.enqueue(AGENT, ROOM_A, _message("the trigger", addressed=True))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1) # connection_state@@ -676,7 +674,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:buffer.enqueue(AGENT, ROOM_A, _message("the trigger", addressed=True))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)@@ -699,7 +697,7 @@ class TestAConnectionWithNoRoomDoesNotConsume:buffer.enqueue(AGENT, ROOM_A, _message("hello", addressed=True))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -712,7 +710,7 @@ async def test_replaced_stream_cannot_capture_its_successors_generation() -> Nonregistry = AgentConnectionRegistry()buffer = EventBuffer(sequence_base=0)conn = _open(registry)- old = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ old = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)_open(registry)assert await _take(old, 1) == []assert registry.beat(AGENT, conn.id, 0, conn.stream_generation).stream_attached@@ -722,7 +720,7 @@ async def test_closed_stream_cannot_detach_a_recreated_connection() -> None:registry = AgentConnectionRegistry()buffer = EventBuffer(sequence_base=0)conn = _open(registry)- old = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ old = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await anext(old)registry.close(conn.id, HEARTBEAT_LAPSED)replacement = _open(registry)@@ -761,7 +759,7 @@ async def test_a_resumed_stream_does_not_replay_a_room_the_agent_was_removed_fro# The supervisor comes back and resumes from before both events.conn = _open(registry, connection_id="c-resumed", scope="all", cursor=0)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "message"@@ -792,7 +790,7 @@ async def test_a_last_event_id_reconnect_does_not_replay_a_removed_room() -> Non# while it was still in room A, resolved to exactly this by# `_resolve_start_cursor`.conn = _open(registry, connection_id="c-reconnect", scope="all", cursor=seen)- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)assert frames[1][0] == "message"@@ -821,7 +819,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:buffer.enqueue(AGENT, ROOM_A, _message("for you", addressed=True))buffer.enqueue(AGENT, ROOM_B, _message("also you", addressed=True, room=ROOM_B))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 3)@@ -842,7 +840,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:registry.claim_room(conn, ROOM_A)chatter = buffer.enqueue(AGENT, ROOM_A, _message("chatter"))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 1)@@ -868,7 +866,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:buffer.enqueue(AGENT, ROOM_B, _message("before the claim", room=ROOM_B))buffer.enqueue(AGENT, ROOM_A, _message("for you", addressed=True))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 2) # connection_state, then the message in room A@@ -887,7 +885,7 @@ class TestWhatADeliveredEventSaysAboutItsRoom:conn = _open(registry, cursor=4812) # from a previous liferegistry.claim_room(conn, ROOM_A)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)assert (await _take(stream, 2))[1][0] == "gap"@@ -915,7 +913,7 @@ class TestAGapNamesTheRoomsThatLostEvents:registry.claim_room(conn, ROOM_A)registry.claim_room(conn, ROOM_B)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -932,7 +930,7 @@ class TestAGapNamesTheRoomsThatLostEvents:conn = _open(registry, cursor=4812)registry.claim_room(conn, ROOM_A)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -954,7 +952,7 @@ class TestAGapNamesTheRoomsThatLostEvents:buffer.enqueue(AGENT, ROOM_A, _message("after the restart"))conn = _open(registry, scope="all", cursor=4812)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -981,7 +979,7 @@ class TestAGapNamesTheRoomsThatLostEvents:conn = _open(registry, cursor=0)registry.claim_room(conn, ROOM_A)- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)frames = await _take(stream, 2)@@ -999,7 +997,7 @@ async def test_a_delivered_event_names_no_session() -> None:registry.place_session(AGENT, "session-1", ROOM_A, conn.id)buffer.enqueue(AGENT, ROOM_A, _message("hello"))- stream = event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None)+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)(_, (name, data)) = await _take(stream, 2)assert name == "message"@@ -1013,7 +1011,7 @@ async def test_new_boot_reports_gap_and_keeps_messages_when_old_cursor_is_below_conn = _open(registry, cursor=(1 << 32) + 2)registry.claim_room(conn, ROOM_A)frames = await _take(- event_stream(conn=conn, registry=registry, buffer=buffer, approvals=None), 5+ event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None), 5)assert frames[1][0] == "gap"assert frames[1][1]["resumed_at"] == 2 << 32@@ -1032,7 +1030,7 @@ async def test_each_delivered_event_counts_as_an_agent_bridge_event_out() -> Nonbuffer.enqueue(AGENT, ROOM_A, _message("one"))buffer.enqueue(AGENT, ROOM_B, _message("two", room=ROOM_B))- stream = event_stream(+ stream = event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)await _take(stream, 3)@@ -1048,3 +1046,43 @@ async def test_each_delivered_event_counts_as_an_agent_bridge_event_out() -> Non}finally:uninstall()+++class TestBothTransportsEncodeTheSameFrames:+ """The socket and the event stream an old client still reads carry the+ same frames from the same loop; only the encoding differs."""++ def test_an_event_carries_its_sequence_as_the_sse_id(self) -> None:+ frame = Frame("message", {"payload": {"body": "hi"}}, seq=7)++ assert encode_sse(frame) == (+ b'id: 7\nevent: message\ndata: {"payload":{"body":"hi"}}\n\n'+ )+ assert encode_ws(frame) == {+ "event": "message",+ "data": {"payload": {"body": "hi"}},+ "id": 7,+ }++ def test_a_frame_without_a_sequence_has_no_id(self) -> None:+ assert encode_sse(Frame("gap", {"reason": "x"})) == (+ b'event: gap\ndata: {"reason":"x"}\n\n'+ )++ def test_the_keepalive_is_a_comment_on_the_event_stream(self) -> None:+ assert encode_sse(KEEPALIVE) == b": keepalive\n\n"++ async def test_closing_the_event_stream_detaches_its_connection(self) -> None:+ registry = AgentConnectionRegistry()+ buffer = EventBuffer(sequence_base=0)+ conn = _open(registry)++ body = sse_stream(+ event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)+ )+ first = await anext(body)+ await body.aclose()++ assert first.startswith(b"event: connection_state\n")+ assert not conn.stream_attached+ assert registry.get(conn.id) is conndiff --git a/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.py b/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.pynew file mode 100644index 000000000..552059179--- /dev/null+++ b/core/tests/switch_core/bridges/agent/test_bearer_auth_websocket.py@@ -0,0 +1,148 @@+"""BearerAuthMiddleware refusing a WebSocket.++A WebSocket client cannot read the status of a refused handshake, so the+refusal is a close code it can read: 4000 plus the status, 4401 for a missing+or bad token.+"""++from __future__ import annotations++import json+from types import SimpleNamespace+from typing import Any++from switch_core.bridges.agent.api_key_cache import ApiKeyCache+from switch_core.bridges.agent.auth import BearerAuthMiddleware, ControllerPrincipal+from switch_core.db.stores.agent_store import AgentStore+from switch_core.db.stores.api_key_store import ApiKeyStore+++def _middleware() -> tuple[BearerAuthMiddleware, list[Any]]:+ reached: list[Any] = []++ async def _app(scope: Any, receive: Any, send: Any) -> None:+ reached.append(scope)++ mw = BearerAuthMiddleware(+ _app,+ agent_store=AgentStore(),+ api_key_store=ApiKeyStore(),+ api_key_cache=ApiKeyCache(ttl_seconds=5, max_entries=8),+ session_factory=None, # type: ignore[arg-type]+ )+ return mw, reached+++async def _call(mw: BearerAuthMiddleware, scope: dict[str, Any]) -> list[dict]:+ sent: list[dict] = []+ inbox = [{"type": "websocket.connect"}]++ async def receive() -> dict:+ return inbox.pop(0)++ async def send(message: dict) -> None:+ sent.append(message)++ await mw(scope, receive, send)+ return sent+++async def test_a_socket_without_a_token_is_accepted_and_closed_with_4401() -> None:+ mw, reached = _middleware()++ sent = await _call(+ mw,+ {+ "type": "websocket",+ "path": "/agents/a1/connection/ws",+ "headers": [],+ "query_string": b"",+ },+ )++ assert [m["type"] for m in sent] == ["websocket.accept", "websocket.close"]+ assert sent[1]["code"] == 4401+ assert reached == []+++async def test_a_request_without_a_token_is_still_an_http_401() -> None:+ mw, reached = _middleware()+ sent: list[dict] = []++ async def receive() -> dict:+ return {"type": "http.request", "body": b"", "more_body": False}++ async def send(message: dict) -> None:+ sent.append(message)++ await mw(+ {+ "type": "http",+ "method": "GET",+ "path": "/agents/a1/events",+ "headers": [],+ "query_string": b"",+ },+ receive,+ send,+ )++ assert sent[0]["status"] == 401+ assert reached == []+++class _NotAssigned:+ """A controller whose token is good and which runs no agent at all."""++ presence = SimpleNamespace(binding=lambda agent_id: None)+ auth_cache = None++ def handles(self, path: str) -> bool:+ return False++ def is_public(self, path: str) -> bool:+ return False++ def is_controller_token(self, token: str) -> bool:+ return True++ async def authenticate(self, token: str) -> ControllerPrincipal:+ return ControllerPrincipal(controller_id="c1", owner_id="u1", tenant_id="t1")+++async def test_a_controller_refused_on_the_socket_is_told_why_in_a_close() -> None:+ reached: list[Any] = []++ async def _app(scope: Any, receive: Any, send: Any) -> None:+ reached.append(scope)++ mw = BearerAuthMiddleware(+ _app,+ agent_store=AgentStore(),+ api_key_store=ApiKeyStore(),+ api_key_cache=ApiKeyCache(ttl_seconds=5, max_entries=8),+ session_factory=None, # type: ignore[arg-type]+ controller_auth=_NotAssigned(), # type: ignore[arg-type]+ )++ sent = await _call(+ mw,+ {+ "type": "websocket",+ "path": "/agents/a1/connection/ws",+ "headers": [(b"authorization", b"Bearer controller-token")],+ "query_string": b"",+ },+ )++ assert [m["type"] for m in sent] == [+ "websocket.accept",+ "websocket.send",+ "websocket.close",+ ]+ frame = json.loads(sent[1]["text"])+ assert frame["event"] == "refused"+ assert frame["data"]["status"] == 403+ assert frame["data"]["detail"]["code"] == "not_assigned"+ assert sent[2]["code"] == 4403+ assert reached == []diff --git a/core/tests/switch_core/gateway/test_hosted_workers.py b/core/tests/switch_core/gateway/test_hosted_workers.pyindex 386611c5c..43708d6ca 100644--- a/core/tests/switch_core/gateway/test_hosted_workers.py+++ b/core/tests/switch_core/gateway/test_hosted_workers.py@@ -13,11 +13,16 @@ from uuid import uuid4import httpximport pytest-from fastapi import FastAPI, HTTPException+from fastapi import FastAPI, HTTPException, WebSocketDisconnectfrom sqlalchemy import event as orm_eventfrom sqlalchemy import select-from switch_core.bridges.agent.api.handlers import connection_placements, poll_events+from switch_core.bridges.agent.api import handlers+from switch_core.bridges.agent.api.handlers import (+ connection_placements,+ connection_socket,+ poll_events,+)from switch_core.bridges.agent.api.hosted_cutover_routes import (router as hosted_cutover_router,)@@ -1231,3 +1236,134 @@ async def test_relay_with_real_auth_opens_two_transactions(worker_app):json={"message": {"health": True}, "timeout_ms": 50},)).status_code == 401+++class _Socket:+ """The handler's side of a WebSocket, in this test's event loop.++ Records what is sent and how it closes, and hands the handler whatever the+ test queues as the client's messages; `None` is the client going away.+ """++ def __init__(self) -> None:+ self.sent: list[dict[str, Any]] = []+ self.close_code: int | None = None+ self._incoming: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()+ self._changed = asyncio.Event()++ async def accept(self) -> None:+ return None++ async def send_json(self, message: dict[str, Any]) -> None:+ self.sent.append(message)+ self._changed.set()++ async def receive_json(self) -> dict[str, Any]:+ message = await self._incoming.get()+ if message is None:+ raise WebSocketDisconnect(1000)+ return message++ async def close(self, code: int = 1000) -> None:+ if self.close_code is None:+ self.close_code = code+ self._changed.set()++ def leave(self) -> None:+ self._incoming.put_nowait(None)++ async def frames(self, count: int) -> list[tuple[str, dict[str, Any]]]:+ """The next `count` frames sent, pings skipped."""++ def received() -> list[tuple[str, dict[str, Any]]]:+ return [(m["event"], m["data"]) for m in self.sent if m["event"] != "ping"]++ async def wait() -> None:+ while len(received()) < count:+ self._changed.clear()+ await self._changed.wait()++ await asyncio.wait_for(wait(), timeout=2)+ return received()[:count]+++async def _open_socket(+ service,+ agent: Agent,+ *,+ capability: str | None,+ connection_id: str | None = None,+ speaks: int = 7,+ state_version: int | None = 1,+) -> tuple[_Socket, asyncio.Task[None]]:+ socket = _Socket()+ task = asyncio.create_task(+ connection_socket(+ websocket=socket, # type: ignore[arg-type]+ agent_id=agent.id,+ agent=agent,+ protocol=service,+ config=service.config,+ connection_id=connection_id or str(uuid4()),+ scope="all",+ spawn_capable=True,+ protocol_version=speaks,+ worker_capability=capability,+ host_boot_id="boot-a",+ host_instance_id="instance-a",+ worker_state_version=state_version,+ )+ )+ return socket, task+++ ("change", "refused"),+ [+ ({"capability": None}, (403, "worker_capability_required")),+ ({"capability": "not-it"}, (403, "worker_capability_obsolete")),+ ({"speaks": 6}, (426, "upgrade_required")),+ ({"state_version": None}, (426, "upgrade_required")),+ ],+)+async def test_the_socket_refuses_a_worker_open_as_the_stream_does(+ worker_app, change: dict[str, Any], refused: tuple[int, str]+):+ _, _, agent_id, service, factory, prepared = worker_app+ agent = await _agent(factory, agent_id)+ opening = {"capability": prepared["worker_capability"], **change}+ assert await _refusal(_open(service, agent, **opening)) == refused++ socket, task = await _open_socket(service, agent, **opening)+ await asyncio.wait_for(task, timeout=2)++ status, code = refused+ ((event, data),) = await socket.frames(1)+ assert event == "refused"+ assert data["status"] == status+ assert data["detail"]["code"] == code+ assert socket.close_code == 4000 + status+ assert service.connections.for_agent(agent_id) == []+++async def test_a_worker_over_the_socket_is_sent_its_frames(worker_app, monkeypatch):+ # Pings every 50 ms, so the socket notices its client leave promptly.+ monkeypatch.setattr(handlers, "HEARTBEAT_INTERVAL_SECONDS", 0.05)+ _, _, agent_id, service, factory, prepared = worker_app+ agent = await _agent(factory, agent_id)+ conn_id = str(uuid4())+ socket, task = await _open_socket(+ service, agent, capability=prepared["worker_capability"], connection_id=conn_id+ )+ try:+ (state, _), (attached, data) = await socket.frames(2)+ assert (state, attached) == ("connection_state", "worker_attached")+ assert set(data) == WORKER_ATTACHED_FIELDS+ conn = service.connections.get(conn_id)+ assert conn.worker is not None and conn.stream_transport == "websocket"++ assert service.connections.ring_worker(agent_id, "wake", {"entries": []})+ assert (await socket.frames(3))[2] == ("wake", {"entries": []})+ finally:+ socket.leave()+ await asyncio.wait_for(task, timeout=2)diff --git a/core/tests/switch_core/session_activity/test_command_relay.py b/core/tests/switch_core/session_activity/test_command_relay.pyindex 4e34b0bbe..8f95f2ce6 100644--- a/core/tests/switch_core/session_activity/test_command_relay.py+++ b/core/tests/switch_core/session_activity/test_command_relay.py@@ -1,20 +1,20 @@from __future__ import annotationsimport asyncio-import jsonimport httpximport pytestfrom fastapi import FastAPIfrom switch_core.bridges.agent.protocol.agent_connections import (+ PROTOCOL_VERSION,SESSION_COMMAND_PROTOCOL_REVISION,AgentConnectionRegistry,ClientDeclaration,)from switch_core.bridges.agent.protocol.event_buffer import EventBufferfrom switch_core.bridges.agent.protocol.hosted_workers import WorkerBinding-from switch_core.bridges.agent.protocol.stream import event_stream+from switch_core.bridges.agent.protocol.stream import event_framesfrom switch_core.db.models import Userfrom switch_core.gateway.agent_sessions import routerfrom switch_core.gateway.auth import get_current_user@@ -33,7 +33,9 @@ class _Protocol:self.connections = registry-def _watcher(registry: AgentConnectionRegistry, *, speaks: int, scope: str = "all"):+def _watcher(+ registry: AgentConnectionRegistry, *, speaks: int | None, scope: str = "all"+):conn = registry.open(agent_id=AGENT,connection_id=f"conn-{scope}-{speaks}",@@ -44,7 +46,7 @@ def _watcher(registry: AgentConnectionRegistry, *, speaks: int, scope: str = "aldeclaration=ClientDeclaration(speaks=speaks),expected_generation=None,)- stream = event_stream(+ stream = event_frames(conn=conn,registry=registry,buffer=EventBuffer(sequence_base=0),@@ -75,15 +77,15 @@ async def owner(session_factory, people) -> str:async def test_a_relayed_session_command_reaches_the_watcher_stream() -> None:registry = AgentConnectionRegistry()- _, stream = _watcher(registry, speaks=SESSION_COMMAND_PROTOCOL_REVISION)+ _, stream = _watcher(registry, speaks=PROTOCOL_VERSION)try:await anext(stream) # connection_statewaiting = asyncio.create_task(anext(stream))await asyncio.sleep(0.05)assert registry.relay_session_command(AGENT, FRAME, worker_only=False) is True- text = (await asyncio.wait_for(waiting, 5)).decode()- assert "event: session_command\n" in text- assert json.loads(text.split("data: ", 1)[1]) == FRAME+ frame = await asyncio.wait_for(waiting, 5)+ assert frame.event == "session_command"+ assert frame.data == FRAMEfinally:await stream.aclose()@@ -91,8 +93,8 @@ async def test_a_relayed_session_command_reaches_the_watcher_stream() -> None:@pytest.mark.parametrize(("speaks", "scope"),[- (SESSION_COMMAND_PROTOCOL_REVISION - 1, "all"),- (SESSION_COMMAND_PROTOCOL_REVISION, "single"),+ (None, "all"),+ (PROTOCOL_VERSION, "single"),],)async def test_a_command_nobody_can_take_is_not_relayed(speaks, scope) -> None:@@ -109,10 +111,8 @@ async def test_switch_does_not_answer_where_an_agents_sessions_are() -> None:"""The agent's room watcher owns its connection and placements; Console asks it."""registry = AgentConnectionRegistry()- _watcher(registry, speaks=SESSION_COMMAND_PROTOCOL_REVISION)- registry.place_session(- AGENT, "session-1", "room-1", f"conn-all-{SESSION_COMMAND_PROTOCOL_REVISION}"- )+ _watcher(registry, speaks=PROTOCOL_VERSION)+ registry.place_session(AGENT, "session-1", "room-1", f"conn-all-{PROTOCOL_VERSION}")async with _client(session_factory, registry, owner) as client:assert (await client.get("/agent-sessions/room-health")).status_code == 404diff --git a/core/tests/switch_core/session_activity/test_stop_press.py b/core/tests/switch_core/session_activity/test_stop_press.pyindex bcc949617..6c8653d6c 100644--- a/core/tests/switch_core/session_activity/test_stop_press.py+++ b/core/tests/switch_core/session_activity/test_stop_press.py@@ -7,7 +7,7 @@ from sqlalchemy import updatefrom switch_core.addressing import owner_only_policyfrom switch_core.bridges.agent.protocol.agent_connections import (- SESSION_COMMAND_PROTOCOL_REVISION,+ PROTOCOL_VERSION,AgentConnectionRegistry,ClientDeclaration,)@@ -49,7 +49,7 @@ def _watching(registry: AgentConnectionRegistry):delivery_filter="addressed",spawn_capable=False,cursor=0,- declaration=ClientDeclaration(speaks=SESSION_COMMAND_PROTOCOL_REVISION),+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),expected_generation=None,)conn.stream_attached = Truediff --git a/core/tests/switch_core/session_activity/test_stream_outcomes.py b/core/tests/switch_core/session_activity/test_stream_outcomes.pyindex c500044cf..44e5b796c 100644--- a/core/tests/switch_core/session_activity/test_stream_outcomes.py+++ b/core/tests/switch_core/session_activity/test_stream_outcomes.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport asyncioimport contextlib-import jsonfrom collections.abc import AsyncIteratorfrom datetime import UTC, datetime, timedelta@@ -12,13 +11,12 @@ from sqlalchemy.ext.asyncio import create_async_enginefrom sqlalchemy.pool import NullPoolfrom switch_core.bridges.agent.protocol.agent_connections import (- APPROVAL_OUTCOME_PROTOCOL_REVISION,PROTOCOL_VERSION,AgentConnectionRegistry,ClientDeclaration,)from switch_core.bridges.agent.protocol.event_buffer import EventBuffer-from switch_core.bridges.agent.protocol.stream import event_stream+from switch_core.bridges.agent.protocol.stream import Frame, event_framesfrom switch_core.db.models import ApprovalRequestfrom switch_core.session_activity.listener import AgentSessionActivityListenerfrom switch_core.session_activity.outcomes import ApprovalOutcomes@@ -61,7 +59,7 @@ def _open_stream(approvals, *, agent_id=AGENT, scope="all", speaks=PROTOCOL_VERSdeclaration=ClientDeclaration(speaks=speaks),expected_generation=None,)- return event_stream(+ return event_frames(conn=conn,registry=registry,buffer=EventBuffer(sequence_base=0),@@ -69,10 +67,9 @@ def _open_stream(approvals, *, agent_id=AGENT, scope="all", speaks=PROTOCOL_VERS)-def _outcome(frame: bytes) -> dict:- text = frame.decode()- assert "event: approval_outcome\n" in text, text- return json.loads(text.split("data: ", 1)[1])+def _outcome(frame: Frame) -> dict:+ assert frame.event == "approval_outcome", frame+ return frame.dataasync def _no_frame_within(stream, seconds: float) -> None:@@ -228,7 +225,7 @@ async def test_a_stream_without_approvals_carries_none(service, people):await stream.aclose()-async def test_a_client_older_than_the_outcome_revision_is_sent_none(+async def test_a_client_that_declared_no_revision_is_sent_none(service, approvals, people):# An older client hands unknown frames to its room-event path and breaks.@@ -240,10 +237,9 @@ async def test_a_client_older_than_the_outcome_revision_is_sent_none(answer=pick("allow"),answerer=PlatformPerson(people.owner),)- for speaks in (APPROVAL_OUTCOME_PROTOCOL_REVISION - 1, None):- stream = _open_stream(approvals, speaks=speaks)- try:- await anext(stream)- await _no_frame_within(stream, 0.3)- finally:- await stream.aclose()+ stream = _open_stream(approvals, speaks=None)+ try:+ await anext(stream)+ await _no_frame_within(stream, 0.3)+ finally:+ await stream.aclose()