Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Remove the agent event stream and beat POST, once old Consoles are gone (CHOO-3013) (#673)

Abel Dantas54ea46c96a46
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f25966de8..37109bbab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -994,6 +994,12 @@ version of their own to them without also giving them a release of their own.
through. Slack apps that list `/run-cmd` in their manifest keep showing it,
answering "Unknown command", until an admin deletes it; Discord and Telegram
drop it from their command lists by themselves (#730).
+- **The agent event stream and `POST /connection/beat`.** Agents connect over
+ the WebSocket at `/agents/{id}/connection/ws`, which Switch Console has used
+ since 0.39.0. A client still asking for the Server-Sent Events stream is
+ refused with 410 `transport_removed` and told to update, and agent-protocol
+ `accepts` rises to 8, so Consoles 0.38.2 and older can no longer connect
+ their agents (#673).
- **Deleting your account now also clears workspaces you were removed from.**
Removing a member leaves what they own in that workspace, and deleting their
account afterwards failed on it. Those rows are now deleted or handed over as
@@ -2938,6 +2944,12 @@ version of their own to them without also giving them a release of their own.
machine whose organization does not enforce the keyring: switch Codex to file
storage, which keeps the login in plaintext, and sign in again (#1048).
+#### Removed
+- **The fallback to the agent event stream.** The agent runtime connects only
+ over the WebSocket, so it can no longer reach a Switch server older than the
+ socket (agent-protocol 7 and older), and it stops sending its own heartbeat
+ requests (#673).
+
### [0.40.1] - 2026-10-21
#### Added
diff --git a/artifacts.yaml b/artifacts.yaml
index bb0e42581..d3b64205b 100644
--- a/artifacts.yaml
+++ b/artifacts.yaml
@@ -224,15 +224,11 @@ contracts:
# heartbeat interval and the client answers `pong` with its cursor. The
# token is checked once, when the socket opens. A hosted worker opens it
# with the headers and under the rules revision 7 set for the stream. It
- # is meant to replace the Server-Sent Events stream and
- # `POST /connection/beat`, which would make it
- # the first revision that is not additive, but that removal is deferred:
- # the stream and the beat are kept beside the socket for a compatibility
- # window, so `accepts` stays at 1 on both sides. switch-core serves either
- # transport, and agent-runtime prefers the socket and falls back to the
- # stream and its beat against a server with no socket. Both go, and
- # `accepts` rises to 8, once `switch.agents.connected{transport:sse}` stays
- # at zero.
+ # replaces the Server-Sent Events stream and `POST /connection/beat`, which
+ # were kept beside the socket for a compatibility window and are now gone:
+ # `accepts` rises to 8 on both sides, the lowest revision with the socket,
+ # and a client still asking for the stream is refused with 410 and told to
+ # update.
#
# Revision 9 makes the resume point a position: one number per source, the
# agent's own events (`s`) and each room's conversation (`r:<room>`), so a
@@ -244,10 +240,10 @@ contracts:
# integer fields stay, so a client that knows only them is served as
# before: a stream that reads only the agent's own events is one number
# already, and one that reads the conversation too is handed tickets that
- # stand for its position. Additive, so `accepts` stays at 1.
+ # stand for its position. Additive, so `accepts` stays at 8.
artifacts:
- switch-core: { speaks: 9, accepts: 1 }
- agent-runtime: { speaks: 9, accepts: 1 }
+ switch-core: { speaks: 9, accepts: 8 }
+ agent-runtime: { speaks: 9, accepts: 8 }
gateway-api:
description: >-
diff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.ts
index 183a741eb..de2d86bec 100644
--- a/console/packages/shared/src/artifacts.ts
+++ b/console/packages/shared/src/artifacts.ts
@@ -44,8 +44,8 @@ export function artifactVersion(artifact: ArtifactName): string {
export const CONTRACTS = {
// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
'agent-protocol': {
- 'switch-core': { speaks: 9, accepts: 1 },
- 'agent-runtime': { speaks: 9, accepts: 1 },
+ 'switch-core': { speaks: 9, accepts: 8 },
+ 'agent-runtime': { speaks: 9, accepts: 8 },
},
// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
'gateway-api': {
diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.ts
index 183a741eb..de2d86bec 100644
--- a/console/packages/switch-agent-runtime/src/artifacts.ts
+++ b/console/packages/switch-agent-runtime/src/artifacts.ts
@@ -44,8 +44,8 @@ export function artifactVersion(artifact: ArtifactName): string {
export const CONTRACTS = {
// The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
'agent-protocol': {
- 'switch-core': { speaks: 9, accepts: 1 },
- 'agent-runtime': { speaks: 9, accepts: 1 },
+ 'switch-core': { speaks: 9, accepts: 8 },
+ 'agent-runtime': { speaks: 9, accepts: 8 },
},
// The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
'gateway-api': {
diff --git a/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts b/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts
deleted file mode 100644
index ddd2aedae..000000000
--- a/console/packages/switch-agent-runtime/src/event-stream.old-server.test.ts
+++ /dev/null
@@ -1,2153 +0,0 @@
-import { afterEach, describe, expect, it, vi } from 'vitest';
-import {
- BEAT_INTERVAL_MS,
- BEAT_SETTLE_LIMIT_MS,
- EVICTION_CREDENTIALS_REJECTED,
- EVICTION_TAKEN_OVER,
- SwitchEventStream,
- type Eviction,
- type SwitchEventStreamDeps,
-} from './event-stream';
-
-/**
- * The runtime against a server from before the socket (agent-protocol 7 and
- * older).
- *
- * Such a server has no `/connection/ws`: the upgrade fails before it opens,
- * which Node's WebSocket reports as an error and a 1006 close and nothing
- * more. The runtime falls back to the event stream and its own heartbeat, and
- * has to behave there exactly as it did before the socket, so these are the
- * tests it had then, run with a socket that never opens. The ones on choosing
- * the transport are at the end.
- *
- * Two ways a client can retry something that can never succeed were measured
- * on a live deployment: a stream re-declaring a room that had been deleted,
- * refused on every open, reopening at once; and a heartbeat treating "your
- * connection is gone" as a normal answer and beating on at full cadence.
- * Neither could self-heal, and between them they produced most of the error
- * volume on that deployment.
- */
-
-const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };
-
-function silentLog() {
- return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
-}
-
-function encodeFrame(name: string, data: unknown): Uint8Array {
- return new TextEncoder().encode(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);
-}
-
-/** The frame a server opens every stream with, naming the incarnation this
- * client is attached to. No `rooms`, so it says nothing about the declared set
- * a test may be asserting on. */
-function connected(generation = 0): Uint8Array {
- return encodeFrame('connection_state', { connection_id: 'conn-1', generation });
-}
-
-/** A stream body carrying one frame and then closing, the way the server ends
- * a displaced stream. */
-function frameThenClose(name: string, data: unknown): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(encodeFrame(name, data));
- controller.close();
- },
- });
-}
-
-/**
- * A connected stream body that stays open, so the loop neither reconnects nor
- * spins.
- *
- * It announces the connection first because the heartbeat waits for that: a
- * client that has not been told which incarnation it is cannot fence its own
- * tick, and a body that never announces is a connection that never beats.
- */
-function openForever(): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- },
- });
-}
-
-/**
- * A connected stream body that ends when the client drops the socket.
- *
- * `openForever` cannot: its reader is parked on a read that never returns, so
- * a reopen is invisible to the test rather than absent. A body that honours
- * the request's signal is what a real socket does, and what a test asserting
- * on reopens needs.
- */
-function openUntilAborted(init: { signal: AbortSignal }): ReadableStream<Uint8Array> {
- return new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- init.signal.addEventListener('abort', () => controller.close(), { once: true });
- },
- });
-}
-
-/** The incarnation each beat carried, in order. */
-function beatGenerations(fetchMock: { mock: { calls: unknown[][] } }): (number | null)[] {
- return fetchMock.mock.calls
- .filter((call) => String(call[0]).includes('connection/beat'))
- .map(
- (call) =>
- (JSON.parse((call[1] as { body: string }).body) as { generation: number | null }).generation
- );
-}
-
-function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {
- return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));
-}
-
-/**
- * The socket an old server's upgrade gives: it never opens. Its error and
- * close arrive on the microtask queue, the way a real socket's arrive as
- * separate tasks, once the client has attached its listeners.
- */
-class SocketThatNeverOpens {
- static opened: string[] = [];
- readonly CONNECTING = 0;
- readonly OPEN = 1;
- readyState = 0;
- private readonly listeners = new Map<string, Set<(event: unknown) => void>>();
-
- constructor(url: string) {
- SocketThatNeverOpens.opened.push(url);
- queueMicrotask(() => {
- this.readyState = 3;
- for (const listener of [...(this.listeners.get('error') ?? [])]) listener({});
- for (const listener of [...(this.listeners.get('close') ?? [])])
- listener({ code: 1006, reason: '' });
- });
- }
-
- addEventListener(type: string, listener: (event: unknown) => void): void {
- const set = this.listeners.get(type) ?? new Set();
- set.add(listener);
- this.listeners.set(type, set);
- }
-
- removeEventListener(type: string, listener: (event: unknown) => void): void {
- this.listeners.get(type)?.delete(listener);
- }
-
- send(): void {
- throw new Error('nothing is sent on a socket that never opened');
- }
-
- close(): void {}
-}
-
-function makeStream(
- fetchMock: ReturnType<typeof vi.fn>,
- deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }
-) {
- vi.stubGlobal('WebSocket', SocketThatNeverOpens);
- SocketThatNeverOpens.opened = [];
- vi.stubGlobal('fetch', fetchMock);
- const abort = new AbortController();
- const log = silentLog();
- const stream = new SwitchEventStream({
- creds,
- connectionId: 'conn-1',
- scope: 'single',
- filter: 'all',
- onEvent: () => {},
- onGap: () => {},
- onEvicted: () => {},
- log,
- signal: abort.signal,
- ...deps,
- });
- stream.start();
- return { stream, abort, log };
-}
-
-async function flush(times = 8): Promise<void> {
- for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));
-}
-
-afterEach(() => {
- vi.unstubAllGlobals();
- vi.useRealTimers();
-});
-
-describe('a room the server refuses', () => {
- /** The refusal switch-core returns when a declared room no longer exists. */
- function roomGone(roomId: string) {
- return {
- ok: false,
- status: 403,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),
- };
- }
-
- it('is dropped and not declared again', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events')) {
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- return String(url).includes('rooms=')
- ? roomGone('room-dead')
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const rejected: { roomId: string; status: number }[] = [];
- const reported: string[][] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-dead'],
- onRooms: (rooms) => reported.push(rooms),
- onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),
- });
- await flush();
-
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(2);
- expect(opens[0]).toContain('rooms=room-dead');
- expect(opens[1]).not.toContain('rooms=');
- expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);
- expect(reported).toEqual([[]]);
- expect(log.error).toHaveBeenCalled();
- abort.abort();
- });
-
- it('keeps the rooms the refusal does not name', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- const u = String(url);
- if (!u.includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return u.includes('room-dead')
- ? roomGone('room-dead')
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });
- await flush();
-
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(2);
- expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');
- abort.abort();
- });
-
- it('stays a transport error when the body names no declared room', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events')) {
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- return {
- ok: false,
- status: 404,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail: 'No such connection' }),
- };
- });
- const rejected: string[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onRoomRejected: ({ roomId }) => rejected.push(roomId),
- });
- await vi.advanceTimersByTimeAsync(0);
-
- // One open, then the backoff, not an immediate retry, and nothing dropped.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(rejected).toEqual([]);
- abort.abort();
- });
-});
-
-describe('credentials the server rejects', () => {
- function refusal(status: number, detail: string) {
- return {
- ok: false,
- status,
- body: null,
- text: async (): Promise<string> => JSON.stringify({ detail }),
- };
- }
-
- /** Open once, be refused, and let a long while pass. */
- async function afterRefusal(status: number, detail: string) {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? refusal(status, detail)
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onEvicted: (eviction) => evicted.push(eviction),
- });
- await vi.advanceTimersByTimeAsync(0);
- const settled = fetchMock.mock.calls.length;
- await vi.advanceTimersByTimeAsync(5 * 60_000);
- abort.abort();
- return { fetchMock, evicted, log, settled };
- }
-
- it('ends the stream and the heartbeat on a 401, and says so once', async () => {
- const { fetchMock, evicted, log, settled } = await afterRefusal(401, 'Invalid agent token');
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- // Nothing beyond the beat already in flight when the refusal landed.
- expect(fetchMock.mock.calls).toHaveLength(settled);
- expect(evicted).toHaveLength(1);
- expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
- expect(evicted[0]?.reason).toContain('credentials');
- expect(evicted[0]?.reason).toContain('401');
- expect(log.error).toHaveBeenCalled();
- });
-
- it('ends the stream on a 403 that names no declared room', async () => {
- const { fetchMock, evicted, settled } = await afterRefusal(
- 403,
- 'Agent is not a member of this room'
- );
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(fetchMock.mock.calls).toHaveLength(settled);
- expect(evicted).toHaveLength(1);
- expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
- expect(evicted[0]?.reason).toContain('credentials');
- expect(evicted[0]?.reason).toContain('403');
- });
-
- it('ends the stream and the heartbeat on a 401 heartbeat, and says so once', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
- : refusal(401, 'Invalid agent token')
- );
- const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onEvicted: (eviction) => evicted.push(eviction),
- });
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS + 1);
- const settled = fetchMock.mock.calls.length;
- await vi.advanceTimersByTimeAsync(5 * 60_000);
- abort.abort();
-
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(fetchMock.mock.calls).toHaveLength(settled);
- expect(evicted).toHaveLength(1);
- expect(evicted[0]?.code).toBe(EVICTION_CREDENTIALS_REJECTED);
- expect(evicted[0]?.reason).toContain('credentials');
- expect(evicted[0]?.reason).toContain('401');
- expect(log.error).toHaveBeenCalledTimes(1);
- });
-
- it('tells a session start to the room itself only where the server says it leaves that to it', async () => {
- for (const [said, expected] of [
- [{ announce_session_starts: true }, true],
- [{}, false],
- ] as const) {
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('connection_state', {
- connection_id: 'conn-1',
- generation: 0,
- ...said,
- }),
- text: async (): Promise<string> => '',
- }
- : new Promise<never>(() => {})
- );
- let connected = false;
- const { stream, abort } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onConnected: () => {
- connected = true;
- },
- });
- expect(stream.announcesSessionStarts).toBe(false);
- for (let i = 0; i < 50 && !connected; i += 1) await flush();
- expect(stream.announcesSessionStarts).toBe(expected);
- abort.abort();
- vi.unstubAllGlobals();
- }
- });
-
- it('tells the owner once when the stream and the heartbeat are refused together', async () => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events')) {
- // Connected once (the heartbeat has nothing to beat for until then)
- // and refused on every reopen after that.
- opens += 1;
- if (opens === 1) {
- return {
- ok: true,
- status: 200,
- body: frameThenClose('connection_state', { connection_id: 'conn-1', generation: 0 }),
- text: async (): Promise<string> => '',
- };
- }
- await new Promise((r) => setTimeout(r, BEAT_INTERVAL_MS));
- return refusal(401, 'Invalid agent token');
- }
- // Slow enough to still be in flight when the reopen is refused: both
- // doors closing at once is the case this test is about.
- await new Promise((r) => setTimeout(r, 2 * BEAT_INTERVAL_MS));
- return refusal(401, 'Invalid agent token');
- });
- const evicted: Eviction[] = [];
- const { abort, log } = makeStream(fetchMock, {
- rooms: ['room-live'],
- onEvicted: (eviction) => evicted.push(eviction),
- });
- await vi.advanceTimersByTimeAsync(5 * 60_000);
- abort.abort();
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(evicted).toHaveLength(1);
- expect(log.error).toHaveBeenCalledTimes(1);
- });
-
- it.each([503, 429])('keeps reconnecting after HTTP %i', async (status) => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- opens += 1;
- return opens === 1
- ? { ok: false, status, body: null, text: async (): Promise<string> => 'try later' }
- : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(2000);
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-
- it('keeps reconnecting after a network error', async () => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- opens += 1;
- if (opens === 1) throw new TypeError('fetch failed');
- return { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(2000);
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-});
-
-describe('the heartbeat', () => {
- /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */
- async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- return { ok: false, status, text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
- await vi.advanceTimersByTimeAsync(windowMs);
- const beats = urlsFor(fetchMock, 'connection/beat').length;
- abort.abort();
- return beats;
- }
-
- it('backs off when the connection is rejected, instead of beating at full rate', async () => {
- const windowMs = 20 * BEAT_INTERVAL_MS;
- // At the base cadence this window holds ~20 beats. Doubling from the base
- // gives 4s, 8s, 16s…, four requests in the same window.
- expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);
- expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);
- });
-
- it('keeps backing off the longer the rejection lasts', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(30_000);
- const early = urlsFor(fetchMock, 'connection/beat').length;
- await vi.advanceTimersByTimeAsync(30_000);
- const late = urlsFor(fetchMock, 'connection/beat').length - early;
-
- expect(late).toBeLessThan(early);
- abort.abort();
- });
-
- it('sends nothing until the server has said which incarnation it is', async () => {
- vi.useFakeTimers();
- const server = { announce: (): void => {} };
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- server.announce = () => controller.enqueue(connected(7));
- },
- }),
- text: async (): Promise<string> => '',
- };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
- // An unfenced tick is what a displaced client sends, and the server refuses
- // it. Beating before the first frame would make every healthy connection
- // open with one.
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(0);
-
- server.announce();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
-
- const beats = fetchMock.mock.calls.filter((c) => String(c[0]).includes('connection/beat'));
- expect(beats.length).toBeGreaterThan(0);
- // And it carries the incarnation that frame named, so the server can fence it.
- const [, sent] = beats[0] as unknown as [string, { body: string }];
- expect(JSON.parse(sent.body).generation).toBe(7);
- abort.abort();
- });
-
- it('waits for the new incarnation after its own reconnect, instead of standing down', async () => {
- vi.useFakeTimers();
- // A server that fences: every attach is a new incarnation, and a tick
- // naming an older one is refused the way a displaced client's is.
- const server = { generation: 0, announce: (): void => {}, drop: (): void => {} };
- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
- if (String(url).includes('/events')) {
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- const announce = (): void => controller.enqueue(connected(attached));
- // The first attach is announced at once; the second is held, so
- // the beat falls due inside the window this test is about.
- if (attached === 1) announce();
- else server.announce = announce;
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
- if (sent === server.generation)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
- };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
-
- await vi.advanceTimersByTimeAsync(2 * BEAT_INTERVAL_MS);
- const first = beatGenerations(fetchMock);
- expect(first.length).toBeGreaterThan(0);
- expect(first.every((generation) => generation === 1)).toBe(true);
-
- server.drop();
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
- // Nothing went out while the reopen was in flight: the incarnation we hold
- // is the one before it, and a tick carrying that is a stand-down.
- expect(beatGenerations(fetchMock)).toHaveLength(first.length);
- expect(evicted).toEqual([]);
-
- server.announce();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- const resumed = beatGenerations(fetchMock).slice(first.length);
- expect(resumed.length).toBeGreaterThan(0);
- expect(resumed.every((generation) => generation === 2)).toBe(true);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-
- it('does not stand down over a beat that was already in flight when it reopened', async () => {
- vi.useFakeTimers();
- // The gate stops a beat *starting* inside the reattach window. It cannot
- // recall one that left before the gate shut, and that beat is answered
- // against the incarnation the reopen replaced, a truthful `taken_over`
- // about a takeover this client performed on itself.
- const server = {
- generation: 0,
- announce: (): void => {},
- drop: (): void => {},
- answerBeat: (): void => {},
- beatsHeld: 0,
- };
- const fetchMock = vi.fn(async (url: string, init: { body: string }) => {
- if (String(url).includes('/events')) {
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) !== server.generation) {
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
- }
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(attached));
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- const sent = (JSON.parse(init.body) as { generation: number | null }).generation;
- // Held rather than answered: the point of the test is an answer that
- // arrives after the connection has moved on beneath it.
- server.beatsHeld += 1;
- await new Promise<void>((resolve) => {
- server.answerBeat = resolve;
- });
- if (sent === server.generation)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'another stream attached' } }),
- };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- expect(server.beatsHeld).toBe(1);
- expect(beatGenerations(fetchMock)).toEqual([1]);
-
- // Reopen with that beat still outstanding. The bound expires, the open goes
- // ahead, and the server is at 2 by the time the old beat is answered.
- server.drop();
- await vi.advanceTimersByTimeAsync(BEAT_SETTLE_LIMIT_MS + 10 * BEAT_INTERVAL_MS);
- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(1);
-
- server.answerBeat();
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
-
- // Undecidable, so inert. Standing down here stops a client that nothing
- // has taken anything from.
- expect(evicted).toEqual([]);
- const resumed = beatGenerations(fetchMock).slice(1);
- expect(resumed.length).toBeGreaterThan(0);
- expect(resumed.every((generation) => generation === 2)).toBe(true);
- abort.abort();
- });
-
- it('stands down when its reattach is refused, rather than trying again', async () => {
- vi.useFakeTimers();
- // The other half: a client that missed its eviction and comes back. The
- // server refuses the reattach without disturbing the holder, and this is
- // the only thing left that can tell the loser it lost.
- const server = { generation: 0, drop: (): void => {} };
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null) {
- // Someone else attached while we were away, so our claim is stale.
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
- }
- server.generation += 1;
- const attached = server.generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(attached));
- server.drop = () => controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- };
- });
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted: (e) => evicted.push(e) });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- server.drop();
- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
-
- expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
- // And it stopped asking: retrying is how the pair trade the connection.
- const attempts = urlsFor(fetchMock, 'expected_generation=1').length;
- await vi.advanceTimersByTimeAsync(20 * BEAT_INTERVAL_MS);
- expect(urlsFor(fetchMock, 'expected_generation=1')).toHaveLength(attempts);
- abort.abort();
- });
-
- it('returns to the base cadence once a beat lands', async () => {
- vi.useFakeTimers();
- let reject = true;
- // A socket that ends when the client drops it: each refusal reopens, and
- // the heartbeat is held from the reopen until the new stream is named.
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- };
- if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(30_000);
- reject = false;
- await vi.advanceTimersByTimeAsync(30_000);
- const recovered = urlsFor(fetchMock, 'connection/beat').length;
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
-
- expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);
- abort.abort();
- });
-});
-
-describe('the heartbeat after a reconnect', () => {
- /**
- * A server that reattaches on every open, can drop the current stream, and
- * records when each stream was opened and each beat arrived. The server's
- * own rule is what these tests hold the client to: a new connection that
- * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.
- */
- const HEARTBEAT_TTL_MS = 6000;
-
- /** Either end may close a stream first; the second close is not an error here. */
- function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {
- try {
- controller.close();
- } catch {
- // Already closed by the other side.
- }
- }
-
- function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {
- const opens: number[] = [];
- const beats: number[] = [];
- let generation = 0;
- let current: ReadableStreamDefaultController<Uint8Array> | null = null;
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('/events')) {
- opens.push(Date.now());
- generation += 1;
- const gen = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- current = controller;
- controller.enqueue(connected(gen));
- init.signal.addEventListener('abort', () => closeQuietly(controller), {
- once: true,
- });
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- beats.push(Date.now());
- const reply = beatReply();
- if (reply === 'fail') throw new TypeError('fetch failed');
- if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- /** The server ends the current stream, as it does a lapsed connection. */
- const drop = (): void => {
- if (current) closeQuietly(current);
- };
- return { fetchMock, opens, beats, drop };
- }
-
- it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {
- // The lockout: beats fail for a while, so the gap between them climbs to
- // its 30-second cap. The stream then reopens. The server starts its
- // 6-second clock on the new connection, and the client used to sit out
- // the rest of the 30 seconds before saying anything, lose the connection
- // for silence, and repeat that on every connection it opened afterwards.
- vi.useFakeTimers();
- let failing = true;
- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- // Long enough of failed beats for the gap to reach its cap.
- await vi.advanceTimersByTimeAsync(70_000);
- failing = false;
- server.drop();
- await vi.advanceTimersByTimeAsync(40_000);
-
- const reopenedAt = server.opens[1];
- expect(reopenedAt).toBeDefined();
- const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);
- expect(firstBeatAfter).toBeDefined();
- expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);
- abort.abort();
- });
-
- it('keeps beating at the base cadence on the new connection once it lands', async () => {
- vi.useFakeTimers();
- let failing = true;
- const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(70_000);
- failing = false;
- server.drop();
- await vi.advanceTimersByTimeAsync(5_000);
- const settled = server.beats.length;
- await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
-
- expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);
- abort.abort();
- });
-
- it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {
- // The brake the old ordering provided, which beating promptly on every
- // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The
- // wait now sits before each reopen and doubles, so a refusing server sees
- // a handful of reopens, not one every couple of seconds per agent.
- vi.useFakeTimers();
- const server = reattachingServer(() => 404);
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(60_000);
-
- // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.
- expect(server.opens.length).toBeLessThanOrEqual(6);
- abort.abort();
- });
-
- it('does not throw away a connection the stream reopened while it waited', async () => {
- // A lapsed connection's socket is closed by the server, and the stream
- // loop reopens it by itself. If that lands during the heartbeat's wait
- // after a refusal, reopening again would discard a healthy connection.
- vi.useFakeTimers();
- let refuse = true;
- const server = reattachingServer(() => (refuse ? 404 : 'ok'));
- const { abort } = makeStream(server.fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
- expect(server.beats.length).toBe(1);
- refuse = false;
- // The server ends the stream now, inside the heartbeat's wait.
- server.drop();
- await vi.advanceTimersByTimeAsync(20_000);
-
- // One open for the start, one for the stream's own reopen, not a third
- // from the heartbeat discarding it.
- expect(server.opens.length).toBe(2);
- abort.abort();
- });
-});
-
-describe('reopening while an open is still in flight', () => {
- /**
- * A server that keeps the incarnation the way the real one does: every open
- * it receives makes a new one, whether or not the client stays to read the
- * answer, and a reattach claiming an older one is refused as a takeover.
- * The next open can be held unanswered, the way a slow connection holds it.
- */
- function incarnationServer() {
- const state = {
- generation: 0,
- opens: 0,
- beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',
- subscribeDrops: false,
- holdNext: false,
- release: (): void => {},
- drop: (): void => {},
- };
- const body = (gen: number, signal: AbortSignal) =>
- new ReadableStream<Uint8Array>({
- start(controller) {
- const close = (): void => {
- try {
- controller.close();
- } catch {
- // already closed
- }
- };
- state.drop = close;
- controller.enqueue(connected(gen));
- signal.addEventListener('abort', close, { once: true });
- },
- });
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (String(url).includes('connection/subscribe')) {
- if (!state.subscribeDrops)
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- state.subscribeDrops = false;
- // The stream ends while the claim is still in flight.
- state.holdNext = true;
- state.drop();
- await new Promise((r) => setTimeout(r, 3_000));
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- if (!String(url).includes('/events')) {
- if (state.beat === 'slow-404-and-drop') {
- // The stream ends while this beat is in flight, and its answer comes
- // back just after the client has started reopening.
- state.beat = 'ok';
- state.holdNext = true;
- state.drop();
- await new Promise((r) => setTimeout(r, 1_200));
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- }
- if (state.beat === 404)
- return { ok: false, status: 404, text: async (): Promise<string> => '' };
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- }
- state.opens += 1;
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) < state.generation)
- return {
- ok: false,
- status: 409,
- body: null,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
- };
- // The server acts on the open the moment it arrives.
- state.generation += 1;
- const gen = state.generation;
- const answer = {
- ok: true,
- status: 200,
- body: body(gen, init.signal),
- text: async (): Promise<string> => '',
- };
- if (!state.holdNext) return answer;
- state.holdNext = false;
- return new Promise((resolve, reject) => {
- state.release = () => resolve(answer);
- init.signal.addEventListener(
- 'abort',
- () => reject(new DOMException('aborted', 'AbortError')),
- { once: true }
- );
- });
- });
- return { state, fetchMock };
- }
-
- it('does not cancel an open the server has already acted on, and so does not stand down', async () => {
- // The self-takeover seen on the pilot. The server closes the stream, the
- // client starts reopening, and a beat sent just before comes back 404.
- // Reopening in answer used to cancel the open already in flight, which
- // the server had acted on, moving the connection to a new incarnation.
- // The client never read that answer, reattached claiming the old one, was
- // refused as a takeover, and stood down for good. The "other client" was
- // its own cancelled request.
- vi.useFakeTimers();
- const { state, fetchMock } = incarnationServer();
- const onEvicted = vi.fn();
- const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });
-
- await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);
- expect(state.generation).toBe(1);
-
- state.beat = 'slow-404-and-drop';
- // Long enough for the refused beat's wait to end while the open is held.
- await vi.advanceTimersByTimeAsync(12_000);
- state.release();
- await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);
-
- expect(onEvicted).not.toHaveBeenCalled();
- expect(state.opens).toBe(2);
- expect(beatGenerations(fetchMock).at(-1)).toBe(2);
- abort.abort();
- });
-
- it('carries out a repoint asked for mid-open once that open is answered', async () => {
- // The other way into the same hole: a room change reopens to include the
- // new room. Mid-open, it must wait for the answer rather than cancel it,
- // and must still happen, since the open in flight was built without it.
- vi.useFakeTimers();
- const { state, fetchMock } = incarnationServer();
- const onEvicted = vi.fn();
- const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
-
- state.subscribeDrops = true;
- const repointing = stream.repoint('room-b');
- // The claim returns while the stream's reopen is held unanswered.
- await vi.advanceTimersByTimeAsync(5_000);
- state.release();
- await vi.advanceTimersByTimeAsync(5_000);
- await repointing;
-
- expect(onEvicted).not.toHaveBeenCalled();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens.at(-1)).toContain('rooms=room-b');
- abort.abort();
- });
-});
-
-describe('a connection another client takes over', () => {
- it('stands down when the room it repoints to is refused, without reopening', async () => {
- // `repoint` claims the room *before* it reopens, so the open's own fence
- // is too late to protect the winner: by the time it refuses, a displaced
- // client has already rewritten the winner's rooms. The claim carries the
- // incarnation for that reason, and a refusal ends this client.
- const opens: string[] = [];
- const subscribes: (number | null)[] = [];
- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
- if (String(url).includes('/events')) {
- opens.push(String(url));
- return {
- ok: true,
- status: 200,
- body: openForever(),
- text: async (): Promise<string> => '',
- };
- }
- if (String(url).includes('connection/subscribe')) {
- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
- subscribes.push(sent.generation);
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'taken_over', message: 'not your connection' } }),
- };
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- const evicted: Eviction[] = [];
- const { stream, abort } = makeStream(fetchMock, {
- rooms: ['!old'],
- onEvicted: (e) => evicted.push(e),
- });
- await flush();
- const opensBefore = opens.length;
-
- await stream.repoint('!new');
- await flush();
-
- // The claim named the incarnation the frame gave us, not nothing.
- expect(subscribes).toEqual([0]);
- expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
- // No reopen: coming back would be the takeover the refusal just denied.
- expect(opens.length).toBe(opensBefore);
- abort.abort();
- });
-
- it('does not claim a room before the frame that names its incarnation', async () => {
- // Claiming nothing is how a client too old to have an incarnation gets
- // through, so a claim sent in the window before the first frame would go
- // through the same door, and that window is exactly when this client may
- // already have been displaced without knowing it.
- let announce: () => void = () => {};
- const subscribes: (number | null)[] = [];
- const fetchMock = vi.fn(async (url: string, init: { body?: string }) => {
- if (String(url).includes('/events'))
- return {
- ok: true,
- status: 200,
- // Attached, but silent: the server has not said which incarnation
- // this socket is yet.
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- announce = () => controller.enqueue(connected());
- },
- }),
- text: async (): Promise<string> => '',
- };
- if (String(url).includes('connection/subscribe')) {
- const sent = JSON.parse(init.body ?? '{}') as { generation: number | null };
- subscribes.push(sent.generation);
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- const { stream, abort } = makeStream(fetchMock, { rooms: ['!old'] });
- await flush();
-
- const repointed = stream.repoint('!new');
- await flush();
-
- expect(subscribes).toEqual([]);
-
- announce();
- await repointed;
-
- expect(subscribes).toEqual([0]);
- abort.abort();
- });
-
- it('halts instead of reopening, because reopening would be a takeover back', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('evicted', {
- code: 'taken_over',
- reason: 'another stream attached to this connection',
- room_id: null,
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(5 * 60_000);
-
- // Two clients reopening each other's connection is the loop this prevents:
- // whoever reopens wins, so a displaced client that reopens starts it again.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(evicted).toEqual([
- {
- code: EVICTION_TAKEN_OVER,
- reason: 'another stream attached to this connection',
- roomId: null,
- },
- ]);
- abort.abort();
- });
-
- it('stands down when its heartbeat is refused, rather than taking the connection back', async () => {
- vi.useFakeTimers();
- // The case the SSE frame cannot cover: this client's socket dropped before
- // the eviction reached it, so the refused heartbeat is the only way it ever
- // hears that it lost.
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' }
- : {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({
- detail: {
- code: 'taken_over',
- message: 'connection conn-1 was reopened since incarnation 0',
- },
- }),
- }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(5 * 60_000);
-
- // One beat and no second open: reopening is a takeover, so a loser that
- // reopens pulls the connection straight back off the client that won it.
- expect(urlsFor(fetchMock, 'connection/beat')).toHaveLength(1);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(evicted.map((e) => e.code)).toEqual([EVICTION_TAKEN_OVER]);
- abort.abort();
- });
-
- it('still reopens on a refusal that names no code, the way an old server sends it', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- }
- : {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: 'connection conn-1 has no stream attached' }),
- }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(5 * 60_000);
-
- // Only the takeover is terminal. Everything else a 409 can mean is still
- // something a reopen fixes, and a server too old to say which is one of them.
- expect(urlsFor(fetchMock, '/events').length).toBeGreaterThan(1);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-
- it('reads an old server’s wording as a takeover when it sends no code', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: frameThenClose('evicted', {
- reason: 'another stream attached to this connection',
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const evicted: Eviction[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(5 * 60_000);
-
- expect(evicted[0]?.code).toBe(EVICTION_TAKEN_OVER);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- abort.abort();
- });
-});
-
-describe('a stream the server closes cleanly', () => {
- /** Opens, closes at once, forever: the shape of a contested connection. */
- function closesAtOnce() {
- return vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.close();
- },
- }),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- }
-
- it('waits before reopening rather than reconnecting at full rate', async () => {
- vi.useFakeTimers();
- const fetchMock = closesAtOnce();
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- // A minute of 1s, 2s, 4s… is seven opens. Without pacing a clean close it
- // is an unbounded spin, limited only by how fast the server can answer.
- await vi.advanceTimersByTimeAsync(60_000);
-
- expect(urlsFor(fetchMock, '/events').length).toBeLessThanOrEqual(8);
- abort.abort();
- });
-
- it('keeps backing off rather than resetting on every handshake', async () => {
- vi.useFakeTimers();
- const fetchMock = closesAtOnce();
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(60_000);
- const early = urlsFor(fetchMock, '/events').length;
- await vi.advanceTimersByTimeAsync(60_000);
- const late = urlsFor(fetchMock, '/events').length - early;
-
- // An open is not evidence of a working stream. Resetting the curve on the
- // handshake held two contending clients at a reconnect a second forever.
- expect(late).toBeLessThan(early);
- abort.abort();
- });
-});
-
-it('does not acknowledge a delivery before the consumer has saved it', async () => {
- let finish: () => void = () => {};
- const saved = new Promise<void>((resolve) => {
- finish = resolve;
- });
- const onEvent = vi.fn(() => saved);
- const fetchMock = vi.fn(async (url: string) => {
- if (!url.includes('/events')) return Response.json({});
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(
- new TextEncoder().encode(
- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"room","sequence":7}\n\n'
- )
- );
- },
- })
- );
- });
- const { stream, abort } = makeStream(fetchMock, { rooms: ['room'], startCursor: 6, onEvent });
- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledOnce());
- expect(stream.position).toEqual({ s: 6 });
- finish();
- await vi.waitFor(() => expect(stream.position).toEqual({ s: 7 }));
- abort.abort();
-});
-
-it('replays from an explicitly saved zero cursor instead of starting at head', async () => {
- const fetchMock = vi
- .fn()
- .mockImplementation(async (url: string) =>
- url.includes('/events?')
- ? { ok: true, body: openForever() }
- : { ok: true, json: async () => ({ rooms: [] }) }
- );
- const { abort } = makeStream(fetchMock, { rooms: [], startCursor: 0 });
- try {
- await flush();
- expect(new URL(urlsFor(fetchMock, '/events?')[0]).searchParams.get('start_from')).toBe('0');
- } finally {
- abort.abort();
- }
-});
-
-function resetFrames(url: string): Response {
- if (!url.includes('/events')) return Response.json({});
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(
- new TextEncoder().encode(
- 'event: gap\ndata: {"from_sequence":0,"resumed_at":0,"reason":"buffer reset"}\n\n' +
- 'id: 1\nevent: message\ndata: {"type":"message","room_id":"room","sequence":1}\n\n'
- )
- );
- },
- })
- );
-}
-
-it('waits for durable reset checkpoint before advancing the cursor or delivering', async () => {
- let finish!: () => void;
- const saved = new Promise<void>((resolve) => {
- finish = resolve;
- });
- const onGap = vi.fn(() => saved);
- const onEvent = vi.fn();
- const { stream, abort } = makeStream(
- vi.fn(async (url: string) => resetFrames(url)),
- { rooms: ['room'], startCursor: 4, onGap, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
- expect(onGap).toHaveBeenCalledWith({
- fromSequence: 0,
- resumedAt: 0,
- cursorReset: true,
- reason: 'buffer reset',
- });
- expect(stream.position).toEqual({ s: 4 });
- expect(onEvent).not.toHaveBeenCalled();
- finish();
- await vi.waitFor(() => expect(stream.position).toEqual({ s: 1 }));
- expect(onEvent).toHaveBeenCalledOnce();
- } finally {
- abort.abort();
- }
-});
-
-it('does not advance or deliver when persisting the reset checkpoint fails', async () => {
- const onGap = vi.fn(async () => {
- throw new Error('journal unavailable');
- });
- const onEvent = vi.fn();
- const { stream, abort } = makeStream(
- vi.fn(async (url: string) => resetFrames(url)),
- { rooms: ['room'], startCursor: 4, onGap, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onGap).toHaveBeenCalledOnce());
- expect(stream.position).toEqual({ s: 4 });
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-describe('the permission to start a session', () => {
- function reopenable() {
- return vi.fn(async (url: string, init: { signal: AbortSignal }) =>
- String(url).includes('/events')
- ? {
- ok: true,
- status: 200,
- body: openUntilAborted(init),
- text: async (): Promise<string> => '',
- }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- }
-
- it('is redeclared on the wire when it changes under a live connection', async () => {
- // Turning automatic sessions off while the controller is connected has to
- // reach the server, or it goes on promising a session nothing will start.
- const fetchMock = reopenable();
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
- await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- expect(urlsFor(fetchMock, '/events')[0]).toContain('spawn_capable=true');
-
- stream.setSpawnCapable(false);
- await flush();
-
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(2);
- expect(opens[1]).not.toContain('spawn_capable');
- // A reattach, not a takeover: the incarnation goes with it, so the server
- // recognises the same client rather than evicting it in favour of itself.
- expect(opens[1]).toContain('expected_generation=0');
- abort.abort();
- });
-
- /**
- * A server that fences reattaches the way the real one does: every accepted
- * open makes a new incarnation, and an open claiming an older one is refused.
- * `holdStateOn` withholds the frame naming the incarnation for one open, so a
- * test can act in the window where the client has not been told yet.
- */
- function fencingServer(holdStateOn: number) {
- let generation = 0;
- let opens = 0;
- const announce: Array<() => void> = [];
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- const claimed = new URL(String(url)).searchParams.get('expected_generation');
- if (claimed !== null && Number(claimed) !== generation)
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: EVICTION_TAKEN_OVER } }),
- };
- opens += 1;
- generation += 1;
- const held = opens === holdStateOn;
- const named = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- let closed = false;
- const send = () => {
- if (!closed) controller.enqueue(connected(named));
- };
- if (held) announce.push(send);
- else send();
- init.signal.addEventListener(
- 'abort',
- () => {
- closed = true;
- controller.close();
- },
- { once: true }
- );
- },
- }),
- text: async (): Promise<string> => '',
- };
- });
- return { fetchMock, announce };
- }
-
- it('holds a second change until the server has answered the first', async () => {
- // Redeclaring reattaches, which claims the incarnation this client believes
- // it holds. A second change sent before the first open was answered claims
- // the one before it, which the server has moved past, and refuses as a
- // takeover. That would take the agent's only connection off the air over
- // two clicks in a row.
- const onEvicted = vi.fn();
- const { fetchMock, announce } = fencingServer(2);
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true, onEvicted });
- try {
- await flush();
- stream.setSpawnCapable(false);
- await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
-
- // The reopen is out but unanswered, so this one has nothing safe to claim.
- stream.setSpawnCapable(true);
- await flush();
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
-
- announce[0]();
- await flush();
-
- // Without the hold, this is where the connection has already gone: the
- // third open claims the incarnation the second one replaced.
- expect(onEvicted).not.toHaveBeenCalled();
- const opens = urlsFor(fetchMock, '/events');
- expect(opens).toHaveLength(3);
- expect(opens[2]).toContain('spawn_capable=true');
- expect(opens[2]).toContain('expected_generation=2');
- } finally {
- abort.abort();
- }
- });
-
- it('carries nothing when a change made in that window is taken back', async () => {
- const { fetchMock, announce } = fencingServer(2);
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
- try {
- await flush();
- stream.setSpawnCapable(false);
- await flush();
- stream.setSpawnCapable(true);
- stream.setSpawnCapable(false);
- await flush();
-
- announce[0]();
- await flush();
-
- // The held open already declares what the setting settled back to.
- expect(urlsFor(fetchMock, '/events')).toHaveLength(2);
- } finally {
- abort.abort();
- }
- });
-
- it('does not reopen the socket when it is set to what it already is', async () => {
- const fetchMock = reopenable();
- const { stream, abort } = makeStream(fetchMock, { rooms: [], spawnCapable: true });
- await flush();
-
- stream.setSpawnCapable(true);
- await flush();
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
- abort.abort();
- });
-});
-
-it('routes command wakeups separately from room events and their cursor', async () => {
- const onCommands = vi.fn();
- const onEvent = vi.fn();
- const fetchMock = vi.fn(async (url: string) =>
- url.includes('/events')
- ? new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- controller.enqueue(encodeFrame('session_commands', { session_ids: ['session'] }));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- )
- : Response.json({})
- );
- const { abort } = makeStream(fetchMock, { rooms: [], scope: 'all', onCommands, onEvent });
- try {
- await vi.waitFor(() => expect(onCommands).toHaveBeenCalledWith(['session']));
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-function streaming(...frames: Uint8Array[]) {
- return vi.fn(async (url: string) =>
- url.includes('/events')
- ? new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- for (const frame of frames) controller.enqueue(frame);
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- )
- : Response.json({})
- );
-}
-
-it('hands an approval outcome to its own callback, not the room-event path', async () => {
- const onApprovalOutcome = vi.fn();
- const onEvent = vi.fn();
- const outcome = {
- session_id: 'session',
- request_id: 'permission',
- state: 'answered',
- answer: '0',
- answered_by: '@person:test',
- answered_at: '2026-09-24T12:00:00Z',
- };
- const { abort } = makeStream(streaming(encodeFrame('approval_outcome', outcome)), {
- rooms: [],
- scope: 'all',
- onApprovalOutcome,
- onEvent,
- });
- try {
- await vi.waitFor(() => expect(onApprovalOutcome).toHaveBeenCalledWith(outcome));
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-it('drops a frame it does not know that is not a room event', async () => {
- const onEvent = vi.fn();
- const { abort } = makeStream(
- streaming(
- encodeFrame('something_new', { detail: 'from a later server' }),
- encodeFrame('approval_outcome', { session_id: 'session' }),
- encodeFrame('message', { type: 'message', room_id: 'room', payload: {} })
- ),
- { rooms: [], scope: 'all', onEvent }
- );
- try {
- await vi.waitFor(() => expect(onEvent).toHaveBeenCalledTimes(1));
- expect(onEvent).toHaveBeenCalledWith({ type: 'message', room_id: 'room', payload: {} });
- } finally {
- abort.abort();
- }
-});
-
-it('hands a relayed session command to its own callback', async () => {
- const onSessionCommand = vi.fn();
- const onEvent = vi.fn();
- const command = {
- contractVersion: 1,
- commandId: 'command-1',
- sessionId: 'session',
- epoch: 'epoch',
- origin: { surface: 'console', actorId: 'owner', roomId: null, threadId: null, messageId: null },
- body: { type: 'turn.interrupt', turnId: 'turn' },
- };
- const { abort } = makeStream(
- streaming(
- encodeFrame('session_command', { commandId: 'no-session' }),
- encodeFrame('session_command', command)
- ),
- { rooms: [], scope: 'all', onSessionCommand, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onSessionCommand).toHaveBeenCalledWith(command));
- expect(onSessionCommand).toHaveBeenCalledTimes(1);
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-it('hands a room another connection took over to its own callback', async () => {
- const onRoomReleased = vi.fn();
- const onEvent = vi.fn();
- const { abort } = makeStream(
- streaming(
- encodeFrame('room_released', { session_id: 'no-room' }),
- encodeFrame('room_released', { room_id: 'room', session_id: 'session' }),
- encodeFrame('room_released', { room_id: 'other', session_id: null })
- ),
- { rooms: [], scope: 'all', onRoomReleased, onEvent }
- );
- try {
- await vi.waitFor(() => expect(onRoomReleased).toHaveBeenCalledTimes(2));
- expect(onRoomReleased).toHaveBeenNthCalledWith(1, { roomId: 'room', sessionId: 'session' });
- expect(onRoomReleased).toHaveBeenNthCalledWith(2, { roomId: 'other', sessionId: null });
- expect(onEvent).not.toHaveBeenCalled();
- } finally {
- abort.abort();
- }
-});
-
-it('says when the server has confirmed an open', async () => {
- const onConnected = vi.fn();
- const { abort } = makeStream(streaming(), { rooms: [], scope: 'all', onConnected });
- try {
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
- } finally {
- abort.abort();
- }
-});
-
-it('says when an open stream ended and it is trying again', async () => {
- const onConnected = vi.fn();
- const onDisconnected = vi.fn();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!url.includes('/events')) return Response.json({});
- opens += 1;
- if (opens === 1)
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected());
- controller.close();
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- return new Response(openForever(), { headers: { 'content-type': 'text/event-stream' } });
- });
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onConnected,
- onDisconnected,
- });
- try {
- await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
- expect(onDisconnected).toHaveBeenCalledWith({ error: 'the server closed the stream' });
- expect(onConnected).toHaveBeenCalledTimes(1);
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
- expect(onDisconnected).toHaveBeenCalledTimes(1);
- } finally {
- abort.abort();
- }
-});
-
-it('states placements on the attached incarnation, and raises on a refusal', async () => {
- const bodies: unknown[] = [];
- let refuse = false;
- const fetchMock = vi.fn(async (url: string, init?: { body?: string }) => {
- if (url.includes('/events'))
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected(4));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- if (url.includes('connection/placements')) {
- bodies.push(JSON.parse(init!.body!));
- return refuse ? new Response('not a member', { status: 403 }) : Response.json({ ok: true });
- }
- return Response.json({});
- });
- const { stream, abort } = makeStream(fetchMock, { rooms: [], scope: 'all' });
- try {
- await stream.replacePlacements({ session: 'room' });
- expect(bodies).toEqual([
- { connection_id: 'conn-1', placements: { session: 'room' }, generation: 4 },
- ]);
- refuse = true;
- await expect(stream.replacePlacements({ session: 'elsewhere' })).rejects.toThrow('HTTP 403');
- } finally {
- abort.abort();
- }
-});
-
-describe('test_gap_above_cursor_is_not_cursor_reset', () => {
- it('reports a restart gap resuming above the cursor as no cursor reset', async () => {
- const body = new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected());
- controller.enqueue(
- new TextEncoder().encode(
- 'id: 7\nevent: message\ndata: {"type":"message","room_id":"r"}\n\n'
- )
- );
- controller.enqueue(
- encodeFrame('gap', {
- from_sequence: 7,
- resumed_at: 2 ** 32 - 1,
- reason: 'server restarted',
- rooms: [],
- })
- );
- },
- });
- const fetchMock = vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body, text: async (): Promise<string> => '' }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];
- const { stream, abort } = makeStream(fetchMock, { rooms: [], onGap: (g) => void gaps.push(g) });
- await flush();
- expect(gaps).toEqual([expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false })]);
- expect(stream.position).toEqual({ s: 2 ** 32 - 1 });
- abort.abort();
- });
-});
-
-it('does not give its connection up to itself over a request answered across its own reopen', async () => {
- // Seen on a real host: placements stated under incarnation 4, the stream
- // reopened (to 5) before the answer came, and the answer, "taken over:
- // reopened since 4, now at 5", named this very client as the new holder.
- // It stood down for good, with nobody else anywhere near the connection.
- const opened: ReadableStreamDefaultController<Uint8Array>[] = [];
- let answerPlacements: (response: Response) => void = () => {};
- const fetchMock = vi.fn(async (url: string) => {
- if (url.includes('/events')) {
- const generation = 4 + opened.length;
- return new Response(
- new ReadableStream({
- start(controller) {
- opened.push(controller);
- controller.enqueue(connected(generation));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- }
- if (url.includes('connection/placements'))
- return new Promise<Response>((resolve) => {
- answerPlacements = resolve;
- });
- return Response.json({});
- });
- const evicted: Eviction[] = [];
- const onConnected = vi.fn();
- const { stream, abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onConnected,
- onEvicted: (eviction) => evicted.push(eviction),
- });
- try {
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
- const stating = stream.replacePlacements({ session: 'room' });
- await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));
-
- opened[0]!.close();
- await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
- answerPlacements(
- Response.json(
- {
- detail: {
- code: 'taken_over',
- message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',
- },
- },
- { status: 409 }
- )
- );
-
- await expect(stating).rejects.toThrow('409');
- expect(evicted).toEqual([]);
- } finally {
- abort.abort();
- }
-});
-
-it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {
- const fetchMock = vi.fn(async (url: string) => {
- if (url.includes('/events'))
- return new Response(
- new ReadableStream({
- start(controller) {
- controller.enqueue(connected(4));
- },
- }),
- { headers: { 'content-type': 'text/event-stream' } }
- );
- if (url.includes('connection/placements'))
- return Response.json(
- { detail: { code: 'taken_over', message: 'another client holds it' } },
- { status: 409 }
- );
- return Response.json({});
- });
- const evicted: Eviction[] = [];
- const { stream, abort } = makeStream(fetchMock, {
- rooms: [],
- scope: 'all',
- onEvicted: (eviction) => evicted.push(eviction),
- });
- try {
- await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');
- expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);
- } finally {
- abort.abort();
- }
-});
-
-describe('the heartbeat across its own reopen after a refusal', () => {
- /**
- * What took every in-Console watcher off the air when a laptop woke
- * (2026-10-01, four agents within ten seconds). The connection had lapsed
- * while the machine slept, so the first beat after waking was refused with a
- * 404. The heartbeat asked the stream to reopen and went straight back to
- * its gate, which it had found open a moment before the stream loop closed
- * it for the reopen. One beat went out under the lapsed incarnation right
- * behind the open that replaced it. The server, holding the new incarnation,
- * answered that beat `taken_over`, and the client believed it: it stood down
- * over a connection it had just made itself. In the server log the open and
- * the stale beat arrived 11 ms apart, every time.
- */
- function lapsingServer() {
- let generation = 0;
- let lapsed = false;
- const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {
- if (String(url).includes('/events')) {
- generation += 1;
- lapsed = false;
- const gen = generation;
- return {
- ok: true,
- status: 200,
- body: new ReadableStream<Uint8Array>({
- start(controller) {
- controller.enqueue(connected(gen));
- init.signal.addEventListener(
- 'abort',
- () => {
- try {
- controller.error(new DOMException('The operation was aborted.', 'AbortError'));
- } catch {
- // Already closed.
- }
- },
- { once: true }
- );
- },
- }),
- text: async (): Promise<string> => '',
- };
- }
- if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };
- const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;
- if (claimed !== null && claimed !== generation) {
- return {
- ok: false,
- status: 409,
- text: async (): Promise<string> =>
- JSON.stringify({
- detail: {
- code: 'taken_over',
- message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,
- },
- }),
- };
- }
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- });
- /** The server sweeps the connection for silence; its socket stays up
- * client-side, the way a socket does across a sleep. */
- const lapse = (): void => {
- lapsed = true;
- };
- return { fetchMock, lapse };
- }
-
- it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {
- vi.useFakeTimers();
- const server = lapsingServer();
- const evicted: Eviction[] = [];
- const { abort } = makeStream(server.fetchMock, {
- rooms: [],
- onEvicted: (eviction) => evicted.push(eviction),
- });
-
- await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);
- server.lapse();
- await vi.advanceTimersByTimeAsync(30_000);
-
- // The beat refused with a 404 is the last one under the lapsed
- // incarnation: what follows it on the wire is the reopen, not another beat
- // that the server could only take for someone else's.
- const sent = server.fetchMock.mock.calls.map((call) =>
- String(call[0]).includes('/events')
- ? 'open'
- : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`
- );
- const reopen = sent.indexOf('open', 1);
- expect(reopen).toBeGreaterThan(0);
- expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);
- expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);
- expect(evicted).toEqual([]);
- abort.abort();
- });
-});
-
-describe('choosing the transport', () => {
- /** A stream body that announces a server speaking `protocol` (none for a
- * server from before the declaration), then ends. */
- function announcingThenClose(protocol?: number): ReadableStream<Uint8Array> {
- return frameThenClose('connection_state', {
- connection_id: 'conn-1',
- generation: 0,
- ...(protocol === undefined ? {} : { protocol }),
- });
- }
-
- function serving(body: () => ReadableStream<Uint8Array>) {
- return vi.fn(async (url: string) =>
- String(url).includes('/events')
- ? { ok: true, status: 200, body: body(), text: async (): Promise<string> => '' }
- : { ok: true, status: 200, text: async (): Promise<string> => '' }
- );
- }
-
- // 6 is a server before the hosted worker, 7 one with it: neither has a socket.
- it.each([6, 7])(
- 'falls back to the event stream, then opens it straight away for a server speaking %i',
- async (protocol) => {
- vi.useFakeTimers();
- const fetchMock = serving(() => announcingThenClose(protocol));
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- // The stream ends at once each time: opens at 0, 1 and 3 seconds.
- await vi.advanceTimersByTimeAsync(4000);
-
- expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
- expect(SocketThatNeverOpens.opened).toHaveLength(1);
- abort.abort();
- }
- );
-
- it('declares a range a server from before the socket shares', async () => {
- const fetchMock = serving(() => openForever());
- const { abort } = makeStream(fetchMock, { rooms: [] });
- await flush();
-
- const params = new URL(urlsFor(fetchMock, '/events')[0]!).searchParams;
- expect(params.get('protocol')).toBe('9');
- expect(Number(params.get('protocol_accepts'))).toBeLessThanOrEqual(6);
- abort.abort();
- });
-
- it('says the stream it asks for after a socket that never opened is a fallback, and opening it straight away is not', async () => {
- vi.useFakeTimers();
- // A server from before the socket: the first open falls back, the next
- // goes to the stream straight away.
- const fetchMock = serving(() => announcingThenClose(6));
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(1500);
-
- const [fellBack, direct] = urlsFor(fetchMock, '/events').map(
- (url) => new URL(url).searchParams
- );
- expect(fellBack!.get('fallback')).toBe('socket_never_opened');
- expect(fellBack!.get('fallback_close_code')).toBe('1006');
- expect(direct!.has('fallback')).toBe(false);
- expect(direct!.has('fallback_close_code')).toBe(false);
- abort.abort();
- });
-
- it('keeps trying the socket first when neither answers, and says it could not connect', async () => {
- vi.useFakeTimers();
- const fetchMock = vi.fn(async () => {
- throw new TypeError('fetch failed');
- });
- const disconnected: string[] = [];
- const { abort } = makeStream(fetchMock, {
- rooms: [],
- onDisconnected: ({ error }) => disconnected.push(error),
- });
-
- await vi.advanceTimersByTimeAsync(4000);
-
- // A network failure decides nothing: every open still starts with the socket.
- expect(SocketThatNeverOpens.opened).toHaveLength(3);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
- expect(disconnected[0]).toContain('could not connect');
- abort.abort();
- });
-
- it('goes back to the socket once the server says it speaks a revision with one', async () => {
- vi.useFakeTimers();
- // The socket failed to open (a proxy, say) but the server is a new one.
- const fetchMock = serving(() => announcingThenClose(8));
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- await vi.advanceTimersByTimeAsync(2000);
-
- expect(SocketThatNeverOpens.opened).toHaveLength(2);
- abort.abort();
- });
-
- it('moves to the socket at once when the stream it went to straight away has been removed', async () => {
- vi.useFakeTimers();
- let opens = 0;
- const fetchMock = vi.fn(async (url: string) => {
- if (!String(url).includes('/events'))
- return { ok: true, status: 200, text: async (): Promise<string> => '' };
- opens += 1;
- if (opens === 1)
- return {
- ok: true,
- status: 200,
- body: announcingThenClose(6),
- text: async (): Promise<string> => '',
- };
- return {
- ok: false,
- status: 410,
- body: null,
- text: async (): Promise<string> =>
- JSON.stringify({ detail: { code: 'transport_removed', message: 'gone' } }),
- };
- });
- const { abort } = makeStream(fetchMock, { rooms: [] });
-
- // The first stream ends at once; the next open, a second later, is the
- // stream again and is refused as removed.
- await vi.advanceTimersByTimeAsync(1500);
- expect(SocketThatNeverOpens.opened).toHaveLength(2);
-
- // Then the backoff, not a spin between the two.
- await vi.advanceTimersByTimeAsync(1000);
- expect(SocketThatNeverOpens.opened).toHaveLength(2);
- expect(urlsFor(fetchMock, '/events')).toHaveLength(3);
- abort.abort();
- });
-});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
index d3773816e..6b6698167 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -1489,3 +1489,53 @@ describe('placements', () => {
}
});
});
+
+describe('what connection_state says', () => {
+ it('tells a session start to the room itself only where the server says it leaves that to it', async () => {
+ for (const [said, expected] of [
+ [{ announce_session_starts: true }, true],
+ [{}, false],
+ ] as const) {
+ server.sockets = [];
+ serve((socket) => {
+ socket.open();
+ socket.frame('connection_state', { connection_id: 'conn-1', generation: 0, ...said });
+ });
+ let connected = false;
+ const { stream, abort } = makeStream({
+ rooms: [],
+ onConnected: () => {
+ connected = true;
+ },
+ });
+ expect(stream.announcesSessionStarts).toBe(false);
+ await vi.waitFor(() => expect(connected).toBe(true));
+ expect(stream.announcesSessionStarts).toBe(expected);
+ abort.abort();
+ }
+ });
+
+ it('reports a restart gap resuming above the cursor as no cursor reset', async () => {
+ serve((socket) => {
+ attach(socket, 0);
+ socket.frame('message', { type: 'message', room_id: 'r' }, 7);
+ socket.frame('gap', {
+ from_sequence: 7,
+ resumed_at: 2 ** 32 - 1,
+ reason: 'server restarted',
+ rooms: [],
+ });
+ });
+ const gaps: { cursorReset?: boolean; resumedAt?: number }[] = [];
+ const { stream, abort } = makeStream({ rooms: [], onGap: (g) => void gaps.push(g) });
+ try {
+ await vi.waitFor(() => expect(gaps).toHaveLength(1));
+ expect(gaps).toEqual([
+ expect.objectContaining({ resumedAt: 2 ** 32 - 1, cursorReset: false }),
+ ]);
+ expect(stream.position).toEqual({ s: 2 ** 32 - 1 });
+ } finally {
+ abort.abort();
+ }
+ });
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index daf68b402..494f5cb91 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -1,7 +1,6 @@
import { contractRange } from './artifacts';
import { furthest, OWN_EVENTS, type Position, readPosition } from './positions';
import { ReattachFence } from './reattach-fence';
-import { readSse } from './sse';
import type { AgentBridgeEvent, SwitchCredentials } from './types';
import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';
@@ -19,13 +18,6 @@ const AGENT_PROTOCOL = contractRange('agent-protocol', RUNTIME_ARTIFACT);
* end the connection, so reconnecting within the heartbeat TTL keeps the room
* slot and the role lease and resumes from the cursor.
*
- * A server from before the socket (agent-protocol 7 and older) has none, and
- * is spoken to the way this runtime used to: the same frames as a Server-Sent
- * Events stream, and a heartbeat we send ourselves, `POST /connection/beat`
- * every two seconds with the cursor. Which one is decided on each open (see
- * `openFrames`). That path is kept for a compatibility window and goes when
- * those servers are gone.
- *
* Each event frame says where the event is numbered, and on reconnect we tell
* the server how far we have read, so it sends exactly the events we missed. A
* server that speaks agent-protocol 9 tracks this as a position: one number for
@@ -36,27 +28,8 @@ const AGENT_PROTOCOL = contractRange('agent-protocol', RUNTIME_ARTIFACT);
* re-read context rather than carry on believing it saw everything.
*/
-/** How long a request on the side (a room claim, placements, a beat) may take. */
+/** How long a request on the side (a room claim, placements) may take. */
const REQUEST_TIMEOUT_MS = 4000;
-/** Cadence of the heartbeat we send on the event stream. Must stay well inside
- * the server's 6s TTL: the server declares the connection dead without it. The
- * socket needs none; the server pings it. */
-export const BEAT_INTERVAL_MS = 2000;
-/**
- * How long a reopen waits for a beat already in flight before disowning it.
- *
- * A beat answered inside this window is still believed, so a takeover that
- * lands while the stream happens to be reopening is acted on. Past it the
- * answer cannot be told apart from one our own reopen provoked, and the reopen
- * matters more: it is what restores delivery, and the server gives the
- * connection six seconds without a beat.
- */
-export const BEAT_SETTLE_LIMIT_MS = 1000;
-/** The agent-protocol revision that brought the socket. A server that speaks
- * an older one has no socket, only the event stream. */
-const SOCKET_PROTOCOL_REVISION = 8;
-/** The code a server answers the event stream with once it has removed it. */
-const TRANSPORT_REMOVED = 'transport_removed';
const INITIAL_BACKOFF_MS = 1000;
const MAX_BACKOFF_MS = 30_000;
/**
@@ -375,21 +348,6 @@ export class OpenRefused extends Error {
/** One frame off the socket: `{event, data, id}`, `id` being its sequence number. */
export type SocketFrame = { event: string; id?: string; data: Record<string, unknown> };
-/** What a connection's frames travel over: the socket, or the event stream an
- * old server offers instead. */
-type Transport = 'socket' | 'stream';
-
-/** The socket closed before it opened: the server is unreachable, or has no
- * socket to offer. Only a request to the event stream tells the two apart. */
-class SocketNeverOpened extends Error {
- constructor(readonly code: number | null) {
- super(
- `could not connect to Switch (socket closed with ${code ?? 'no code'}, and the event stream did not answer either): the server is unreachable`
- );
- this.name = 'SocketNeverOpened';
- }
-}
-
/** Node's WebSocket, which, unlike a browser's, takes request headers. */
type HeaderedWebSocket = new (url: string, init: { headers: Record<string, string> }) => WebSocket;
@@ -453,31 +411,6 @@ export class SwitchEventStream {
/** Until when the server is taken to be restarting (see `SERVICE_RESTART`). */
private restartingUntil = 0;
private openCarries = 0;
- /**
- * Whether to open the event stream rather than try the socket first.
- *
- * Set when a server we reached over the stream says it speaks a revision
- * from before the socket, so each reopen does not first knock on a door that
- * is not there. Cleared as soon as the server says otherwise: a
- * `connection_state` naming a later revision, or the stream refused as
- * removed. A socket that merely fails to open never sets it; only an answer
- * from the server does, so a network failure cannot leave us on the stream.
- */
- private preferStream = false;
- /** What the open in progress travels over, and what the attached one does. */
- private openingOver: Transport = 'socket';
- private attachedOver: Transport | null = null;
- /**
- * How many times the server has named an incarnation for this client, and
- * how to cut the heartbeat's current wait short when it does.
- *
- * A fresh attach starts the server's 6-second clock on a connection that has
- * never beaten, so on the event stream the heartbeat must go out promptly,
- * whatever back-off an earlier, failing connection had built up. See
- * `beatLoop`.
- */
- private attaches = 0;
- private wakeBeat: () => void = () => {};
constructor(deps: SwitchEventStreamDeps) {
this.deps = deps;
@@ -515,7 +448,6 @@ export class SwitchEventStream {
start(): void {
void this.streamLoop();
- void this.beatLoop();
}
/** Repoint the stream at a different room without dropping the connection.
@@ -595,15 +527,6 @@ export class SwitchEventStream {
this.socketAbort?.abort();
}
- /**
- * Reopen only to get attached again, the heartbeat's reason. An open already
- * in flight does exactly that, so there is nothing to add to it; and one
- * scheduled after a dropped stream carries the request anyway.
- */
- private reattach(): void {
- if (this.fence.admitting) this.reopen();
- }
-
/**
* Pass the server's room list on, and keep our own copy in step.
*
@@ -844,11 +767,9 @@ export class SwitchEventStream {
const socketAbort = new AbortController();
this.socketAbort = socketAbort;
// Closed before the open rather than after it lands: it is this open
- // that makes the new incarnation, so a reopen and the event stream's
- // heartbeat have to wait from here until the frame naming it arrives. A
- // beat already in flight is given a moment to come back and be
- // believed, and disowned after that. The socket has none in flight.
- await this.fence.detaching(BEAT_SETTLE_LIMIT_MS);
+ // that makes the new incarnation, so a reopen has to wait from here
+ // until the frame naming it arrives.
+ await this.fence.detaching(0);
if (signal.aborted || this.halt.signal.aborted) return;
let openedAt = 0;
let failure: unknown = null;
@@ -888,7 +809,8 @@ export class SwitchEventStream {
// deliberate takeover still works: it has no incarnation to claim.
if (this.generation !== null) params.set('expected_generation', String(this.generation));
- for await (const frame of this.openFrames(params, socketAbort.signal, () => {
+ const url = `${creds.apiEndpoint.replace(/^http/, 'ws')}/agents/${creds.agentId}/connection/ws?${params}`;
+ for await (const frame of this.readSocket(url, socketAbort.signal, () => {
openedAt = Date.now();
this.restartingUntil = 0;
log.debug('SwitchEventStream: connection open', {
@@ -919,10 +841,6 @@ export class SwitchEventStream {
this.standDown();
return;
}
- // The event stream we went to straight away is gone, so the server
- // has the socket now (`openFrames` has stopped preferring the
- // stream). A different request from the one refused, so retry at once.
- if (status === 410 && refusalCode(body) === TRANSPORT_REMOVED) continue;
}
failure = error;
}
@@ -963,91 +881,6 @@ export class SwitchEventStream {
}
}
- /**
- * Open the connection and yield its frames, over the socket if the server
- * has one and over the event stream if it does not.
- *
- * Node's WebSocket does not say why a handshake failed, so a server with no
- * socket (it answers the upgrade with 404) looks exactly like one we cannot
- * reach. The event stream is what tells them apart: a socket that never
- * opened is followed at once by a request for the stream, and a server that
- * answers it is one we can talk to that way. Decided on each open rather
- * than once for good, so a network failure costs one request and decides
- * nothing, and a server upgraded under us is moved back onto the socket.
- */
- private async *openFrames(
- params: URLSearchParams,
- abort: AbortSignal,
- onOpen: () => void
- ): AsyncGenerator<SocketFrame> {
- const { creds } = this.deps;
- if (!this.preferStream) {
- this.openingOver = 'socket';
- const url = `${creds.apiEndpoint.replace(/^http/, 'ws')}/agents/${creds.agentId}/connection/ws?${params}`;
- try {
- yield* this.readSocket(url, abort, onOpen);
- return;
- } catch (error) {
- if (!(error instanceof SocketNeverOpened) || abort.aborted) throw error;
- this.openingOver = 'stream';
- yield* this.readEventStream(params, abort, onOpen, error);
- return;
- }
- }
- this.openingOver = 'stream';
- yield* this.readEventStream(params, abort, onOpen, null);
- }
-
- /**
- * Open the connection as a Server-Sent Events stream, the way a server from
- * before the socket serves it, and yield its frames until it ends.
- *
- * `probing` is the socket's failure when this request is the one that tells
- * an old server from an unreachable one. Then anything short of a stream
- * means unreachable, and that failure is what is reported. A refusal is
- * still the server's answer, and handled as one. The request also says it
- * is a fallback, and the socket's close code, so the server's log can tell
- * it from a client built before the socket.
- */
- private async *readEventStream(
- params: URLSearchParams,
- abort: AbortSignal,
- onOpen: () => void,
- probing: SocketNeverOpened | null
- ): AsyncGenerator<SocketFrame> {
- const { creds, signal } = this.deps;
- const query = new URLSearchParams(params);
- if (probing) {
- query.set('fallback', 'socket_never_opened');
- if (probing.code !== null) query.set('fallback_close_code', String(probing.code));
- }
- let resp: Response;
- try {
- resp = await fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/events?${query}`, {
- headers: {
- Authorization: `Bearer ${creds.token}`,
- Accept: 'text/event-stream',
- ...(this.cursor > 0 ? { 'Last-Event-ID': String(this.cursor) } : {}),
- },
- signal: AbortSignal.any([abort, signal, this.halt.signal]),
- });
- } catch (error) {
- throw probing ?? error;
- }
- if (!resp.ok || !resp.body) {
- const body = await resp.text();
- if (resp.status === 410 && refusalCode(body) === TRANSPORT_REMOVED) {
- // A server that has removed the stream has the socket.
- this.preferStream = false;
- throw probing ?? new OpenRefused(resp.status, body);
- }
- if (probing && resp.ok) throw probing;
- throw new OpenRefused(resp.status, body);
- }
- onOpen();
- yield* readSse(resp.body, abort);
- }
-
/**
* Open the connection's socket and yield its frames until it closes.
*
@@ -1132,7 +965,11 @@ export class SwitchEventStream {
JSON.stringify({ detail: refused ?? ending.reason })
);
}
- if (!opened && !abort.aborted) throw new SocketNeverOpened(ending?.code ?? null);
+ if (!opened && !abort.aborted) {
+ throw new Error(
+ `could not connect to Switch (socket closed with ${ending?.code ?? 'no code'}): the server is unreachable, or older than this runtime and has no agent WebSocket`
+ );
+ }
} finally {
abort.removeEventListener('abort', stop);
if (socket.readyState === socket.CONNECTING || socket.readyState === socket.OPEN) {
@@ -1157,19 +994,10 @@ export class SwitchEventStream {
const startedAt = readPosition(frame.data.position);
if (startedAt) this._position = furthest(this._position, startedAt);
this.announces = frame.data.announce_session_starts === true;
- this.attachedOver = this.openingOver;
- // Reached over the stream: whether the server has a socket for next
- // time. One that names no revision is older than the declaration, and
- // so older than the socket.
- if (this.openingOver === 'stream') {
- const speaks = frame.data.protocol;
- this.preferStream = !(typeof speaks === 'number' && speaks >= SOCKET_PROTOCOL_REVISION);
- }
log.debug('SwitchEventStream: connection established', {
event: 'switch_stream_connected',
rooms: frame.data.rooms,
generation: this.generation,
- transport: this.attachedOver,
// What the server says it is (CHOO-1865). Recorded, not acted on —
// logging it is what makes "which versions are actually talking to
// each other" answerable from a bug report rather than a guess.
@@ -1185,8 +1013,6 @@ export class SwitchEventStream {
return;
}
this.fence.attached();
- this.attaches += 1;
- this.wakeBeat();
this.redeclare();
this.deps.onConnected?.();
return;
@@ -1320,218 +1146,6 @@ export class SwitchEventStream {
(frame.id ? { [OWN_EVENTS]: Number(frame.id) || 0 } : null);
if (eventPosition) this._position = furthest(this._position, eventPosition);
}
-
- /**
- * The event stream's heartbeat. Proves we are alive and reports the cursor.
- *
- * Only while the connection is attached over the event stream: the socket is
- * pinged by the server and answers from `readSocket`, and a beat beside it
- * would be a request every two seconds for nothing.
- *
- * A 404 or 409 means we are not receiving: the connection expired, or it has
- * no stream attached. Both are recovered by reopening, which resumes from the
- * cursor, and both count as a beat that did not land. The exception is the
- * 409 that says another client has taken the connection over, which no
- * reopen recovers because a reopen is itself a takeover: that one ends here.
- * For the rest the remedy is a reopen, and a reopen that keeps being refused
- * is a client that cannot succeed and must not be retried at full rate
- * forever. Failing quietly here is the one thing that must not happen: a
- * client that has stopped receiving while believing it is connected is
- * exactly the bug this transport exists to remove.
- *
- * While beats succeed the cadence is fixed and short: the server declares the
- * connection dead without them. While they fail it backs off, because a beat
- * that has already missed the TTL cannot save the connection: reopening the
- * stream is what re-establishes it, and that loop is doing its own retrying.
- * Hammering a dead endpoint every two seconds forever is what this avoids:
- * an endpoint that is simply gone (a managed server's port after the stack was
- * destroyed) should cost a trickle of requests and a handful of log lines, not
- * a permanent stream of both.
- */
- private async beatLoop(): Promise<void> {
- const { log, signal, connectionId } = this.deps;
- let failures = 0;
- let backoff = BEAT_INTERVAL_MS;
- // Rejections in a row, which pace reopening rather than beating. Kept
- // apart from `backoff` because a fresh attach resets that and must not
- // reset this: see the 404/409 branch below.
- let rejections = 0;
- let seenAttach = this.attaches;
-
- /**
- * Wait `ms`, but stop waiting the moment the server names a new
- * incarnation, and do not start waiting if it already has.
- *
- * The back-off used to be served in full across a reattach. A beat that
- * had been failing left the loop sleeping for up to 30 seconds; the stream
- * reopened, the server started its 6-second clock on the new connection,
- * and the next beat arrived long after the server had closed it for
- * silence. That beat was refused, which counted as another failure, which
- * kept the wait at 30 seconds. The client could not get out of it on its
- * own: one hiccup, and it lapsed every connection it ever opened again.
- */
- const pause = async (ms: number): Promise<void> => {
- if (this.attaches !== seenAttach) return;
- await new Promise<void>((resolve) => {
- const timer = setTimeout(resolve, ms);
- this.wakeBeat = () => {
- clearTimeout(timer);
- resolve();
- };
- });
- this.wakeBeat = () => {};
- };
-
- /** Count a beat that did not land and slow the loop down. Returns whether
- * to report this one: the first, then powers of two, so a permanent
- * failure costs a handful of lines rather than one per beat. */
- const slowDown = (): boolean => {
- failures += 1;
- backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
- return (failures & (failures - 1)) === 0;
- };
-
- const fail = (error: unknown): void => {
- if (!slowDown()) return;
- log.warn('SwitchEventStream: heartbeat failed', {
- event: 'switch_beat_failed',
- endpoint: this.deps.creds.apiEndpoint,
- failures,
- error: String(error),
- backoffMs: backoff,
- });
- };
-
- /**
- * One beat, as a value rather than a throw.
- *
- * A beat the fence disowns must have no outcome at all, and an exception
- * is an outcome: it would slow the loop down over a request the reattach
- * itself invalidated. So the whole exchange, body included, happens inside
- * the flight the fence is timing, and comes back as something to ignore or
- * act on once it is known which.
- */
- const beat = async (): Promise<
- | { answered: true; status: number; ok: boolean; body: string }
- | { answered: false; error: unknown }
- > => {
- try {
- const resp = await this.post('connection/beat', {
- connection_id: connectionId,
- cursor: this.cursor,
- position: this._position,
- generation: this.generation,
- });
- return { answered: true, status: resp.status, ok: resp.ok, body: await resp.text() };
- } catch (error) {
- return { answered: false, error };
- }
- };
-
- while (!signal.aborted && !this.halt.signal.aborted) {
- // Every pass, not only the first: a reconnect makes a new incarnation,
- // and the tick has to name the one the server is on. Nothing may be
- // awaited between passing the gate and registering the flight, or the
- // beat could be sent under an incarnation later than the one recorded.
- await Promise.race([this.fence.reached, until(signal, this.halt.signal)]);
- if (signal.aborted || this.halt.signal.aborted) return;
- // The gate this pass awaited may already have been closed by the time
- // the await resumed: a reopen closes it synchronously, and the resolved
- // promise we were handed is the old one. Checked here, with nothing
- // awaited before the tick, so no beat leaves while admission is shut.
- if (!this.fence.admitting) continue;
- if (this.attachedOver !== 'stream') {
- // Attached over the socket, which carries its own heartbeat. Sleep
- // until the next attach, which may be over the stream.
- seenAttach = this.attaches;
- await pause(MAX_BACKOFF_MS);
- continue;
- }
- // A connection the server has just attached has never beaten, and has
- // 6 seconds to. Whatever an earlier connection's failures built up says
- // nothing about this one, so it starts at the base cadence.
- if (this.attaches !== seenAttach) {
- seenAttach = this.attaches;
- backoff = BEAT_INTERVAL_MS;
- }
- const tick = await this.fence.tick(beat);
- if (signal.aborted || this.halt.signal.aborted) return;
-
- if (!tick.current) {
- // A reattach began while this was in flight, so the answer describes
- // an incarnation we are no longer on and cannot be told apart from
- // one that arrived late about our own reopen. Inert: no stand-down,
- // no reopen, no cursor, and no mark against the beat rate either.
- log.debug('SwitchEventStream: heartbeat answer discarded: reattached in flight', {
- event: 'switch_beat_stale',
- connectionId,
- });
- } else if (!tick.value.answered) {
- fail(tick.value.error);
- } else {
- const { status, ok, body } = tick.value;
- if (status === 401 || status === 403) {
- this.rejectCredentials(status, body);
- return;
- }
- if (status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
- // The other door onto a takeover. A displaced client whose socket
- // dropped before the `evicted` frame reached it learns here instead,
- // and must end the same way: reopening is a takeover, so a loser
- // that reopens takes the connection straight back off the winner.
- this.standDown();
- return;
- }
- if (status === 404 || status === 409) {
- // The server answered, so this is not an outage, but it is not a
- // beat that landed either: we are not attached, and only a reopen
- // fixes that. A reopen that keeps being refused is a client that
- // cannot currently succeed, and it must not spend the endpoint at
- // full rate while it fails.
- //
- // So the wait goes *before* the reopen, and grows with each refusal
- // in a row. It used to go after (reopen at once, then sleep the
- // back-off before beating), which starved the very connection the
- // reopen had just made. Pacing the reopen instead keeps the same
- // brake on a refusing server while letting each new connection beat
- // as soon as it is attached.
- failures += 1;
- rejections += 1;
- const waitMs = Math.min(BEAT_INTERVAL_MS * 2 ** rejections, MAX_BACKOFF_MS);
- if ((failures & (failures - 1)) === 0) {
- log.warn('SwitchEventStream: heartbeat rejected: reopening', {
- event: 'switch_beat_rejected',
- status,
- connectionId,
- failures,
- backoffMs: waitMs,
- });
- }
- await pause(waitMs);
- if (signal.aborted || this.halt.signal.aborted) return;
- // The stream may have reattached on its own while we waited: the
- // server closes a lapsed connection's socket, and the stream loop
- // reopens it. Reopening again would throw that healthy connection
- // away; beat on it instead.
- if (this.attaches === seenAttach) this.reattach();
- continue;
- } else if (!ok) {
- fail(new Error(`HTTP ${status}`));
- } else {
- if (failures > 0) {
- log.warn('SwitchEventStream: heartbeat recovered', {
- event: 'switch_beat_recovered',
- afterFailures: failures,
- });
- }
- failures = 0;
- rejections = 0;
- backoff = BEAT_INTERVAL_MS;
- }
- }
- await pause(backoff);
- }
- }
}
/** The rooms a placements answer says the agent is not in. */
diff --git a/console/packages/switch-agent-runtime/src/sse.test.ts b/console/packages/switch-agent-runtime/src/sse.test.ts
deleted file mode 100644
index da53e9838..000000000
--- a/console/packages/switch-agent-runtime/src/sse.test.ts
+++ /dev/null
@@ -1,94 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { readSse, type SseFrame } from './sse';
-
-/**
- * The wire format, pinned.
- *
- * These moved here with the parser. They used to sit beside a drift check that
- * compared two copies of this file; the copies are gone, so the check is too,
- * but the behaviour it protected still needs testing, and now testing it once
- * genuinely covers every client, which was the point of the move.
- */
-
-function stream(chunks: string[]): ReadableStream<Uint8Array> {
- const encoder = new TextEncoder();
- return new ReadableStream<Uint8Array>({
- start(controller) {
- for (const chunk of chunks) controller.enqueue(encoder.encode(chunk));
- controller.close();
- },
- });
-}
-
-async function parse(chunks: string[]): Promise<SseFrame[]> {
- const frames: SseFrame[] = [];
- for await (const frame of readSse(stream(chunks), new AbortController().signal)) {
- frames.push(frame);
- }
- return frames;
-}
-
-describe('SSE framing', () => {
- it('parses event, id and data', async () => {
- const frames = await parse(['id: 7\nevent: message\ndata: {"body":"hi"}\n\n']);
- expect(frames).toEqual([{ event: 'message', id: '7', data: { body: 'hi' } }]);
- });
-
- it('defaults the event name to message when absent', async () => {
- const frames = await parse(['data: {"a":1}\n\n']);
- expect(frames[0].event).toBe('message');
- });
-
- it('leaves id undefined when the server sends none', async () => {
- // Control frames carry no sequence, treating a missing id as 0 would drag
- // the cursor backwards and replay the whole buffer on reconnect.
- const frames = await parse(['event: connection_state\ndata: {"rooms":[]}\n\n']);
- expect(frames[0].id).toBeUndefined();
- });
-
- it('skips keepalive comments without emitting a frame', async () => {
- const frames = await parse([': keepalive\n\n', 'event: gap\ndata: {"from_sequence":3}\n\n']);
- expect(frames).toHaveLength(1);
- expect(frames[0].event).toBe('gap');
- });
-
- it('reassembles a frame split across chunk boundaries', async () => {
- // The network splits wherever it likes; a frame torn mid-JSON must not be
- // dropped or parsed twice.
- const frames = await parse(['id: 1\nevent: mes', 'sage\ndata: {"bo', 'dy":"split"}\n\n']);
- expect(frames).toEqual([{ event: 'message', id: '1', data: { body: 'split' } }]);
- });
-
- it('emits several frames arriving in one chunk', async () => {
- const frames = await parse([
- 'id: 1\nevent: message\ndata: {"n":1}\n\nid: 2\nevent: message\ndata: {"n":2}\n\n',
- ]);
- expect(frames.map((f) => f.data.n)).toEqual([1, 2]);
- });
-
- it('joins multi-line data fields', async () => {
- const frames = await parse(['event: message\ndata: {"a":\ndata: 1}\n\n']);
- expect(frames[0].data).toEqual({ a: 1 });
- });
-
- it('ignores a frame with no data lines', async () => {
- const frames = await parse(['event: message\nid: 4\n\n']);
- expect(frames).toEqual([]);
- });
-
- it('throws on unparseable JSON rather than skipping it', async () => {
- // A frame we cannot read is a frame we have lost. Silently dropping it
- // would advance nothing and hide the loss.
- await expect(parse(['event: message\ndata: {not json}\n\n'])).rejects.toThrow();
- });
-
- it('stops when the signal aborts', async () => {
- const abort = new AbortController();
- abort.abort();
- const frames: SseFrame[] = [];
- for await (const frame of readSse(stream(['data: {"a":1}\n\n']), abort.signal)) {
- frames.push(frame);
- }
- expect(frames).toEqual([]);
- });
-});
diff --git a/console/packages/switch-agent-runtime/src/sse.ts b/console/packages/switch-agent-runtime/src/sse.ts
deleted file mode 100644
index 13b8f2204..000000000
--- a/console/packages/switch-agent-runtime/src/sse.ts
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * The agent bridge's SSE framing.
- *
- * One implementation, imported by everything that speaks the protocol: the
- * local MCP runtime next to the agent, and Switch Console. It used to exist twice
- * (separate deployables, no shared package) and the copies drifted within a day
- * of being made. This package is what removed the excuse.
- *
- * Deliberately free of imports and of anything specific to a host: it is the
- * wire format and nothing else. Reporting, reconnection, heartbeats and cursors
- * belong to the caller, which is where clients legitimately differ.
- */
-
-export type SseFrame = { event: string; id?: string; data: Record<string, unknown> };
-
-/** Parse an SSE byte stream into frames, ending when the stream or signal does. */
-export async function* readSse(
- body: ReadableStream<Uint8Array>,
- signal: AbortSignal
-): AsyncGenerator<SseFrame> {
- const reader = body.getReader();
- const decoder = new TextDecoder();
- let buffered = '';
-
- try {
- while (!signal.aborted) {
- const { done, value } = await reader.read();
- if (done) return;
- buffered += decoder.decode(value, { stream: true });
-
- let split: number;
- while ((split = buffered.indexOf('\n\n')) !== -1) {
- const raw = buffered.slice(0, split);
- buffered = buffered.slice(split + 2);
-
- let event = 'message';
- let id: string | undefined;
- const dataLines: string[] = [];
- for (const line of raw.split('\n')) {
- // A comment line is the server's keepalive: it stops proxies timing
- // the stream out for idleness and carries nothing.
- if (line.startsWith(':')) continue;
- if (line.startsWith('event: ')) event = line.slice(7);
- else if (line.startsWith('id: ')) id = line.slice(4);
- else if (line.startsWith('data: ')) dataLines.push(line.slice(6));
- }
- if (!dataLines.length) continue;
- // A frame we cannot parse is a frame we have lost. Surface it rather
- // than skipping quietly; the caller decides how loud to be.
- yield { event, id, data: JSON.parse(dataLines.join('\n')) };
- }
- }
- } finally {
- void reader.cancel().catch(() => {});
- }
-}
diff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.py
index 483dcbd78..1c363d757 100644
--- a/core/switch_core/artifacts.py
+++ b/core/switch_core/artifacts.py
@@ -37,8 +37,8 @@ ARTIFACT_VERSIONS: Final[dict[str, str]] = {
CONTRACTS: Final[dict[str, dict[str, ContractRange]]] = {
# The wire protocol between switch-core's agent bridge and an agent runtime: the event stream, its frame shapes, the resume cursor, and the tool-call channel.
"agent-protocol": {
- "switch-core": ContractRange(speaks=9, accepts=1),
- "agent-runtime": ContractRange(speaks=9, accepts=1),
+ "switch-core": ContractRange(speaks=9, accepts=8),
+ "agent-runtime": ContractRange(speaks=9, accepts=8),
},
# The HTTP API switch-core serves to first-party UI clients. Excludes the authentication surface, which is deliberately frozen and unversioned so that a client can always authenticate far enough to be told what is wrong.
"gateway-api": {
diff --git a/core/switch_core/bridges/agent/api/handlers.py b/core/switch_core/bridges/agent/api/handlers.py
index 6628c9164..efe378759 100644
--- a/core/switch_core/bridges/agent/api/handlers.py
+++ b/core/switch_core/bridges/agent/api/handlers.py
@@ -21,11 +21,10 @@ from fastapi import (
)
from pydantic import ValidationError
from sqlalchemy.ext.asyncio import AsyncSession
-from starlette.responses import Response, StreamingResponse
+from starlette.responses import Response
from switch_core.bridges.agent.api.schemas import (
BulkRegisterResult,
- ConnectionBeatRequest,
ConnectionPlacementsRequest,
ConnectionRenewRequest,
ConnectionSubscribeRequest,
@@ -74,19 +73,14 @@ from switch_core.bridges.agent.protocol.agent_connections import (
Closure,
ConnectionError_,
DeliveryFilter,
- NoStreamAttachedError,
ProtocolVersionError,
RoomOccupiedError,
Scope,
- SupersededConnectionError,
SupersededControlError,
SupersededReattachError,
- Transport,
- UnfencedBeatError,
UnfencedControlError,
UnknownConnectionError,
evicted_session_warning,
- sse_fallback,
)
from switch_core.bridges.agent.protocol.agent_core import AgentCore, AgentExistsError
from switch_core.bridges.agent.protocol.event_buffer import TICKET_FLOOR, Reader
@@ -96,7 +90,6 @@ from switch_core.bridges.agent.protocol.stream import (
Frame,
encode_ws,
event_frames,
- sse_stream,
)
from switch_core.bridges.agent.registration_bootstrap import (
BOOTSTRAP_KEY_TYPE,
@@ -751,7 +744,6 @@ async def poll_events(
agent_id: str,
agent: Annotated[Agent, Depends(get_agent_from_scope)],
protocol: Annotated[AgentCore, Depends(get_protocol)],
- config: Annotated[SwitchConfig, Depends(get_config)],
timeout: Annotated[float, Query()] = 10,
accept: Annotated[str | None, Header()] = None,
connection_id: Annotated[str | None, Query()] = None,
@@ -766,46 +758,26 @@ async def poll_events(
client: Annotated[str | None, Query()] = None,
client_version: Annotated[str | None, Query()] = None,
rooms: Annotated[str | None, Query()] = None,
- fallback: Annotated[str | None, Query()] = None,
- fallback_close_code: Annotated[str | None, Query()] = None,
- last_event_id: Annotated[str | None, Header(alias="last-event-id")] = None,
) -> EventResponse | Response:
- """Deliver the agent's events, as a push stream or a long poll.
-
- The live connection is the WebSocket at `/connection/ws`. `Accept:
- text/event-stream` opens the same connection as a Server-Sent Events
- stream, with its heartbeat on `POST /connection/beat`: that is how an agent
- runtime built before the WebSocket connects (agent-protocol revision 7 and
- older), and it is kept for those clients for a compatibility window. It
- goes once `switch.agents.connected{transport:sse}` stays at zero. Anything
- else falls back to the long poll, served from the same buffer.
-
- The declaration parameters are all optional and all default to None,
- meaning *unknown* (CHOO-1865): a client that says nothing still connects.
- `fallback` (and `fallback_close_code`) come from a runtime that has the
- WebSocket but whose socket never opened, and only reach the log line.
+ """Deliver the agent's events as a long poll.
+
+ The live connection is the WebSocket at `/connection/ws`. A request for the
+ old Server-Sent Events stream is refused with what to do about it.
"""
if accept and "text/event-stream" in accept:
- return await _open_event_stream(
- agent=agent,
- protocol=protocol,
- config=config,
- connection_id=connection_id,
- scope=scope,
- event_filter=event_filter,
- start_from=start_from,
- start_at=start_at,
- spawn_capable=spawn_capable,
- declaration=ClientDeclaration(
- speaks=protocol_version,
- accepts=protocol_accepts,
- artifact=client,
- version=client_version,
- fallback=sse_fallback(fallback, fallback_close_code),
- ),
- rooms=rooms,
- last_event_id=last_event_id,
- expected_generation=expected_generation,
+ # The agent connection moved to one WebSocket, which carries both the
+ # events and the heartbeat. A client still asking for the stream is
+ # an old runtime, and is told what to do rather than left retrying.
+ metrics().increment(AGENT_CONNECTIONS_REFUSED, {"reason": "transport_removed"})
+ raise HTTPException(
+ status_code=410,
+ detail={
+ "code": "transport_removed",
+ "message": "The event stream over Server-Sent Events was "
+ "replaced by a WebSocket at /agents/{agent_id}/connection/ws.",
+ "remedy": "Update Switch Console, or the agent runtime, to a "
+ "version that connects over the WebSocket.",
+ },
)
events = await protocol.poll_events(agent.id, timeout=timeout)
@@ -820,7 +792,6 @@ async def _resolve_start_position(
*,
start_at: str | None,
start_from: str,
- last_event_id: str | None,
reads_conversation: bool,
) -> tuple[Position, int | None]:
"""Work out the position a new connection starts sending events from.
@@ -834,9 +805,7 @@ async def _resolve_start_position(
no `s` entry, the connection starts at the current end of the agent's
own events.
- An older client holds one integer and sends it as `start_from`, or as
- `Last-Event-ID` when an event stream reconnects. `Last-Event-ID` wins
- because it is the id of the last event the client processed. When the
+ An older client holds one integer and sends it as `start_from`. When the
connection reads only the agent's own events, the integer is their `seq`.
When it also reads the room conversation (`reads_conversation`), the
integer is a ticket id from `agent_position_tickets` and the position is
@@ -858,7 +827,7 @@ async def _resolve_start_position(
if OWN_EVENTS not in position:
position[OWN_EVENTS] = await buffer.head(agent.tenant_id, agent.id)
return position, None
- raw = last_event_id or start_from
+ raw = start_from
if raw in ("", "head"):
return {OWN_EVENTS: await buffer.head(agent.tenant_id, agent.id)}, None
try:
@@ -887,7 +856,6 @@ def _register_connection(
position: Position,
declaration: ClientDeclaration,
expected_generation: int | None,
- transport: Transport,
) -> AgentConnection:
try:
conn = protocol.connections.open(
@@ -899,7 +867,6 @@ def _register_connection(
position=position,
declaration=declaration,
expected_generation=expected_generation,
- transport=transport,
)
except SupersededReattachError as exc:
# Structured, like the refused heartbeat: this is the same ending, and
@@ -947,14 +914,10 @@ async def _open_connection(
declaration: ClientDeclaration,
rooms: str | None,
expected_generation: int | None,
- last_event_id: str | None = None,
- transport: Transport = "websocket",
) -> tuple[AgentConnection, AsyncGenerator[Frame]]:
"""Open or reattach the connection and claim its declared rooms.
- Shared by every transport, so the socket and the event stream fence and
- claim rooms in exactly the same way. Returns
- the connection and its frames, for the transport to encode.
+ Returns the connection and its frames, for the socket to encode.
"""
if not connection_id:
raise HTTPException(
@@ -984,7 +947,6 @@ async def _open_connection(
agent,
start_at=start_at,
start_from=start_from,
- last_event_id=last_event_id,
reads_conversation=event_filter == "all",
)
@@ -1005,7 +967,6 @@ async def _open_connection(
position=position,
declaration=declaration,
expected_generation=expected_generation,
- transport=transport,
)
except UnknownConnectionError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@@ -1091,60 +1052,6 @@ def _refusal_reason(exc: HTTPException) -> str:
return "protocol" if isinstance(exc.__cause__, ProtocolVersionError) else "other"
-async def _open_event_stream(
- *,
- agent: Agent,
- protocol: AgentCore,
- config: SwitchConfig,
- connection_id: str | None,
- scope: str,
- event_filter: str,
- start_from: str,
- spawn_capable: bool,
- start_at: str | None = None,
- declaration: ClientDeclaration,
- rooms: str | None,
- last_event_id: str | None,
- expected_generation: int | None,
-) -> StreamingResponse:
- """The connection as a Server-Sent Events stream, for an old client.
-
- Opens exactly as the socket does and streams the same frames from the same
- loop, so the two cannot drift while both exist. A refusal is the HTTP
- error the socket would have sent as its `refused` frame.
- """
- try:
- _conn, frames = await _open_connection(
- agent=agent,
- protocol=protocol,
- config=config,
- connection_id=connection_id,
- scope=scope,
- event_filter=event_filter,
- start_from=start_from,
- start_at=start_at,
- spawn_capable=spawn_capable,
- declaration=declaration,
- rooms=rooms,
- expected_generation=expected_generation,
- last_event_id=last_event_id,
- transport="sse",
- )
- except HTTPException as exc:
- metrics().increment(AGENT_CONNECTIONS_REFUSED, {"reason": _refusal_reason(exc)})
- raise
- return StreamingResponse(
- sse_stream(frames),
- media_type="text/event-stream",
- headers={
- "Cache-Control": "no-cache",
- "Connection": "keep-alive",
- # Proxies that buffer would defeat the point of a push channel.
- "X-Accel-Buffering": "no",
- },
- )
-
-
def _record_beat(
protocol: AgentCore,
agent_id: str,
@@ -1153,12 +1060,11 @@ def _record_beat(
) -> AgentConnection:
"""Count a client's beat: the client is still there.
- A beat is a pong on the WebSocket, or a `POST /connection/beat` from an
- event-stream client. Clients also report how far they have read; it is not
- read here. The connection's position is moved only by the stream, as it
- sends, and a client that drops resumes from the position it sends when it
- reopens, which is checked against the database then. So a beat costs no
- database query.
+ A beat is a pong on the WebSocket. Clients also report how far they have
+ read; it is not read here. The connection's position is moved only by the
+ stream, as it sends, and a client that drops resumes from the position it
+ sends when it reopens, which is checked against the database then. So a
+ beat costs no database query.
Raises the registry's refusal when the connection is no longer this
client's to beat for: another socket took it over (`taken_over`), or it
@@ -1167,59 +1073,6 @@ def _record_beat(
return protocol.connections.beat(agent_id, connection_id, generation)
[email protected]("/{agent_id}/connection/beat")
-async def connection_beat(
- agent_id: str,
- req: ConnectionBeatRequest,
- agent: Annotated[Agent, Depends(get_agent_from_scope)],
- protocol: Annotated[AgentCore, Depends(get_protocol)],
-) -> dict[str, Any]:
- """The heartbeat of a connection held over the event stream.
-
- The socket carries its own heartbeat; this is for a client built before it,
- kept with the event stream for the same compatibility window. Proves the
- client is alive. Rejected when the connection is
- unknown, dead, has no stream attached, or belongs to another incarnation.
- A refusal carries a code beside its prose, because the remedies differ:
- `taken_over` is terminal for the client that receives it, and the rest are
- recovered by reopening.
- """
- try:
- try:
- conn = _record_beat(protocol, agent.id, req.connection_id, req.generation)
- except (
- UnknownConnectionError,
- SupersededConnectionError,
- UnfencedBeatError,
- ):
- if _held_here(protocol, req.connection_id) and not _presents_later(
- protocol, req.connection_id, req.generation
- ):
- raise
- # The beat reached a process that does not hold the stream, or
- # that held it before a later incarnation was opened elsewhere,
- # and its copy of the connection may be a moment old: a refusal is
- # final for the client, so it is given only on the recorded row.
- await protocol.connections.refresh(agent.id)
- conn = _record_beat(protocol, agent.id, req.connection_id, req.generation)
- except (
- NoStreamAttachedError,
- SupersededConnectionError,
- UnfencedBeatError,
- ) as exc:
- raise HTTPException(
- status_code=409, detail={"code": exc.code, "message": str(exc)}
- ) from exc
- except UnknownConnectionError as exc:
- raise HTTPException(status_code=404, detail=str(exc)) from exc
- return {
- "ok": True,
- "rooms": sorted(conn.rooms),
- "cursor": conn.cursor,
- "position": dict(conn.position),
- }
-
-
# A WebSocket close code for each refusal that would otherwise be an HTTP
# status: 4000 plus the status, so a client maps them onto one set of remedies.
def _ws_close_code(status: int) -> int:
@@ -1248,13 +1101,11 @@ async def connection_socket(
) -> None:
"""The agent's connection over one WebSocket: events down, heartbeats up.
- Opens exactly as the event stream does (same parameters, same fencing,
- same room claims) and carries the same frames, as JSON
- `{"event", "data", "id"}`. Authenticated once, when the socket opens.
+ Carries the agent's frames as JSON `{"event", "data", "id"}`.
+ Authenticated once, when the socket opens.
- The heartbeat runs the other way round from `/connection/beat`: the server
- sends a `ping` frame every heartbeat interval and the client answers
- `{"type": "pong", "cursor": n}`, which counts as a beat. A client that
+ The server sends a `ping` frame every heartbeat interval and the client
+ answers `{"type": "pong", "cursor": n}`, which counts as a beat. A client that
stops answering lapses like one that stops beating. No request per beat,
and no token lookup.
diff --git a/core/switch_core/bridges/agent/api/schemas.py b/core/switch_core/bridges/agent/api/schemas.py
index 7ef01c5d9..7f38f3308 100644
--- a/core/switch_core/bridges/agent/api/schemas.py
+++ b/core/switch_core/bridges/agent/api/schemas.py
@@ -138,27 +138,6 @@ class ConnectionPlacementsRequest(BaseModel):
generation: int | None = None
-class ConnectionBeatRequest(BaseModel):
- """The heartbeat of a connection held over the event stream (CHOO-1857).
-
- Kept for clients built before the WebSocket, whose socket carries the
- heartbeat itself, for a compatibility window. Proves the client is alive.
- Clients also send how far they have read (`cursor`, `position`); the
- server does not read it, because a client resumes from the position it
- sends when it reopens.
- """
-
- connection_id: str
- #: The incarnation of the connection this client is attached to, as the
- #: server told it on `connection_state`. Fences the tick: a client that has
- #: been displaced still holds the id and the token, and is otherwise
- #: indistinguishable from the one that replaced it. Null is accepted only
- #: while the connection's holder is a client built before the fence existed
- #: (unknown, not current); from a holder that declares the revision which
- #: carries it, a tick without one is refused.
- generation: int | None = None
-
-
class StatusRequest(BaseModel):
room_id: str
presence: Literal["online", "offline"] | None = None
diff --git a/core/switch_core/bridges/agent/protocol/agent_connections.py b/core/switch_core/bridges/agent/protocol/agent_connections.py
index ed377d76f..aeef465d3 100644
--- a/core/switch_core/bridges/agent/protocol/agent_connections.py
+++ b/core/switch_core/bridges/agent/protocol/agent_connections.py
@@ -70,10 +70,6 @@ logger = logging.getLogger(__name__)
Scope = Literal["single", "all"]
DeliveryFilter = Literal["all", "addressed"]
-# How a connection's stream reaches its client. `sse` is the event stream an
-# agent runtime built before the WebSocket still asks for, served for a
-# compatibility window and counted apart so we can see those clients drain.
-Transport = Literal["websocket", "sse"]
# Refuse a client that cannot meet this server's protocol rather than degrading
# in ways neither side can see. The runtime lives on the user's machine and
@@ -334,9 +330,6 @@ class ClientDeclaration:
accepts: int | None = None
artifact: str | None = None
version: str | None = None
- # Why a client that has the WebSocket opened the event stream instead, as
- # `sse_fallback` cleaned it. Logged only: not recorded and not reported.
- fallback: str | None = None
@property
def declares_protocol(self) -> bool:
@@ -364,31 +357,6 @@ class ClientDeclaration:
}
-SSE_FALLBACK_REASONS = frozenset({"socket_never_opened"})
-
-
-def sse_fallback(reason: str | None, close_code: str | None) -> str | None:
- """Why a client opened the event stream, as it goes in the log line.
-
- A runtime that has the WebSocket asks for the event stream only when its
- socket never opened, and says so, so the log can tell it apart from a
- client built before the socket. Both values are the client's own, so an
- unknown reason becomes "other" and a close code that is not a short number
- is dropped.
- """
- if not reason:
- return None
- label = reason if reason in SSE_FALLBACK_REASONS else "other"
- if (
- close_code
- and close_code.isascii()
- and close_code.isdigit()
- and len(close_code) <= 4
- ):
- return f"{label}:{close_code}"
- return label
-
-
def client_label(declaration: ClientDeclaration) -> str:
"""The declared artifact as a metric value: a registered name, or a bucket.
@@ -403,14 +371,6 @@ def client_label(declaration: ClientDeclaration) -> str:
return "other"
-def _fallback_suffix(declaration: ClientDeclaration) -> str:
- """` fallback=<why>` for the `[CONN]` log lines, or nothing when the
- client did not fall back, so the line reads as it always has."""
- if declaration.fallback is None:
- return ""
- return f" fallback={declaration.fallback}"
-
-
class ProtocolVersionError(ConnectionError_):
"""A client's declared agent-protocol range cannot meet this server's.
@@ -464,8 +424,6 @@ class AgentConnection:
# agent belongs to that no sibling has claimed.
rooms: set[str] = field(default_factory=set)
stream_attached: bool = False
- # What the attached stream travels over, as of the last attach.
- stream_transport: Transport = "websocket"
# How many heartbeats this connection has received. Diagnostic: it is the
# difference between a client that never started beating and one that beat
# and then stopped, which the timestamp alone cannot tell you.
@@ -548,10 +506,9 @@ CONTROLLER_LABEL = ("controller", "agents-controller")
def connection_transport(conn: AgentConnection) -> str:
- """`websocket` or `sse` while a stream is attached, by what it travels over;
- `detached` while it has dropped and the connection waits out its heartbeat
- window for a reconnect."""
- return conn.stream_transport if conn.stream_attached else "detached"
+ """`websocket` while the socket is attached; `detached` while the socket has
+ dropped and the connection waits out its heartbeat window for a reconnect."""
+ return "websocket" if conn.stream_attached else "detached"
#: How a deleted agent's connections end, in whichever process holds them. A
@@ -708,7 +665,6 @@ class AgentConnectionRegistry:
position: Position,
declaration: ClientDeclaration,
expected_generation: int | None,
- transport: Transport = "websocket",
) -> AgentConnection:
"""Open a connection, or reattach to one the client already owns.
@@ -770,7 +726,6 @@ class AgentConnectionRegistry:
existing.last_beat = time.monotonic()
existing.closure = None
existing.stream_attached = True
- existing.stream_transport = transport
existing.stream_generation = self._new_incarnation()
# Wake the stream this one replaces, so its client hears it was
# taken over now rather than on the stream's next idle tick.
@@ -781,15 +736,12 @@ class AgentConnectionRegistry:
existing.declaration = declaration
metrics().increment(AGENT_CONNECTIONS_OPENED, {"kind": "reattach"})
logger.info(
- "[CONN] reattached agent=%s connection=%s scope=%s generation=%s "
- "transport=%s%s%s",
+ "[CONN] reattached agent=%s connection=%s scope=%s generation=%s%s",
agent_id,
connection_id,
scope,
existing.stream_generation,
- transport,
f" from process={moved_from}" if moved_from is not None else "",
- _fallback_suffix(declaration),
)
return existing
@@ -808,7 +760,6 @@ class AgentConnectionRegistry:
last_beat=now,
opened_at=now,
stream_attached=True,
- stream_transport=transport,
stream_generation=self._new_incarnation(),
declaration=declaration,
)
@@ -820,7 +771,7 @@ class AgentConnectionRegistry:
)
logger.info(
"[CONN] opened agent=%s connection=%s scope=%s filter=%s spawn=%s "
- "client=%s version=%s protocol=%s transport=%s%s",
+ "client=%s version=%s protocol=%s",
agent_id,
connection_id,
scope,
@@ -831,8 +782,6 @@ class AgentConnectionRegistry:
f"{floor}-{declaration.speaks}"
if declaration.declares_protocol
else "unknown",
- transport,
- _fallback_suffix(declaration),
)
return conn
@@ -1603,7 +1552,6 @@ class AgentConnectionRegistry:
last_beat=now,
opened_at=now,
stream_attached=True,
- stream_transport=cast(Transport, row.transport),
stream_generation=row.incarnation,
declaration=ClientDeclaration(speaks=row.speaks, accepts=row.accepts),
held_by=row.process_id,
diff --git a/core/switch_core/bridges/agent/protocol/connection_rows.py b/core/switch_core/bridges/agent/protocol/connection_rows.py
index 34e5fe918..e7a551c82 100644
--- a/core/switch_core/bridges/agent/protocol/connection_rows.py
+++ b/core/switch_core/bridges/agent/protocol/connection_rows.py
@@ -39,7 +39,7 @@ lapsed. What a signal asks this process to read goes through the bus's
`PendingWork`, so a read that fails is kept and tried again.
Three requests are handed to the holder rather than served where they land: a
-`POST /connection/beat` from an old event-stream client (forwarded in batches,
+beat for a connection another process holds (forwarded in batches,
`agent_connections.beats`), a session command for a stream another process
holds, the agent's watcher or its controller's (stored as a relay row for that
process and signalled, `agent_connections.session_command`, since its content
@@ -265,7 +265,9 @@ class SharedConnections:
scope=conn.scope,
delivery_filter=conn.delivery_filter,
spawn_capable=conn.spawn_capable,
- transport=conn.stream_transport,
+ # The column outlived the event stream: every
+ # connection travels over the WebSocket now.
+ transport="websocket",
speaks=conn.declaration.speaks,
accepts=conn.declaration.protocol_floor,
)
diff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.py
index 5d86251a4..d2d8ef131 100644
--- a/core/switch_core/bridges/agent/protocol/stream.py
+++ b/core/switch_core/bridges/agent/protocol/stream.py
@@ -2,9 +2,8 @@
Turns a connection plus the event buffer into a sequence of frames: catch-up
from the client's cursor, then live delivery as events are appended. The client
-never asks again; it opens once and reads. `api/handlers.py` sends them over
-the WebSocket, or, for a client built before it, as a Server-Sent Events
-stream: both encode the same frames from the same loop, so they cannot drift.
+never asks again; it opens once and reads. The WebSocket in
+`api/handlers.py` sends them.
Every event says where it sits, so a client that reconnects can say how far it
got and resume exactly where it stopped. A client on agent-protocol revision 9
@@ -22,10 +21,9 @@ events must never see a stream that looks complete.
from __future__ import annotations
import asyncio
-import json
import logging
import time
-from collections.abc import AsyncGenerator, AsyncIterator
+from collections.abc import AsyncGenerator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
@@ -81,9 +79,7 @@ class Frame:
# Not a message: the loop's signal that it waited a keepalive interval with
-# nothing to send. The WebSocket has its own ping and drops it; the event
-# stream writes a comment, which only stops proxies dropping an idle
-# connection. Liveness comes from the client's beat, never from this.
+# nothing to send. The WebSocket has its own ping and drops it.
KEEPALIVE = Frame("keepalive", {})
@@ -100,35 +96,6 @@ def encode_ws(frame: Frame) -> dict[str, Any]:
return message
-def encode_sse(frame: Frame) -> bytes:
- """A frame as a Server-Sent Event. `id` is the event's sequence number,
- which a reconnecting client sends back as `Last-Event-ID`."""
- if frame is KEEPALIVE:
- return b": keepalive\n\n"
- lines = []
- if frame.seq is not None:
- lines.append(f"id: {frame.seq}")
- lines.append(f"event: {frame.event}")
- lines.append(f"data: {json.dumps(frame.data, separators=(',', ':'))}")
- return ("\n".join(lines) + "\n\n").encode()
-
-
-async def sse_stream(frames: AsyncGenerator[Frame]) -> AsyncIterator[bytes]:
- """The connection's frames as a `text/event-stream` body.
-
- Kept for clients built before the WebSocket (agent-protocol revision 7 and
- older), for a compatibility window: it goes once
- `switch.agents.connected{transport:sse}` stays at zero.
- """
- try:
- async for frame in frames:
- yield encode_sse(frame)
- finally:
- # Closed here, so the stream detaches from its connection when the
- # client goes rather than when the generator is garbage collected.
- await frames.aclose()
-
-
def _connection_state(conn: AgentConnection) -> dict[str, Any]:
"""The first frame of every stream, and where the server declares itself.
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index b332d254a..90bd7c095 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -471,11 +471,9 @@ AGENT_BUFFER_SCAN_DURATION = _spec(
"operation",
bounds=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 100.0),
)
-# `reason` is from a fixed set the refusal sites choose. `transport_removed` is
+# `reason` is from a fixed set the refusal sites choose: `transport_removed` is
# a client still asking for the retired event stream, the one way to see old
-# clients that never manage to connect at all. Nothing sends it while the event
-# stream is kept for its compatibility window; it is kept here for the release
-# that removes the stream, when it starts firing again.
+# clients that never manage to connect at all.
AGENT_CONNECTIONS_REFUSED = _spec(
"switch.agent.connections_refused",
"sum",
@@ -523,18 +521,16 @@ AGENT_CONNECTIONS_OPENED = _spec(
# ── Agents and clients ───────────────────────────────────────────────────────
# `client` is the declared artifact when the registry knows the name, else
# `other` or `unknown`. Never the declared version: the client chooses it.
-# `transport` is `websocket`, `sse` (an old client on the event stream, kept
-# for a compatibility window that ends when this stays at zero), `detached`
-# (the stream dropped, and the connection waits out its heartbeat window), or
-# `controller` (run by an agent controller, on that controller's connection).
+# `transport` is `websocket`, `detached` (the stream dropped, and the
+# connection waits out its heartbeat window), or `controller` (run by an
+# agent controller, on that controller's connection).
AGENTS_CONNECTED = _spec(
"switch.agents.connected",
"gauge",
"{agent}",
- "Agents holding a live protocol connection, by transport (websocket, sse, "
- "detached, controller) and client. An agent connected more than one way counts once "
- "under each, so summing across the attributes can exceed the number of "
- "agents.",
+ "Agents holding a live protocol connection, by transport and client. An "
+ "agent connected more than one way counts once under each, so summing "
+ "across the attributes can exceed the number of agents.",
"transport",
"client",
)
diff --git a/core/tests/e2e/follow_the_conversation/harness.py b/core/tests/e2e/follow_the_conversation/harness.py
index 1a4bc58df..73cbdd655 100644
--- a/core/tests/e2e/follow_the_conversation/harness.py
+++ b/core/tests/e2e/follow_the_conversation/harness.py
@@ -333,43 +333,25 @@ async def run_agent(name: str, agent: dict) -> None:
connection_id = str(uuid.uuid4())
headers = {"Authorization": f"Bearer {agent['api_key']}"}
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as http:
-
- async def beat() -> None:
- while True:
- await asyncio.sleep(2)
- try:
- async with http.post(
- f"{SWITCH}/agents/{agent['id']}/connection/beat",
- headers=headers,
- json={"connection_id": connection_id},
- ) as r:
- await r.read()
- except aiohttp.ClientError:
- pass
-
+ # The connection is one WebSocket: events come down as JSON frames,
+ # and the server's `ping` is answered with a `pong` to stay alive.
url = (
- f"{SWITCH}/agents/{agent['id']}/events?connection_id={connection_id}"
+ f"{SWITCH.replace('http', 'ws', 1)}/agents/{agent['id']}/connection/ws"
+ f"?connection_id={connection_id}"
"&scope=all&filter=all&start_from=head&client=ftc-e2e"
)
- async with http.get(
- url, headers={**headers, "Accept": "text/event-stream"}
- ) as stream:
- if stream.status != 200:
- raise RuntimeError(
- f"{name} stream: {stream.status} {await stream.text()}"
- )
- beater = asyncio.create_task(beat())
+ async with http.ws_connect(url, headers=headers) as socket:
print(f"{name} connected", flush=True)
- data_lines: list[str] = []
- async for raw in stream.content:
- line = raw.decode().rstrip("\n")
- if line.startswith("data:"):
- data_lines.append(line[5:].strip())
+ async for raw in socket:
+ if raw.type != aiohttp.WSMsgType.TEXT:
continue
- if line or not data_lines:
+ frame = json.loads(raw.data)
+ if frame.get("event") == "refused":
+ raise RuntimeError(f"{name} socket refused: {frame.get('data')}")
+ if frame.get("event") == "ping":
+ await socket.send_json({"type": "pong", "cursor": 0})
continue
- event = json.loads("\n".join(data_lines))
- data_lines = []
+ event = frame.get("data") or {}
if event.get("type") != "message":
continue
p = event["payload"]
@@ -387,7 +369,6 @@ async def run_agent(name: str, agent: dict) -> None:
f.write(json.dumps(record) + "\n")
if p.get("addressed") and p.get("sender_name") not in AGENTS:
asyncio.create_task(reply(http, name, agent, connection_id, event))
- beater.cancel()
def why(message_id: str, name: str) -> str:
diff --git a/core/tests/switch_core/bridges/agent/api/test_beat_carries_no_position.py b/core/tests/switch_core/bridges/agent/api/test_beat_carries_no_position.py
deleted file mode 100644
index 694f21776..000000000
--- a/core/tests/switch_core/bridges/agent/api/test_beat_carries_no_position.py
+++ /dev/null
@@ -1,154 +0,0 @@
-"""A heartbeat says the client is still there, and nothing more.
-
-Clients report how far they have read on every beat: a position from clients
-on agent-protocol revision 9, one number from older ones, a ticket from older
-ones reading the room conversation. The server does not read it. The
-connection's position is moved only by the stream, as it sends, and a client
-that drops resumes from the position it sends when it reopens, which is
-checked against the database then.
-
-So a beat cannot move the server past events it never sent, whatever number it
-carries, and it costs no database query. Beats arrive every two seconds from
-every client, and a burst of them follows any restart.
-"""
-
-from __future__ import annotations
-
-from types import SimpleNamespace
-from typing import Any
-
-import pytest
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-
-from switch_core.bridges.agent.api.handlers import connection_beat
-from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
-from switch_core.bridges.agent.protocol.agent_connections import (
- PROTOCOL_VERSION,
- AgentConnection,
- AgentConnectionRegistry,
- ClientDeclaration,
-)
-from switch_core.bridges.agent.protocol.event_buffer import TICKET_FLOOR, EventBuffer
-from switch_core.bridges.agent.protocol.positions import OWN_EVENTS, room_source
-from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
-from switch_core.db.models import TENANT_ZERO_ID
-from tests.db_buffer import seed_agent
-
-AGENT_ID = "agent-1"
-CONN_ID = "conn-1"
-ROOM_ID = "room-1"
-
-
-class _Protocol:
- def __init__(self, event_buffer: EventBuffer | None) -> None:
- self.event_buffer = event_buffer
- self.connections = AgentConnectionRegistry()
-
-
-def _connect(protocol: _Protocol) -> AgentConnection:
- conn = protocol.connections.open(
- agent_id=AGENT_ID,
- connection_id=CONN_ID,
- scope="single",
- delivery_filter="all",
- spawn_capable=False,
- position={OWN_EVENTS: 4, room_source(ROOM_ID): 7},
- declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
- expected_generation=None,
- )
- conn.stream_attached = True
- return conn
-
-
-async def _beat(protocol: _Protocol, conn: AgentConnection, **body: Any) -> Any:
- return await connection_beat(
- AGENT_ID,
- ConnectionBeatRequest.model_validate(
- {"connection_id": CONN_ID, "generation": conn.stream_generation, **body}
- ),
- agent=SimpleNamespace(id=AGENT_ID, tenant_id=TENANT_ZERO_ID), # type: ignore[arg-type]
- protocol=protocol, # type: ignore[arg-type]
- )
-
-
-async def test_a_beat_counts_and_reads_nothing_from_the_database() -> None:
- protocol = _Protocol(event_buffer=None)
- conn = _connect(protocol)
-
- await _beat(protocol, conn, cursor=9)
-
- assert conn.beats == 1
-
-
- "report",
- [
- {"position": {OWN_EVENTS: 500, room_source(ROOM_ID): 90}},
- {"position": {room_source("elsewhere"): 3}},
- {"position": {"not a source": 1}},
- {"cursor": 500},
- {"cursor": TICKET_FLOOR + 1},
- ],
- ids=["ahead", "another room", "not a position", "one number", "ticket"],
-)
-async def test_whatever_a_beat_reports_the_position_stays_where_the_stream_put_it(
- report: dict[str, Any],
-) -> None:
- """Ahead of what was sent, as from another history (a database restored
- from a backup, another server); a room the connection has no place in;
- something that is not a position at all. The beat counts, and the position
- is untouched."""
- protocol = _Protocol(event_buffer=None)
- conn = _connect(protocol)
-
- await _beat(protocol, conn, **report)
-
- assert conn.beats == 1
- assert conn.position == {OWN_EVENTS: 4, room_source(ROOM_ID): 7}
-
-
-async def test_the_next_event_after_a_beat_from_another_history_is_delivered(
- db_buffer: EventBuffer, session_factory: async_sessionmaker[AsyncSession]
-) -> None:
- """The loss a beat could once cause: had the server adopted 500, the
- connection would sit past the agent's next event and skip it silently."""
- await seed_agent(session_factory, AGENT_ID)
- protocol = _Protocol(db_buffer)
- conn = protocol.connections.open(
- agent_id=AGENT_ID,
- connection_id=CONN_ID,
- scope="single",
- delivery_filter="addressed",
- spawn_capable=False,
- position={OWN_EVENTS: 0},
- declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
- expected_generation=None,
- )
- conn.stream_attached = True
- protocol.connections.claim_room(conn, ROOM_ID, takeover=True)
-
- await _beat(protocol, conn, position={OWN_EVENTS: 500})
- sequence = await db_buffer.enqueue(
- TENANT_ZERO_ID,
- AGENT_ID,
- ROOM_ID,
- AgentEvent(
- type="message",
- room_id=ROOM_ID,
- payload=MessagePayload(
- addressed=True,
- sender="@u:s",
- sender_name="u",
- message_id="$m",
- body="hi",
- timestamp=0,
- ),
- ),
- room_seq=1,
- message_key="$m",
- )
-
- delivered = await db_buffer.read_from(
- TENANT_ZERO_ID, AGENT_ID, conn.position[OWN_EVENTS], rooms={ROOM_ID}
- )
- assert [item.seq for item in delivered] == [sequence]
diff --git a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
index 6cd7a3155..584b921fe 100644
--- a/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
+++ b/core/tests/switch_core/bridges/agent/api/test_beat_refusal_codes.py
@@ -11,14 +11,11 @@ winner; it did not stop the loser from coming back.
from __future__ import annotations
-from types import SimpleNamespace
from typing import Any
import pytest
-from fastapi import HTTPException
-from switch_core.bridges.agent.api.handlers import _record_beat, connection_beat
-from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
+from switch_core.bridges.agent.api.handlers import _record_beat
from switch_core.bridges.agent.protocol.agent_connections import (
PROTOCOL_VERSION,
AgentConnectionRegistry,
@@ -87,82 +84,3 @@ async def test_a_tick_with_no_stream_is_refused_as_something_to_reopen(
# Recoverable, and distinct from the takeover: this client still holds the
# connection and only has to reconnect.
assert caught.value.code == "no_stream"
-
-
-# ── Over HTTP, for a client on the event stream ─────────────────────────────
-#
-# A client built before the WebSocket beats with `POST /connection/beat`, kept
-# with the event stream for a compatibility window. Its refusals are the same
-# ones, answered as HTTP with the code beside the prose.
-
-
-async def _post_beat(protocol: _Protocol, generation: int | None) -> Any:
- return await connection_beat(
- AGENT_ID,
- ConnectionBeatRequest(connection_id=CONN_ID, generation=generation),
- SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]
- protocol, # type: ignore[arg-type]
- )
-
-
-async def test_a_superseded_beat_is_answered_409_taken_over(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- displaced = _open(protocol).stream_generation
- _open(protocol)
-
- with pytest.raises(HTTPException) as caught:
- await _post_beat(protocol, displaced)
-
- assert caught.value.status_code == 409
- assert caught.value.detail["code"] == "taken_over" # type: ignore[index]
- message = caught.value.detail["message"] # type: ignore[index]
- assert str(displaced) in message and str(displaced + 1) in message
-
-
-async def test_a_beat_with_no_stream_is_answered_409_no_stream(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- conn = _open(protocol)
- protocol.connections.detach_stream(conn, conn.stream_generation)
-
- with pytest.raises(HTTPException) as caught:
- await _post_beat(protocol, conn.stream_generation)
-
- assert caught.value.status_code == 409
- assert caught.value.detail["code"] == "no_stream" # type: ignore[index]
-
-
-async def test_an_unfenced_beat_from_a_fenced_holder_is_refused_on_its_own_code(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- _open(protocol)
-
- with pytest.raises(HTTPException) as caught:
- await _post_beat(protocol, None)
-
- assert caught.value.status_code == 409
- assert caught.value.detail["code"] == "unfenced" # type: ignore[index]
-
-
-async def test_a_beat_for_an_unknown_connection_is_answered_404(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
-
- with pytest.raises(HTTPException) as caught:
- await _post_beat(protocol, None)
-
- assert caught.value.status_code == 404
-
-
-async def test_a_client_that_cannot_be_fenced_still_beats(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- _open(protocol, speaks=None)
-
- assert (await _post_beat(protocol, None))["ok"] is True
diff --git a/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py b/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py
deleted file mode 100644
index 30f932778..000000000
--- a/core/tests/switch_core/bridges/agent/api/test_event_stream_compat.py
+++ /dev/null
@@ -1,449 +0,0 @@
-"""An old client keeps working against this server.
-
-Agent runtimes up to 0.8.x connect over the Server-Sent Events stream and beat
-with `POST /connection/beat`, declaring agent-protocol 6 (0.7.x) or 7 (0.8.0,
-which added the hosted worker).
-The WebSocket replaces both, and both are kept beside it for a compatibility
-window (the expand half of expand/contract), so these drive the server the way
-such a client does: open the stream, read events, beat, and lapse when it stops
-beating. The stream encodes the same frames the socket sends, from the same
-loop, so the rules themselves are tested once, on the frames.
-
-Such a client keeps one number. Its stream reads the room's conversation as
-well as the agent's own events, so the number it is handed with each event is
-a ticket standing for the whole position after it.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import time
-from collections.abc import AsyncIterator
-from types import SimpleNamespace
-from typing import Any
-
-import pytest
-from fastapi import HTTPException
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from starlette.responses import StreamingResponse
-
-from switch_core.artifacts import contract_range
-from switch_core.bridges.agent.api.handlers import connection_beat, poll_events
-from switch_core.bridges.agent.api.schemas import ConnectionBeatRequest
-from switch_core.bridges.agent.api.session_reporter import SessionReporter
-from switch_core.bridges.agent.protocol.agent_connections import (
- HEARTBEAT_TTL_SECONDS,
- AgentConnectionRegistry,
- ClientDeclaration,
-)
-from switch_core.bridges.agent.protocol.event_buffer import TICKET_FLOOR, EventBuffer
-from switch_core.bridges.agent.protocol.positions import OWN_EVENTS, room_source
-from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
-from switch_core.db.models import TENANT_ZERO_ID
-from switch_core.observability.metrics import MetricsRegistry, install, uninstall
-from tests.db_buffer import say, seed_member
-
-AGENT_ID = "agent-1"
-ROOM = "room-a"
-# What agent-runtime 0.7.x declares: agent-protocol 6, accepting 1.
-OLD_RUNTIME = {
- "protocol_version": 6,
- "protocol_accepts": 1,
- "client": "agent-runtime",
- "client_version": "0.7.2",
-}
-
-
[email protected](autouse=True)
-async def _agent_row(session_factory: async_sessionmaker[AsyncSession]) -> None:
- """The agent these streams are for, in the room they declare, so its
- events have a row to belong to and the room a conversation to read."""
- await seed_member(session_factory, AGENT_ID, ROOM)
-
-
-class _Protocol:
- def __init__(self, event_buffer: EventBuffer) -> None:
- self.event_buffer = event_buffer
- self.connections = AgentConnectionRegistry()
- self.approval_outcomes = None
- self.sessions = SessionReporter(None)
-
- async def record_client_declaration(
- self, agent_id: str, connection_id: str, declaration: ClientDeclaration
- ) -> None:
- return None
-
- async def require_room_member(self, agent_id: str, room_id: str) -> None:
- return None
-
-
-def _agent() -> Any:
- return SimpleNamespace(
- id=AGENT_ID,
- tenant_id=TENANT_ZERO_ID,
- metadata_={"known_agent_type": "claude-code"},
- )
-
-
-def _message(body: str) -> AgentEvent:
- return AgentEvent(
- type="message",
- room_id=ROOM,
- payload=MessagePayload(
- addressed=True,
- sender="@u:s",
- sender_name="u",
- message_id=f"${body}",
- body=body,
- timestamp=0,
- ),
- )
-
-
-async def _keep(protocol: _Protocol, body: str) -> int:
- """A message addressed to the agent, kept as one of its own events."""
- return await protocol.event_buffer.enqueue(
- TENANT_ZERO_ID,
- AGENT_ID,
- ROOM,
- _message(body),
- room_seq=0,
- message_key=f"${body}",
- )
-
-
-async def _open(protocol: _Protocol, **kw: Any) -> StreamingResponse:
- params: dict[str, Any] = {
- "accept": "text/event-stream",
- "connection_id": "c1",
- "rooms": ROOM,
- **OLD_RUNTIME,
- **kw,
- }
- resp = await poll_events(
- agent_id=AGENT_ID,
- agent=_agent(),
- protocol=protocol, # type: ignore[arg-type]
- config=None, # type: ignore[arg-type]
- **params,
- )
- assert isinstance(resp, StreamingResponse)
- return resp
-
-
-class _Reader:
- """Reads a response body the way the old runtime's `readSse` did."""
-
- def __init__(self, body: AsyncIterator[Any]) -> None:
- self.body = body
- self.buffered = ""
-
- async def frame(self, timeout: float = 2.0) -> tuple[str, str | None, dict]:
- while "\n\n" not in self.buffered:
- chunk = await asyncio.wait_for(anext(self.body), timeout)
- self.buffered += chunk.decode() if isinstance(chunk, bytes) else chunk
- raw, self.buffered = self.buffered.split("\n\n", 1)
- event, seq, data = "message", None, None
- for line in raw.split("\n"):
- if line.startswith(":"):
- continue
- if line.startswith("event: "):
- event = line[len("event: ") :]
- elif line.startswith("id: "):
- seq = line[len("id: ") :]
- elif line.startswith("data: "):
- data = line[len("data: ") :]
- if data is None:
- return await self.frame(timeout)
- return event, seq, json.loads(data)
-
- async def close(self) -> None:
- await self.body.aclose() # type: ignore[attr-defined]
-
-
-async def _beat(protocol: _Protocol, cursor: int, generation: int | None) -> Any:
- return await connection_beat(
- AGENT_ID,
- ConnectionBeatRequest.model_validate(
- {"connection_id": "c1", "cursor": cursor, "generation": generation}
- ),
- SimpleNamespace(id=AGENT_ID), # type: ignore[arg-type]
- protocol, # type: ignore[arg-type]
- )
-
-
-async def test_an_old_client_connects_receives_beats_and_lapses(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- stream = _Reader((await _open(protocol)).body_iterator)
-
- event, _, state = await stream.frame()
- assert event == "connection_state"
- assert state["rooms"] == [ROOM]
- generation = state["generation"]
-
- seq = await _keep(protocol, "hello")
- event, sent_id, data = await stream.frame()
- assert event == "message"
- assert data["payload"]["body"] == "hello"
- # A ticket, not the event's own number, and no position it would not read.
- assert sent_id is not None
- ticket = int(sent_id)
- assert ticket >= TICKET_FLOOR
- assert data["sequence"] == ticket
- assert "position" not in data
-
- # The beat carries the ticket, which is not read. It is answered with where
- # the connection stands: the last ticket sent and the position it stands for.
- beaten = await _beat(protocol, ticket, generation)
- assert beaten == {
- "ok": True,
- "rooms": [ROOM],
- "cursor": ticket,
- "position": {OWN_EVENTS: seq, room_source(ROOM): 0},
- }
-
- conn = protocol.connections.get("c1")
- assert conn is not None
- conn.last_beat = time.monotonic() - (HEARTBEAT_TTL_SECONDS + 1)
- await _keep(protocol, "not after a lapse")
-
- event, _, data = await stream.frame()
- assert event == "evicted"
- assert data["code"] == "heartbeat_lapsed"
- await stream.close()
-
- with pytest.raises(HTTPException) as caught:
- await _beat(protocol, ticket, generation)
- assert caught.value.status_code == 404
-
-
-async def test_an_old_client_resumes_from_last_event_id(db_buffer: EventBuffer) -> None:
- """A number below the ticket floor is one of the agent's own events: what
- a client was handed before its stream read the conversation."""
- protocol = _Protocol(db_buffer)
- first = await _keep(protocol, "seen")
- await _keep(protocol, "missed")
-
- # The old runtime sent the cursor both ways; the header wins.
- stream = _Reader(
- (
- await _open(protocol, start_from="head", last_event_id=str(first))
- ).body_iterator
- )
- await stream.frame()
- event, _, data = await stream.frame()
- await stream.close()
-
- assert event == "message"
- assert data["payload"]["body"] == "missed"
-
-
-async def test_an_old_client_resumes_from_the_ticket_it_was_handed(
- db_buffer: EventBuffer, session_factory: async_sessionmaker[AsyncSession]
-) -> None:
- """The ticket holds its place in the room as well as in the agent's own
- events, so a reconnect with it is sent both what was said while it was
- away and what was kept for it."""
- protocol = _Protocol(db_buffer)
- stream = _Reader((await _open(protocol)).body_iterator)
- await stream.frame()
- await _keep(protocol, "seen")
- _, ticket, _ = await stream.frame()
- await stream.close()
-
- await say(session_factory, ROOM, "said while away")
- await _keep(protocol, "kept while away")
-
- stream = _Reader(
- (await _open(protocol, start_from="head", last_event_id=ticket)).body_iterator
- )
- await stream.frame()
- said = await stream.frame()
- kept = await stream.frame()
- await stream.close()
-
- assert said[2]["payload"]["body"] == "said while away"
- assert kept[2]["payload"]["body"] == "kept while away"
-
-
-async def test_an_unknown_ticket_opens_with_a_gap(db_buffer: EventBuffer) -> None:
- """A ticket the server no longer holds (expired, or another server's)
- cannot be honoured. The stream opens where things are now and says so
- first, so the client re-reads its rooms rather than believing it is caught
- up."""
- protocol = _Protocol(db_buffer)
- lost = TICKET_FLOOR + 12345
-
- stream = _Reader((await _open(protocol, last_event_id=str(lost))).body_iterator)
- first, _, _ = await stream.frame()
- event, _, gap = await stream.frame()
- await stream.close()
-
- assert first == "connection_state"
- assert event == "gap"
- assert gap["from_sequence"] == lost
- assert gap["rooms"] == [ROOM]
- assert gap["all_rooms"] is True
- assert "no longer known" in gap["reason"]
-
-
-async def test_the_number_a_gap_hands_an_old_client_resumes_where_it_left_it(
- db_buffer: EventBuffer,
-) -> None:
- """The old runtime adopts a gap's `resumed_at` as its number. For a client
- reading the conversation too, that number is a ticket for where the stream
- resumed: sent back, it resumes there, rather than being read as the
- agent's own events from the start and replaying all of them."""
- protocol = _Protocol(db_buffer)
- await _keep(protocol, "before the gap")
- lost = TICKET_FLOOR + 12345
-
- stream = _Reader((await _open(protocol, last_event_id=str(lost))).body_iterator)
- await stream.frame()
- _, _, gap = await stream.frame()
- await stream.close()
- await _keep(protocol, "after the gap")
-
- stream = _Reader(
- (await _open(protocol, last_event_id=str(gap["resumed_at"]))).body_iterator
- )
- await stream.frame()
- event, _, data = await stream.frame()
- await stream.close()
-
- assert gap["resumed_at"] >= TICKET_FLOOR
- assert event == "message"
- assert data["payload"]["body"] == "after the gap"
-
-
-async def test_closing_the_stream_detaches_it_and_keeps_the_connection(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- stream = _Reader((await _open(protocol)).body_iterator)
- await stream.frame()
-
- await stream.close()
-
- conn = protocol.connections.get("c1")
- assert conn is not None
- assert not conn.stream_attached
- assert protocol.connections.live_agents_by_transport() == {
- ("detached", "agent-runtime"): 1
- }
-
-
-async def test_a_stream_attached_over_sse_is_reported_as_sse(
- db_buffer: EventBuffer,
-) -> None:
- protocol = _Protocol(db_buffer)
- stream = _Reader((await _open(protocol)).body_iterator)
- await stream.frame()
-
- assert protocol.connections.live_agents_by_transport() == {
- ("sse", "agent-runtime"): 1
- }
- await stream.close()
-
-
-async def test_a_client_that_moves_to_the_socket_is_reported_on_the_socket(
- db_buffer: EventBuffer,
-) -> None:
- """The transport is the last attach's: an upgraded Console reattaching the
- same connection over the socket stops counting as an old client."""
- protocol = _Protocol(db_buffer)
- stream = _Reader((await _open(protocol)).body_iterator)
- await stream.frame()
-
- protocol.connections.open(
- agent_id=AGENT_ID,
- connection_id="c1",
- scope="single",
- delivery_filter="all",
- spawn_capable=False,
- position={OWN_EVENTS: 0},
- declaration=ClientDeclaration(speaks=8, accepts=1, artifact="agent-runtime"),
- expected_generation=None,
- transport="websocket",
- )
-
- assert protocol.connections.live_agents_by_transport() == {
- ("websocket", "agent-runtime"): 1
- }
- await stream.close()
-
-
-async def test_a_refused_stream_is_counted_like_a_refused_socket(
- db_buffer: EventBuffer,
-) -> None:
- registry = MetricsRegistry()
- install(registry)
- try:
- with pytest.raises(HTTPException) as caught:
- await _open(_Protocol(db_buffer), protocol_version=99, protocol_accepts=99)
- finally:
- uninstall()
-
- assert caught.value.status_code == 409
- refused = next(
- p for p in registry.collect() if p.name == "switch.agent.connections_refused"
- )
- assert [(dict(p.attributes), p.value) for p in refused.numbers] == [
- ({"reason": "protocol"}, 1.0)
- ]
-
-
-# ── The declared ranges still meet ──────────────────────────────────────────
-
-
-def _overlap(a: tuple[int, int], b: tuple[int, int]) -> bool:
- """Whether two (speaks, accepts) ranges share a revision."""
- return a[1] <= b[0] and b[1] <= a[0]
-
-
-# What the releases before the socket declare, on either side: agent-protocol
-# 6 (agent-runtime 0.7.x, and switch-core before the hosted worker) and 7
-# (agent-runtime 0.8.0, and switch-core with it).
-BEFORE_THE_SOCKET = [(6, 1), (7, 1)]
-
-
[email protected]("old", BEFORE_THE_SOCKET)
-def test_an_old_runtime_and_this_server_share_a_revision(old: tuple[int, int]) -> None:
- server = contract_range("agent-protocol", "switch-core")
-
- assert _overlap(old, (server.speaks, server.accepts))
-
-
[email protected]("old", BEFORE_THE_SOCKET)
-def test_this_runtime_and_an_old_server_share_a_revision(old: tuple[int, int]) -> None:
- runtime = contract_range("agent-protocol", "agent-runtime")
-
- assert _overlap((runtime.speaks, runtime.accepts), old)
-
-
-def test_this_runtime_and_this_server_share_the_newest_revision() -> None:
- server = contract_range("agent-protocol", "switch-core")
- runtime = contract_range("agent-protocol", "agent-runtime")
-
- assert runtime.speaks == server.speaks
-
-
[email protected]("speaks", [6, 7])
-def test_this_server_admits_an_old_runtime(speaks: int) -> None:
- registry = AgentConnectionRegistry()
- conn = registry.open(
- agent_id=AGENT_ID,
- connection_id="c1",
- scope="single",
- delivery_filter="all",
- spawn_capable=False,
- position={OWN_EVENTS: 0},
- declaration=ClientDeclaration(speaks=speaks, accepts=1),
- expected_generation=None,
- transport="sse",
- )
-
- assert conn.stream_transport == "sse"
diff --git a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
index eb6efc0d1..7bb5decd0 100644
--- a/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
+++ b/core/tests/switch_core/bridges/agent/api/test_events_endpoint_dispatch.py
@@ -1,9 +1,9 @@
-"""Opening an agent connection, and the /events endpoint.
+"""Opening an agent connection, and what is left of the /events endpoint.
-The WebSocket and the event stream both open through `_open_connection`,
-which these drive directly: validation, protocol ranges, declarations, room
-claims and session reporting. `/events` with `Accept: text/event-stream` opens
-the event stream an old client still asks for; anything else long polls.
+The connection opens over the WebSocket through `_open_connection`, which
+these drive directly: validation, protocol ranges, declarations, room claims
+and session reporting. `/events` itself is now only the long poll, and a
+request for the old Server-Sent Events stream is refused with what to do.
"""
from __future__ import annotations
@@ -15,7 +15,6 @@ from typing import Any
import pytest
from fastapi import HTTPException
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from starlette.responses import StreamingResponse
from switch_core.bridges.agent.api import session_reporter
from switch_core.bridges.agent.api.handlers import (
@@ -100,7 +99,6 @@ async def _call(protocol: _Protocol, **kw: Any) -> Any:
"agent_id": AGENT_ID,
"agent": _agent(),
"protocol": protocol,
- "config": None,
"timeout": 0,
"accept": None,
"connection_id": None,
@@ -150,108 +148,25 @@ async def test_without_the_sse_accept_header_it_long_polls(
assert resp.status_code == 204
-async def test_with_the_sse_accept_header_it_opens_the_event_stream(
+async def test_asking_for_the_old_stream_is_refused_with_a_remedy(
db_buffer: EventBuffer,
) -> None:
- """How a runtime built before the WebSocket connects, kept for a
- compatibility window."""
+ """An old runtime is told to update, not left retrying a stream that is
+ gone."""
protocol = _Protocol(db_buffer)
- resp = await poll_events(
- agent_id=AGENT_ID,
- agent=_agent(),
- protocol=protocol, # type: ignore[arg-type]
- config=None, # type: ignore[arg-type]
- accept="text/event-stream",
- connection_id="c1",
- scope="all",
- )
+ with pytest.raises(HTTPException) as excinfo:
+ await poll_events(
+ agent_id=AGENT_ID,
+ agent=_agent(),
+ protocol=protocol, # type: ignore[arg-type]
+ accept="text/event-stream",
+ )
- assert isinstance(resp, StreamingResponse)
- assert resp.media_type == "text/event-stream"
- # Buffering proxies would defeat the point of a push channel.
- assert resp.headers["x-accel-buffering"] == "no"
- assert resp.headers["connection"] == "keep-alive"
+ assert excinfo.value.status_code == 410
+ assert excinfo.value.detail["code"] == "transport_removed" # type: ignore[index]
+ assert "connection/ws" in excinfo.value.detail["message"] # type: ignore[index]
assert not protocol.polled
- conn = protocol.connections.get("c1")
- assert conn is not None
- assert conn.scope == "all"
- assert conn.stream_transport == "sse"
-
-
-async def _open_sse(protocol: _Protocol, **kw: Any) -> StreamingResponse:
- resp = await poll_events(
- agent_id=AGENT_ID,
- agent=_agent(),
- protocol=protocol, # type: ignore[arg-type]
- config=None, # type: ignore[arg-type]
- accept="text/event-stream",
- connection_id="c1",
- client="agent-runtime",
- client_version="0.8.0",
- **kw,
- )
- assert isinstance(resp, StreamingResponse)
- return resp
-
-
-def _opened_lines(caplog: pytest.LogCaptureFixture) -> list[str]:
- return [r.getMessage() for r in caplog.records if "[CONN] opened" in r.getMessage()]
-
-
-async def test_a_runtime_that_fell_back_from_the_socket_says_so_in_the_log(
- db_buffer: EventBuffer, caplog: pytest.LogCaptureFixture
-) -> None:
- """A new runtime whose socket never opened asks for the event stream too,
- and must not read in the log as a client built before the socket."""
- caplog.set_level("INFO")
- await _open_sse(
- _Protocol(db_buffer),
- fallback="socket_never_opened",
- fallback_close_code="1006",
- )
-
- (line,) = _opened_lines(caplog)
- assert line.endswith("transport=sse fallback=socket_never_opened:1006")
-
-
-async def test_a_client_that_did_not_fall_back_logs_the_line_as_before(
- db_buffer: EventBuffer, caplog: pytest.LogCaptureFixture
-) -> None:
- caplog.set_level("INFO")
- await _open_sse(_Protocol(db_buffer))
-
- (line,) = _opened_lines(caplog)
- assert line.endswith("transport=sse")
- assert "fallback" not in line
-
-
- ("reason", "close_code", "logged"),
- [
- ("socket_never_opened", None, "socket_never_opened"),
- ("x" * 5000, "1006", "other:1006"),
- ("socket_never_opened", "1006 rm -rf", "socket_never_opened"),
- ("socket_never_opened", "123456", "socket_never_opened"),
- ],
-)
-async def test_what_the_client_says_about_its_fallback_is_clamped(
- reason: str,
- close_code: str | None,
- logged: str,
- db_buffer: EventBuffer,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- """Both values are the client's own, so only known reasons and short
- numeric close codes reach the log."""
- caplog.set_level("INFO")
- await _open_sse(
- _Protocol(db_buffer), fallback=reason, fallback_close_code=close_code
- )
-
- (line,) = _opened_lines(caplog)
- assert line.endswith(f" fallback={logged}")
-
async def test_opening_registers_the_connection(db_buffer: EventBuffer) -> None:
protocol = _Protocol(db_buffer)
@@ -262,7 +177,6 @@ async def test_opening_registers_the_connection(db_buffer: EventBuffer) -> None:
assert protocol.connections.get("c1") is conn
assert conn.scope == "all"
assert conn.stream_attached
- assert conn.stream_transport == "websocket"
async def test_streaming_without_a_connection_id_is_refused(
@@ -283,14 +197,7 @@ async def test_streaming_with_an_over_long_connection_id_is_refused(
which Postgres caps at 8000 bytes, so one this long cannot be opened."""
protocol = _Protocol(db_buffer)
with pytest.raises(HTTPException) as excinfo:
- await poll_events(
- agent_id=AGENT_ID,
- agent=_agent(),
- protocol=protocol, # type: ignore[arg-type]
- config=None, # type: ignore[arg-type]
- accept="text/event-stream",
- connection_id="c" * 7900,
- )
+ await _call(protocol, accept="text/event-stream", connection_id="c" * 7900)
assert excinfo.value.status_code == 400
assert "connection_id" in excinfo.value.detail
@@ -454,7 +361,6 @@ async def _start(
*,
start_from: str = "head",
start_at: str | None = None,
- last_event_id: str | None = None,
reads_conversation: bool = False,
) -> tuple[Position, int | None]:
return await _resolve_start_position(
@@ -462,7 +368,6 @@ async def _start(
_agent(),
start_at=start_at,
start_from=start_from,
- last_event_id=last_event_id,
reads_conversation=reads_conversation,
)
@@ -481,14 +386,6 @@ async def test_head_means_only_what_happens_next(db_buffer: EventBuffer) -> None
assert await _start(protocol, start_at="head") == ({OWN_EVENTS: seq}, None)
-async def test_last_event_id_wins_over_start_from(db_buffer: EventBuffer) -> None:
- protocol = _Protocol(db_buffer)
- assert await _start(protocol, start_from="head", last_event_id="42") == (
- {OWN_EVENTS: 42},
- None,
- )
-
-
async def test_explicit_start_from_is_honoured(db_buffer: EventBuffer) -> None:
protocol = _Protocol(db_buffer)
assert await _start(protocol, start_from="17") == ({OWN_EVENTS: 17}, None)
@@ -631,30 +528,36 @@ async def test_reconnect_during_bookkeeping_cannot_detach_the_new_stream(
await release.wait()
async def open_stream() -> Any:
- return await poll_events(
- agent_id=AGENT_ID,
+ _conn, frames = await _open_connection(
agent=_agent(),
protocol=protocol, # type: ignore[arg-type]
config=None, # type: ignore[arg-type]
- accept="text/event-stream",
connection_id="c1",
+ scope="single",
+ event_filter="all",
+ start_from="head",
+ spawn_capable=False,
+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
+ rooms=None,
+ expected_generation=None,
)
+ return frames
monkeypatch.setattr(protocol, "record_client_declaration", record)
first = asyncio.create_task(open_stream())
await entered.wait()
newer = await open_stream()
- await anext(newer.body_iterator)
+ await anext(newer)
release.set()
older = await first
- frames = [frame async for frame in older.body_iterator]
+ frames = [frame async for frame in older]
assert frames == []
conn = protocol.connections.get("c1")
assert conn is not None
assert protocol.connections.beat(
AGENT_ID, "c1", conn.stream_generation
).stream_attached
- await newer.body_iterator.aclose()
+ await newer.aclose()
class _RecordingSink:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_connections.py b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
index 0dc09a2c1..c657e1f54 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_connections.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
@@ -7,8 +7,6 @@ import time
import pytest
from switch_core.bridges.agent.protocol.agent_connections import (
- FENCED_PROTOCOL_REVISION,
- HEARTBEAT_INTERVAL_SECONDS,
HEARTBEAT_LAPSED,
HEARTBEAT_TTL_SECONDS,
MAX_CONNECTIONS_PER_AGENT,
@@ -27,6 +25,7 @@ from switch_core.bridges.agent.protocol.agent_connections import (
UnfencedBeatError,
UnknownConnectionError,
)
+from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_INTERVAL_SECONDS
from switch_core.bridges.agent.protocol.positions import OWN_EVENTS
AGENT = "agent-1"
@@ -304,16 +303,6 @@ def test_a_beat_from_a_client_that_cannot_be_fenced_is_accepted() -> None:
assert registry.beat(AGENT, "c1", None).beats == 1
-def test_a_client_declaring_the_revision_before_the_fence_ticks_unfenced() -> None:
- registry = AgentConnectionRegistry()
- _open(registry, "c1", speaks=FENCED_PROTOCOL_REVISION - 1)
-
- # The server still accepts revision 1, and that revision has no incarnation
- # to return, so its tick is taken as it always was. Refusing it would break
- # every client released before the fence rather than fencing it.
- assert registry.beat(AGENT, "c1", None).beats == 1
-
-
def test_a_holder_that_carries_an_incarnation_may_not_tick_without_one() -> None:
registry = AgentConnectionRegistry()
conn = _open(registry, "c1")
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_connections_across_processes.py b/core/tests/switch_core/bridges/agent/protocol/test_connections_across_processes.py
index b40c5bce9..4a8698b2a 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_connections_across_processes.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_connections_across_processes.py
@@ -25,13 +25,11 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
from sqlalchemy.pool import NullPool
from switch_core.bridges.agent.api.handlers import (
- connection_beat,
connection_placements,
connection_subscribe,
)
from switch_core.bridges.agent.api.operations import resolve_caller
from switch_core.bridges.agent.api.schemas import (
- ConnectionBeatRequest,
ConnectionPlacementsRequest,
ConnectionSubscribeRequest,
)
@@ -581,28 +579,6 @@ class TestServedAnywhere:
)
assert claimant == "conn-2"
- async def test_an_old_clients_beat_on_the_other_process_reaches_the_holder(
- self, processes: tuple[Process, Process]
- ) -> None:
- a, b = processes
- conn = await _open(a)
- conn.last_beat = time.monotonic() - 4
- stale = conn.last_beat
-
- with tenant_scope(TENANT_ZERO_ID):
- answer = await connection_beat(
- AGENT,
- ConnectionBeatRequest(
- connection_id=CONN, generation=conn.stream_generation
- ),
- agent=_Agent(), # type: ignore[arg-type]
- protocol=b.protocol,
- )
-
- assert answer["ok"] is True
- await _eventually(lambda: conn.last_beat > stale)
- assert conn.beats == 1
-
async def test_a_session_command_relayed_on_the_other_process_reaches_the_watcher(
self, processes: tuple[Process, Process]
) -> None:
@@ -630,17 +606,6 @@ class TestTakeover:
await _eventually(lambda: loser.closure is not None)
assert loser.closure is not None
assert loser.closure.code == "taken_over"
- # The loser's late beat on the process it was on is refused as taken
- # over, on the recorded row rather than on that process's memory.
- with tenant_scope(TENANT_ZERO_ID), pytest.raises(HTTPException) as refused:
- await connection_beat(
- AGENT,
- ConnectionBeatRequest(connection_id=CONN, generation=held),
- agent=_Agent(), # type: ignore[arg-type]
- protocol=a.protocol,
- )
- assert refused.value.status_code == 409
- assert refused.value.detail["code"] == "taken_over"
# And its reattach naming what it held is refused without moving
# anything, wherever it lands.
with pytest.raises(SupersededReattachError):
@@ -709,18 +674,8 @@ class TestTakeover:
agent=_Agent(), # type: ignore[arg-type]
protocol=a.protocol,
)
- with tenant_scope(TENANT_ZERO_ID):
- beat = await connection_beat(
- AGENT,
- ConnectionBeatRequest(
- connection_id=CONN, generation=second.stream_generation
- ),
- agent=_Agent(), # type: ignore[arg-type]
- protocol=a.protocol,
- )
assert answer["placements"] == {"session-1": ROOM}
- assert beat["ok"] is True
assert first.closure is not None
assert first.closure.code == "taken_over"
assert second.closure is None
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py b/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
index f01ceba85..bdfe9368f 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_room_released.py
@@ -10,7 +10,6 @@ import pytest
from switch_core.bridges.agent.protocol.agent_connections import (
HEARTBEAT_LAPSED,
PROTOCOL_VERSION,
- ROOM_RELEASED_PROTOCOL_REVISION,
AgentConnection,
AgentConnectionRegistry,
ClientDeclaration,
@@ -100,12 +99,10 @@ def test_displacing_a_sibling_on_the_same_connection_releases_nothing() -> None:
assert conn.released_rooms == {}
[email protected]("speaks", [ROOM_RELEASED_PROTOCOL_REVISION - 1, None])
-def test_a_client_that_cannot_take_the_frame_is_not_sent_it(speaks: int | None) -> None:
- """A client from before the frame, or one that declared no revision, gets
- only what every client takes."""
+def test_a_client_that_cannot_take_the_frame_is_not_sent_it() -> None:
+ """A client that declared no revision gets only what every client takes."""
registry = AgentConnectionRegistry()
- loser = _open(registry, "loser", speaks=speaks, scope="single")
+ loser = _open(registry, "loser", speaks=None, scope="single")
winner = _open(registry, "winner", speaks=PROTOCOL_VERSION, scope="all")
registry.claim_room(loser, ROOM_A)
@@ -248,7 +245,7 @@ async def test_a_client_that_declared_nothing_is_not_sent_room_released(
def test_closing_a_local_connection_keeps_its_placements() -> None:
registry = AgentConnectionRegistry()
- conn = _open(registry, "local", speaks=ROOM_RELEASED_PROTOCOL_REVISION, scope="all")
+ conn = _open(registry, "local", speaks=PROTOCOL_VERSION, scope="all")
registry.replace_placements(conn, {"session-1": ROOM_A})
registry.close(conn.id, HEARTBEAT_LAPSED)
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_stream.py b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
index 4d001b881..11a5c1ce9 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_stream.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
@@ -38,10 +38,7 @@ from switch_core.bridges.agent.protocol.positions import OWN_EVENTS, room_source
from switch_core.bridges.agent.protocol.stream import (
KEEPALIVE,
Frame,
- encode_sse,
- encode_ws,
event_frames,
- sse_stream,
)
from switch_core.bridges.agent.protocol.types import (
AgentEvent,
@@ -1599,48 +1596,6 @@ async def test_each_delivered_event_counts_as_an_agent_bridge_event_out(
uninstall()
-class TestBothTransportsEncodeTheSameFrames:
- """The socket and the event stream an old client still reads carry the
- same frames from the same loop; only the encoding differs."""
-
- def test_an_event_carries_its_sequence_as_the_sse_id(self) -> None:
- frame = Frame("message", {"payload": {"body": "hi"}}, seq=7)
-
- assert encode_sse(frame) == (
- b'id: 7\nevent: message\ndata: {"payload":{"body":"hi"}}\n\n'
- )
- assert encode_ws(frame) == {
- "event": "message",
- "data": {"payload": {"body": "hi"}},
- "id": 7,
- }
-
- def test_a_frame_without_a_sequence_has_no_id(self) -> None:
- assert encode_sse(Frame("gap", {"reason": "x"})) == (
- b'event: gap\ndata: {"reason":"x"}\n\n'
- )
-
- def test_the_keepalive_is_a_comment_on_the_event_stream(self) -> None:
- assert encode_sse(KEEPALIVE) == b": keepalive\n\n"
-
- async def test_closing_the_event_stream_detaches_its_connection(
- self, db_buffer: EventBuffer
- ) -> None:
- registry = AgentConnectionRegistry()
- buffer = db_buffer
- conn = _open(registry)
-
- body = sse_stream(
- event_frames(conn=conn, registry=registry, buffer=buffer, approvals=None)
- )
- first = await anext(body)
- await body.aclose()
-
- assert first.startswith(b"event: connection_state\n")
- assert not conn.stream_attached
- assert registry.get(conn.id) is conn
-
-
async def test_a_pass_reads_in_one_session(
session_factory: async_sessionmaker[AsyncSession], room_bells: MessageListener
) -> None:
diff --git a/core/tests/switch_core/observability/test_connection_metrics.py b/core/tests/switch_core/observability/test_connection_metrics.py
index 39f212c01..0915d355f 100644
--- a/core/tests/switch_core/observability/test_connection_metrics.py
+++ b/core/tests/switch_core/observability/test_connection_metrics.py
@@ -224,46 +224,20 @@ def test_the_gauge_still_reports_zero_with_nobody_connected() -> None:
# ── Refused ──────────────────────────────────────────────────────────────────
-async def test_the_event_stream_is_served_so_never_refused_as_removed(
- registry,
-) -> None:
- """Kept for old clients for a compatibility window, so `transport_removed`
- does not fire; a stream refused for another reason is counted for it."""
+async def test_a_request_for_the_old_event_stream_is_counted(registry) -> None:
with pytest.raises(HTTPException):
await handlers.poll_events(
agent_id=AGENT_ID,
agent=Agent(id=AGENT_ID, name="agent"),
protocol=None, # type: ignore[arg-type]
- config=None, # type: ignore[arg-type]
accept="text/event-stream",
)
assert _counts(registry, "switch.agent.connections_refused") == {
- (("reason", "other"),): 1.0
+ (("reason", "transport_removed"),): 1.0
}
-def test_a_stream_attached_over_sse_is_counted_as_sse() -> None:
- connections = AgentConnectionRegistry()
- connections.open(
- agent_id="a1",
- connection_id="c1",
- scope="all",
- delivery_filter="all",
- spawn_capable=False,
- position={OWN_EVENTS: 0},
- declaration=ClientDeclaration(speaks=6, accepts=1, artifact="agent-runtime"),
- expected_generation=None,
- transport="sse",
- )
- _open(connections, "a2", "c2", "agent-runtime")
-
- readings = _connected_readings(_probes(connections.live_agents_by_transport))
-
- assert readings[(("client", "agent-runtime"), ("transport", "sse"))] == 1.0
- assert readings[(("client", "agent-runtime"), ("transport", "websocket"))] == 1.0
-
-
class _EmptyBuffer(EventBuffer):
"""A buffer with nothing queued, answered without a database.