Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(client): stop two loops that retry an impossible request forever (#353)

Louis Amaudruz9994ec57d3fd
diff --git a/artifacts.yaml b/artifacts.yaml
index f04c95d03..b092b08c1 100644
--- a/artifacts.yaml
+++ b/artifacts.yaml
@@ -82,7 +82,7 @@ artifacts:
agent-runtime:
description: The Switch protocol client and MCP runtime, published to a registry.
- version: 0.3.4
+ version: 0.4.1
declared_in: console/packages/switch-agent-runtime/package.json
sidecar:
@@ -90,7 +90,7 @@ artifacts:
The remote runtime Switch Console deploys to an agent host. Deployed rather
than published, so nothing else declares its version — this file is its
only source and the constant is generated from it.
- version: 1.9.6
+ version: 1.9.7
# ── Published under the switch-core release ────────────────────────────────
# No version of their own: `helm package` and the image build stamp them with
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts
index 3e5e4652a..ebcf6ff59 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts
@@ -3,7 +3,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { InjectionTarget } from './injection-sink';
-import { type PromptInjector, RoomConnection } from './room-connection';
+import { MAX_ROOM_TURN_MS, type PromptInjector, RoomConnection } from './room-connection';
import { resolveSessionControl } from './session-control';
import type { AgentBridgeEvent, AttachmentRef } from './switch-event-format';
@@ -1454,3 +1454,225 @@ describe('repointing a restored session', () => {
conn.stop();
});
});
+
+/**
+ * A turn that cannot end (CHOO-2274).
+ *
+ * Measured on a live deployment: a session reported `working` against a room it
+ * no longer held, every five seconds, for 32 hours. The server rejected each
+ * report — the room id it carried was null — and nothing here noticed, because
+ * the only thing that ever closed a turn was the agent going idle, and that
+ * signal was never coming. Three ways out now: the room being withdrawn, the
+ * report being refused, and the clock.
+ */
+describe('a turn that cannot end', () => {
+ function pushable() {
+ const encoder = new TextEncoder();
+ let controller!: ReadableStreamDefaultController<Uint8Array>;
+ const stream = new ReadableStream<Uint8Array>({
+ start(c) {
+ controller = c;
+ },
+ });
+ return { stream, push: (frame: string) => controller.enqueue(encoder.encode(frame)) };
+ }
+
+ /** A session already in a room, already mid-turn: exactly the shape the
+ * stuck session was in. `spawnTurn` opens the turn as `start()` runs. */
+ function workingSession(opts: { runtimeStateStatus?: number } = {}) {
+ const { stream, push } = pushable();
+ const fetchMock = vi.fn(async (url: string) => {
+ const u = String(url);
+ if (u.includes('/events')) {
+ return { ok: true, status: 200, body: stream, text: async (): Promise<string> => '' };
+ }
+ if (u.includes('/runtime-state') && opts.runtimeStateStatus) {
+ return {
+ ok: false,
+ status: opts.runtimeStateStatus,
+ text: async (): Promise<string> => 'room_id must be a string',
+ };
+ }
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ vi.stubGlobal('fetch', fetchMock);
+ const conn = new RoomConnection({
+ creds,
+ roomId: 'room-1',
+ roomName: 'Room One',
+ connectionId: 'conn-1',
+ sessionId: 'session-1',
+ sink: { acquire: () => ({ write: vi.fn() }) },
+ injector,
+ control,
+ deeplinkScheme: 'switchdash',
+ isHumanTyping: () => false,
+ mediaDir,
+ spawnTurn: { threadId: 'thread-1', anchorId: 'msg-1' },
+ log: silentLog,
+ });
+ conn.start();
+ return { conn, fetchMock, push };
+ }
+
+ function loggedAt(level: 'warn' | 'error', fragment: string): boolean {
+ return silentLog[level].mock.calls.some((c) => String(c[0]).includes(fragment));
+ }
+
+ /** runtime-state posts only: the heartbeat keeps beating regardless, and it
+ * is the reporting that must stop. */
+ function reports(fetchMock: { mock: { calls: unknown[][] } }): number {
+ return fetchMock.mock.calls.filter((c) => String(c[0]).includes('/runtime-state')).length;
+ }
+
+ beforeEach(() => {
+ silentLog.debug.mockClear();
+ silentLog.info.mockClear();
+ silentLog.warn.mockClear();
+ silentLog.error.mockClear();
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ });
+
+ it('ends when the server withdraws the room', async () => {
+ const { conn, fetchMock, push } = workingSession();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(runtimeStates(fetchMock)).toContain('working');
+
+ push(`event: subscription_changed\ndata: ${JSON.stringify({ rooms: [] })}\n\n`);
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(loggedAt('warn', 'the room was withdrawn')).toBe(true);
+ expect(conn.room).toBeNull();
+
+ // The ticker is what kept the 422s coming; nothing more may be reported.
+ const settled = reports(fetchMock);
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(reports(fetchMock)).toBe(settled);
+ conn.stop();
+ });
+
+ it('gives up when the room keeps refusing the report', async () => {
+ const { conn, fetchMock } = workingSession({ runtimeStateStatus: 422 });
+ await vi.advanceTimersByTimeAsync(0);
+
+ // Five consecutive refusals: the opening push plus four ticks.
+ await vi.advanceTimersByTimeAsync(4 * 5_000);
+ expect(loggedAt('error', 'abandoning the turn')).toBe(true);
+
+ const settled = reports(fetchMock);
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(reports(fetchMock)).toBe(settled);
+ conn.stop();
+ });
+
+ it('gives up when the turn outruns the wall clock', async () => {
+ const { conn, fetchMock } = workingSession();
+ await vi.advanceTimersByTimeAsync(0);
+
+ vi.setSystemTime(Date.now() + MAX_ROOM_TURN_MS);
+ await vi.advanceTimersByTimeAsync(5_000);
+
+ expect(loggedAt('error', 'outrun the cap')).toBe(true);
+ // The room is told the turn is over rather than left showing "working".
+ expect(runtimeStates(fetchMock).at(-1)).toBe('idle');
+
+ const settled = reports(fetchMock);
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(reports(fetchMock)).toBe(settled);
+ conn.stop();
+ });
+
+ it('keeps ticking while the reports land', async () => {
+ const { conn, fetchMock } = workingSession();
+ await vi.advanceTimersByTimeAsync(0);
+
+ await vi.advanceTimersByTimeAsync(20_000);
+
+ expect(loggedAt('error', 'abandoning the turn')).toBe(false);
+ expect(runtimeStates(fetchMock).filter((s) => s === 'working').length).toBeGreaterThan(4);
+ conn.stop();
+ });
+});
+
+/**
+ * A room the server refuses outright.
+ *
+ * The open is refused, not the room — so the connection never gets as far as a
+ * room list, and re-declaring the same dead room means never connecting again.
+ * Measured on a live deployment at roughly 29 errors a minute, for over a day,
+ * with no path back.
+ */
+describe('a room the server refuses', () => {
+ beforeEach(() => {
+ silentLog.warn.mockClear();
+ silentLog.error.mockClear();
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('is surfaced and dropped, and the session is left room-less', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ const u = String(url);
+ if (!u.includes('/events')) {
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ if (u.includes('rooms=')) {
+ return {
+ ok: false,
+ status: 403,
+ body: null,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: 'Room not found: room-gone' }),
+ };
+ }
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({ start() {} }),
+ text: async (): Promise<string> => '',
+ };
+ });
+ vi.stubGlobal('fetch', fetchMock);
+ const refused: { roomId: string; status: number }[] = [];
+ const rooms: (string | null)[] = [];
+ const conn = new RoomConnection({
+ creds,
+ roomId: 'room-gone',
+ roomName: null,
+ connectionId: 'conn-1',
+ sessionId: 'session-1',
+ sink: { acquire: () => ({ write: vi.fn() }) },
+ injector,
+ control,
+ deeplinkScheme: 'switchdash',
+ isHumanTyping: () => false,
+ mediaDir,
+ onRoomChanged: (room) => rooms.push(room),
+ onRoomRejected: ({ roomId, status }) => refused.push({ roomId, status }),
+ log: silentLog,
+ });
+ conn.start();
+ await flush(8);
+
+ expect(refused).toEqual([{ roomId: 'room-gone', status: 403 }]);
+ expect(rooms).toEqual([null]);
+ expect(conn.room).toBeNull();
+ expect(silentLog.error.mock.calls.some((c) => String(c[0]).includes('refused the room'))).toBe(
+ true
+ );
+
+ const opens = fetchMock.mock.calls
+ .map((c) => String(c[0]))
+ .filter((u) => u.includes('/events'));
+ expect(opens).toHaveLength(2);
+ expect(opens[1]).not.toContain('rooms=');
+ conn.stop();
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts
index e5a8b8810..526e1a895 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts
@@ -44,6 +44,21 @@ const CONTROL_STEP_GAP_MS = 600;
// While a turn is working, re-push the activity line this often with a refreshed
// elapsed-time suffix (e.g. "· 15s") so a long-running step visibly ticks.
const ACTIVITY_TICK_INTERVAL_MS = 5_000;
+// Two bounds on a working turn, because a turn that never ends re-pushes its
+// activity line every tick for as long as the process lives. One was observed
+// running for 32 hours against a room the session no longer held, rejected
+// every five seconds. Both give up loudly rather than keep a room reading
+// "working on it" against a turn nobody is going to close.
+//
+// The failure cap catches the case where the report itself is refused — the
+// server is answering, and answering that this report can never be accepted,
+// so repeating it is the definition of a loop that cannot succeed.
+const MAX_ACTIVITY_REPORT_FAILURES = 5;
+// The wall clock catches everything else: a turn whose completion signal was
+// lost (an interrupt fires no hook) leaves reports succeeding forever against
+// work that finished long ago. Set well past any real turn, so reaching it is
+// evidence of a bug rather than of a slow agent.
+export const MAX_ROOM_TURN_MS = 4 * 60 * 60 * 1000;
export type SwitchCredentials = { agentId: string; apiEndpoint: string; token: string };
@@ -211,6 +226,14 @@ export interface RoomConnectionDeps {
* `connect_to_room` call landed on.
*/
onRoomChanged?: (roomId: string | null) => void;
+ /**
+ * The server has refused a room this connection declared — it is gone, or
+ * this agent may no longer subscribe to it. The stream has dropped it, so
+ * anything that remembers the room has to forget it here: a room id we keep
+ * is a room id we re-declare on the next connection, and the server refuses
+ * the whole connection over it.
+ */
+ onRoomRejected?: (info: { roomId: string; status: number; detail: string }) => void;
log: RoomConnectionLogger;
}
@@ -278,6 +301,8 @@ export class RoomConnection {
private workingStartedAt = 0;
/** Ticker that re-pushes the activity line with a refreshed elapsed suffix. */
private activityTicker: ReturnType<typeof setInterval> | null = null;
+ /** Consecutive rejected activity reports, reset by the first that lands. */
+ private activityFailures = 0;
private busyFallback: ReturnType<typeof setTimeout> | null = null;
private humanGateTimer: ReturnType<typeof setTimeout> | null = null;
private noTargetTimer: ReturnType<typeof setTimeout> | null = null;
@@ -295,6 +320,10 @@ export class RoomConnection {
private spawnTurn: SpawnTurn | null;
/** Notified whenever the server tells us which room this session is in. */
private readonly onRoomChanged: ((roomId: string | null) => void) | null;
+ /** Notified when the server refuses a room we declared. */
+ private readonly onRoomRejected:
+ | ((info: { roomId: string; status: number; detail: string }) => void)
+ | null;
/**
* The last room we passed to `onRoomChanged`. Deliberately not seeded from
* the declared room: a connection opened declaring one is *told* the same
@@ -331,6 +360,7 @@ export class RoomConnection {
this.startCursor = deps.startCursor;
this.spawnTurn = deps.spawnTurn ?? null;
this.onRoomChanged = deps.onRoomChanged ?? null;
+ this.onRoomRejected = deps.onRoomRejected ?? null;
this.log = deps.log;
}
@@ -349,6 +379,7 @@ export class RoomConnection {
startCursor: this.startCursor,
onEvent: (event) => this.handleEvent(event),
onRooms: (rooms) => this.adoptRoom(rooms),
+ onRoomRejected: (info) => this.handleRoomRejected(info),
onGap: (info) => this.handleGap(info),
onEvicted: (reason) =>
this.log.warn('RoomConnection: connection evicted', {
@@ -440,11 +471,13 @@ export class RoomConnection {
*/
private adoptRoom(rooms: string[]): void {
const next = rooms[0] ?? null;
- // Against what we last *reported*, not what we hold: a session launched
- // into a room declares it at open and the server confirms the same value,
- // which is the only signal the rest of the app ever gets that the session
- // is in that room.
- if (next === this.reportedRoom) return;
+ // Against what we last *reported*, not merely what we hold: a session
+ // launched into a room declares it at open and the server confirms the
+ // same value, which is the only signal the rest of the app ever gets that
+ // the session is in that room. What we hold still has to agree, or a room
+ // declared but never confirmed — then refused — would read as "no change"
+ // and nobody would learn it had gone.
+ if (next === this.reportedRoom && next === this.roomId) return;
const previous = this.roomId;
this.roomId = next;
@@ -459,11 +492,65 @@ export class RoomConnection {
roomId: next,
});
this.onRoomChanged?.(next);
- if (next && !this.openSpawnTurn()) {
+ if (next === null) {
+ // The room has been taken and there is nowhere left to report. A turn
+ // left open here is the 32-hour "working on it" bug: every field it
+ // needs is still set, so the ticker keeps pushing runtime state at a
+ // room this connection no longer covers, and the server rejects each
+ // one. End it here — the same clearing stop() does — rather than let it
+ // keep asking for something that cannot be granted.
+ this.log.warn('RoomConnection: the room was withdrawn — ending the turn', {
+ event: 'room_connection_room_withdrawn',
+ sessionId: this.sessionId,
+ previous,
+ turnWasActive: this.roomTurnActive,
+ });
+ this.clearRoomTurn();
+ return;
+ }
+ if (!this.openSpawnTurn()) {
void this.postRuntimeState('idle', null).catch(() => {});
}
}
+ /**
+ * A declared room the server refuses. Terminal, not transient.
+ *
+ * The stream has already dropped it, which arrives here as an empty room
+ * list and ends the turn. What this adds is the disclosure — at error level,
+ * because a session pointed at a room that does not exist is broken and not
+ * merely quiet — and the hand-off to whoever persisted the room, so a
+ * restart does not resurrect it.
+ */
+ private handleRoomRejected(info: { roomId: string; status: number; detail: string }): void {
+ this.log.error('RoomConnection: the server refused the room this session declared', {
+ event: 'room_connection_room_refused',
+ sessionId: this.sessionId,
+ roomId: info.roomId,
+ status: info.status,
+ detail: info.detail,
+ });
+ this.onRoomRejected?.(info);
+ }
+
+ /**
+ * Drop the current room turn without reporting into the room.
+ *
+ * For the cases where reporting is impossible or is itself what is failing:
+ * the room is gone, or the report is being refused. Every caller says why at
+ * warn or error level — a turn that ends without the room being told is a
+ * degraded outcome and must not be silent.
+ */
+ private clearRoomTurn(): void {
+ this.roomTurnActive = false;
+ this.currentThreadId = null;
+ this.currentAnchorId = null;
+ this.runtimeState = 'idle';
+ this.lastActivityDetail = null;
+ this.activityFailures = 0;
+ this.stopActivityTicker();
+ }
+
/** Stop the loops and clear any lingering runtime-state surface. */
stop(): void {
if (this.stopped) return;
@@ -801,7 +888,10 @@ export class RoomConnection {
// the generic indicator, and idle/awaiting-input carry no activity.
this.lastActivityDetail = null;
if (state === 'working') {
- if (!wasWorking) this.workingStartedAt = Date.now();
+ if (!wasWorking) {
+ this.workingStartedAt = Date.now();
+ this.activityFailures = 0;
+ }
this.startActivityTicker();
// Post the "working on it…" message with an elapsed suffix right away so
// the timer ticks from the start of the turn, not only once the first
@@ -848,13 +938,32 @@ export class RoomConnection {
/** Push the current activity line (base + elapsed) to the bridge. */
private pushActivity(): void {
const detail = this.composeActivityDetail();
- void this.postRuntimeState('working', this.currentThreadId, {}, detail).catch((error) => {
- if (this.abort.signal.aborted) return;
- this.log.warn('RoomConnection: failed to report activity', {
- roomId: this.roomId,
- error: String(error),
- });
- });
+ void this.postRuntimeState('working', this.currentThreadId, {}, detail).then(
+ () => {
+ this.activityFailures = 0;
+ },
+ (error) => {
+ if (this.abort.signal.aborted) return;
+ this.activityFailures += 1;
+ if (this.activityFailures < MAX_ACTIVITY_REPORT_FAILURES) {
+ this.log.warn('RoomConnection: failed to report activity', {
+ roomId: this.roomId,
+ failures: this.activityFailures,
+ error: String(error),
+ });
+ return;
+ }
+ this.log.error('RoomConnection: abandoning the turn — the room keeps refusing it', {
+ event: 'room_connection_turn_abandoned',
+ reason: 'activity_report_refused',
+ sessionId: this.sessionId,
+ roomId: this.roomId,
+ failures: this.activityFailures,
+ error: String(error),
+ });
+ this.clearRoomTurn();
+ }
+ );
}
private startActivityTicker(): void {
@@ -864,6 +973,25 @@ export class RoomConnection {
this.stopActivityTicker();
return;
}
+ const elapsed = Date.now() - this.workingStartedAt;
+ if (elapsed >= MAX_ROOM_TURN_MS) {
+ // Past any turn a real agent runs, so the completion signal was lost
+ // rather than late. Clear the room's "working on it" instead of
+ // leaving it to age indefinitely, and say plainly that we gave up —
+ // the session may well still be working, and the room will no longer
+ // show it.
+ this.log.error('RoomConnection: abandoning the turn — it has outrun the cap', {
+ event: 'room_connection_turn_abandoned',
+ reason: 'max_turn_duration',
+ sessionId: this.sessionId,
+ roomId: this.roomId,
+ elapsedMs: elapsed,
+ capMs: MAX_ROOM_TURN_MS,
+ });
+ this.clearRoomTurn();
+ void this.postRuntimeState('idle', null).catch(() => {});
+ return;
+ }
this.pushActivity();
}, ACTIVITY_TICK_INTERVAL_MS);
}
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts
index 82462efcd..124e0fd96 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts
@@ -284,6 +284,19 @@ class SwitchNotificationPoller {
switchRoomService.clearSession(ctx.sessionId);
}
},
+ // A refused room must not survive the process. The in-memory clear
+ // arrives a moment later on the room-changed path above; this is the
+ // durable half, and without it every restart re-declares the same dead
+ // room and is refused all over again.
+ onRoomRejected: ({ roomId }) => {
+ void switchRoomService.forgetRefusedRoom(ctx.sessionId, roomId).catch((error) => {
+ log.warn('SwitchNotificationPoller: failed to forget a refused room', {
+ sessionId: ctx.sessionId,
+ roomId,
+ error: String(error),
+ });
+ });
+ },
log,
});
this.connections.set(ctx.sessionId, connection);
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts
index c2eae4ef8..473482543 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts
@@ -16,6 +16,7 @@ vi.mock('./switch-notification-poller', () => ({
}));
const dbInsert = vi.fn();
+const dbDelete = vi.fn();
vi.mock('@main/db/client', () => {
const where = () => Promise.resolve([] as unknown[]);
const from = () => ({ where });
@@ -28,6 +29,10 @@ vi.mock('@main/db/client', () => {
dbInsert();
return { values };
},
+ delete: () => {
+ dbDelete();
+ return { where: () => Promise.resolve(undefined) };
+ },
},
};
});
@@ -46,6 +51,7 @@ describe('SwitchRoomService', () => {
switchRoomService.dispose();
emit.mockClear();
dbInsert.mockClear();
+ dbDelete.mockClear();
});
it('records a connection and emits a change', () => {
@@ -140,4 +146,22 @@ describe('SwitchRoomService', () => {
switchRoomService.setSessionRoom(ctx, 'room-a', 'agent-1', 'Room A');
await vi.waitFor(() => expect(dbInsert).toHaveBeenCalled());
});
+
+ it('drops the persisted room when the server refuses it', async () => {
+ // The persisted row is what a restart re-declares. A room id outlives the
+ // room it named, so leaving it there means the next launch opens a
+ // connection the server refuses outright — for as long as the install
+ // lives.
+ switchRoomService.setSessionRoom(ctx, 'room-gone', 'agent-1', null);
+ await switchRoomService.forgetRefusedRoom('session-1', 'room-gone');
+ expect(dbDelete).toHaveBeenCalled();
+ });
+
+ it('a session ending does not drop its persisted room', async () => {
+ // clearSession runs on every session exit, and the row is exactly what a
+ // resumed session is restored from.
+ switchRoomService.setSessionRoom(ctx, 'room-a', 'agent-1', null);
+ switchRoomService.clearSession('session-1');
+ expect(dbDelete).not.toHaveBeenCalled();
+ });
});
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts
index ebcc866a7..a32351a40 100644
--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts
+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts
@@ -186,6 +186,24 @@ class SwitchRoomService implements IDisposable {
switchNotificationPoller.connect(ctx, persisted.roomId, persisted.roomName);
}
+ /**
+ * Forget the persisted room for a session whose room the server refused.
+ *
+ * The persisted row is what a restart re-declares, and a room id outlives
+ * the room it named: keep it and the next launch opens a connection naming a
+ * room that no longer exists, which the server refuses outright. Nothing
+ * else in the restore path can tell the difference, so the refusal has to be
+ * spent here or it repeats for the life of the install.
+ */
+ async forgetRefusedRoom(sessionId: string, roomId: string): Promise<void> {
+ log.error('SwitchRoomService: forgetting the persisted room — the server refused it', {
+ event: 'switch_room_forgotten_after_refusal',
+ sessionId,
+ roomId,
+ });
+ await forgetRoomConnections([sessionId]);
+ }
+
/** Session ids that had a live room connection before the last shutdown. */
async listPersistedSessionIds(): Promise<string[]> {
return listPersistedRoomSessionIds();
diff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.ts
index 00dcdc255..2f818052f 100644
--- a/console/packages/shared/src/artifacts.ts
+++ b/console/packages/shared/src/artifacts.ts
@@ -22,8 +22,8 @@ export interface ContractRange {
export const ARTIFACT_VERSIONS = {
'switch-core': '0.22.1',
'switch-console': '0.31.3',
- 'agent-runtime': '0.3.4',
- sidecar: '1.9.6',
+ 'agent-runtime': '0.4.1',
+ sidecar: '1.9.7',
gateway: '0.22.1',
setup: '0.22.1',
'helm-chart': '0.22.1',
diff --git a/console/packages/switch-agent-runtime/package.json b/console/packages/switch-agent-runtime/package.json
index babf087e9..2742696ee 100644
--- a/console/packages/switch-agent-runtime/package.json
+++ b/console/packages/switch-agent-runtime/package.json
@@ -1,6 +1,6 @@
{
"name": "@sandboxaq/switch-agent-runtime",
- "version": "0.3.4",
+ "version": "0.4.1",
"description": "Switch agent protocol client, and the local MCP runtime built on it",
"license": "Apache-2.0",
"author": {
diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.ts
index 00dcdc255..2f818052f 100644
--- a/console/packages/switch-agent-runtime/src/artifacts.ts
+++ b/console/packages/switch-agent-runtime/src/artifacts.ts
@@ -22,8 +22,8 @@ export interface ContractRange {
export const ARTIFACT_VERSIONS = {
'switch-core': '0.22.1',
'switch-console': '0.31.3',
- 'agent-runtime': '0.3.4',
- sidecar: '1.9.6',
+ 'agent-runtime': '0.4.1',
+ sidecar: '1.9.7',
gateway: '0.22.1',
setup: '0.22.1',
'helm-chart': '0.22.1',
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
new file mode 100644
index 000000000..cec1b4902
--- /dev/null
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -0,0 +1,205 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { BEAT_INTERVAL_MS, SwitchEventStream, type SwitchEventStreamDeps } from './event-stream';
+
+/**
+ * Two ways a client can retry something that can never succeed.
+ *
+ * Both were measured on a live deployment: a stream re-declaring a room that
+ * had been deleted, refused on every open, reopening at once; and a heartbeat
+ * treating "your connection is gone" as a normal answer and beating on at full
+ * cadence. Neither could self-heal, and between them they produced most of the
+ * error volume on that deployment.
+ */
+
+const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };
+
+function silentLog() {
+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };
+}
+
+/** A stream body that stays open, so the loop neither reconnects nor spins. */
+function openForever(): ReadableStream<Uint8Array> {
+ return new ReadableStream<Uint8Array>({ start() {} });
+}
+
+function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {
+ return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));
+}
+
+function makeStream(
+ fetchMock: ReturnType<typeof vi.fn>,
+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }
+) {
+ vi.stubGlobal('fetch', fetchMock);
+ const abort = new AbortController();
+ const log = silentLog();
+ const stream = new SwitchEventStream({
+ creds,
+ connectionId: 'conn-1',
+ scope: 'single',
+ filter: 'all',
+ onEvent: () => {},
+ onGap: () => {},
+ onEvicted: () => {},
+ log,
+ signal: abort.signal,
+ ...deps,
+ });
+ stream.start();
+ return { stream, abort, log };
+}
+
+async function flush(times = 8): Promise<void> {
+ for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));
+}
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+});
+
+describe('a room the server refuses', () => {
+ /** The refusal switch-core returns when a declared room no longer exists. */
+ function roomGone(roomId: string) {
+ return {
+ ok: false,
+ status: 403,
+ body: null,
+ text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),
+ };
+ }
+
+ it('is dropped and not declared again', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events')) {
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ return String(url).includes('rooms=')
+ ? roomGone('room-dead')
+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const rejected: { roomId: string; status: number }[] = [];
+ const reported: string[][] = [];
+ const { abort, log } = makeStream(fetchMock, {
+ rooms: ['room-dead'],
+ onRooms: (rooms) => reported.push(rooms),
+ onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),
+ });
+ await flush();
+
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(2);
+ expect(opens[0]).toContain('rooms=room-dead');
+ expect(opens[1]).not.toContain('rooms=');
+ expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);
+ expect(reported).toEqual([[]]);
+ expect(log.error).toHaveBeenCalled();
+ abort.abort();
+ });
+
+ it('keeps the rooms the refusal does not name', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ const u = String(url);
+ if (!u.includes('/events'))
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ return u.includes('room-dead')
+ ? roomGone('room-dead')
+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });
+ await flush();
+
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens).toHaveLength(2);
+ expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');
+ abort.abort();
+ });
+
+ it('stays a transport error when the body names no declared room', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (!String(url).includes('/events')) {
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ return {
+ ok: false,
+ status: 403,
+ body: null,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: 'Agent is not a member of this room' }),
+ };
+ });
+ const rejected: string[] = [];
+ const { abort } = makeStream(fetchMock, {
+ rooms: ['room-live'],
+ onRoomRejected: ({ roomId }) => rejected.push(roomId),
+ });
+ await vi.advanceTimersByTimeAsync(0);
+
+ // One open, then the backoff — not an immediate retry, and nothing dropped.
+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);
+ expect(rejected).toEqual([]);
+ abort.abort();
+ });
+});
+
+describe('the heartbeat', () => {
+ /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */
+ async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events')) return new Promise(() => {});
+ return { ok: false, status, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+ await vi.advanceTimersByTimeAsync(windowMs);
+ const beats = urlsFor(fetchMock, 'connection/beat').length;
+ abort.abort();
+ return beats;
+ }
+
+ it('backs off when the connection is rejected, instead of beating at full rate', async () => {
+ const windowMs = 20 * BEAT_INTERVAL_MS;
+ // At the base cadence this window holds ~20 beats. Doubling from the base
+ // gives 4s, 8s, 16s… — four requests in the same window.
+ expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);
+ expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);
+ });
+
+ it('keeps backing off the longer the rejection lasts', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events')) return new Promise(() => {});
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(30_000);
+ const early = urlsFor(fetchMock, 'connection/beat').length;
+ await vi.advanceTimersByTimeAsync(30_000);
+ const late = urlsFor(fetchMock, 'connection/beat').length - early;
+
+ expect(late).toBeLessThan(early);
+ abort.abort();
+ });
+
+ it('returns to the base cadence once a beat lands', async () => {
+ vi.useFakeTimers();
+ let reject = true;
+ const fetchMock = vi.fn(async (url: string) => {
+ if (String(url).includes('/events')) return new Promise(() => {});
+ if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ const { abort } = makeStream(fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(30_000);
+ reject = false;
+ await vi.advanceTimersByTimeAsync(30_000);
+ const recovered = urlsFor(fetchMock, 'connection/beat').length;
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+
+ expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);
+ abort.abort();
+ });
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index 1587358cc..eb2eba980 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -86,6 +86,14 @@ export interface SwitchEventStreamDeps {
* the agent's tool calls.
*/
onRooms?: (rooms: string[]) => void;
+ /**
+ * A room we declared that the server refuses to serve, and has therefore
+ * been dropped from the declared set.
+ *
+ * Terminal for that room: the id outlives the room, so whatever remembers it
+ * has to forget, or the next connection declares the same dead room again.
+ */
+ onRoomRejected?: (info: { roomId: string; status: number; detail: string }) => void;
/** Fired when the server reports missed events it cannot replay. */
onGap(info: { fromSequence: number; reason: string }): void;
/** Fired when another stream took this connection over, or it was closed. */
@@ -149,6 +157,39 @@ export class SwitchEventStream {
this.deps.onRooms?.(rooms);
}
+ /**
+ * Give up on a declared room the server refuses, keeping the connection.
+ *
+ * The server refuses the **whole** connection when one declared room cannot
+ * be served, so re-declaring it means never connecting again: a room id
+ * outlives its room, and nothing else in the open path would ever notice.
+ * The room is dropped, the refusal is reported at error level, and the room
+ * list goes out over the same callback the server's own updates arrive on,
+ * so anything holding the room learns it is gone.
+ *
+ * Only rooms the body actually names are dropped. A refusal that names none
+ * of them says nothing about which room is at fault — it stays a transport
+ * error and keeps its backoff.
+ */
+ private dropRefusedRooms(status: number, body: string): boolean {
+ if (status !== 403 && status !== 404) return false;
+ const refused = this.rooms.filter((room) => body.includes(room));
+ if (refused.length === 0) return false;
+ const remaining = this.rooms.filter((room) => !refused.includes(room));
+ const detail = body.slice(0, 500);
+ this.deps.log.error('SwitchEventStream: the server refused a declared room — dropping it', {
+ event: 'switch_stream_room_refused',
+ status,
+ rooms: refused,
+ remaining,
+ detail,
+ });
+ this.rooms = remaining;
+ for (const roomId of refused) this.deps.onRoomRejected?.({ roomId, status, detail });
+ this.deps.onRooms?.(remaining);
+ return true;
+ }
+
private async subscribe(roomId: string): Promise<void> {
const resp = await this.post('connection/subscribe', {
connection_id: this.deps.connectionId,
@@ -223,7 +264,13 @@ export class SwitchEventStream {
});
if (!resp.ok || !resp.body) {
- throw new Error(`HTTP ${resp.status}: ${await resp.text()}`);
+ const body = await resp.text();
+ // Reopening without the refused room is a different request from the
+ // one that just failed, and the declared set strictly shrinks, so
+ // this cannot spin: retry now rather than serving the backoff a
+ // transport failure earned.
+ if (this.dropRefusedRooms(resp.status, body)) continue;
+ throw new Error(`HTTP ${resp.status}: ${body}`);
}
backoff = INITIAL_BACKOFF_MS;
@@ -315,9 +362,12 @@ export class SwitchEventStream {
*
* A 404 or 409 means we are not receiving — the connection expired, or it has
* no stream attached. Both are recovered by reopening, which resumes from the
- * cursor. Failing quietly here is the one thing that must not happen: a client
- * that has stopped receiving while believing it is connected is exactly the
- * bug this transport exists to remove.
+ * cursor, and both count as a beat that did not land: the remedy is a reopen,
+ * and a reopen that keeps being refused is a client that cannot succeed and
+ * must not be retried at full rate forever. Failing quietly here is the one
+ * thing that must not happen: a client that has stopped receiving while
+ * believing it is connected is exactly the bug this transport exists to
+ * remove.
*
* While beats succeed the cadence is fixed and short — the server declares the
* connection dead without them. While they fail it backs off, because a beat
@@ -333,19 +383,24 @@ export class SwitchEventStream {
let failures = 0;
let backoff = BEAT_INTERVAL_MS;
- const fail = (error: unknown): void => {
+ /** Count a beat that did not land and slow the loop down. Returns whether
+ * to report this one: the first, then powers of two, so a permanent
+ * failure costs a handful of lines rather than one per beat. */
+ const slowDown = (): boolean => {
failures += 1;
- // Report the first failure, then on a curve: powers of two, so a
- // permanent outage costs a handful of lines rather than one per beat.
- if ((failures & (failures - 1)) === 0) {
- log.warn('SwitchEventStream: heartbeat failed', {
- event: 'switch_beat_failed',
- endpoint: this.deps.creds.apiEndpoint,
- failures,
- error: String(error),
- });
- }
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
+ return (failures & (failures - 1)) === 0;
+ };
+
+ const fail = (error: unknown): void => {
+ if (!slowDown()) return;
+ log.warn('SwitchEventStream: heartbeat failed', {
+ event: 'switch_beat_failed',
+ endpoint: this.deps.creds.apiEndpoint,
+ failures,
+ error: String(error),
+ backoffMs: backoff,
+ });
};
while (!signal.aborted) {
@@ -355,16 +410,22 @@ export class SwitchEventStream {
cursor: this.cursor,
});
if (resp.status === 404 || resp.status === 409) {
- // Not an outage: the server answered. We are simply not attached, so
- // reopen at the normal cadence rather than backing off.
- log.warn('SwitchEventStream: heartbeat rejected — reopening', {
- event: 'switch_beat_rejected',
- status: resp.status,
- connectionId,
- });
+ // The server answered, so this is not an outage — but it is not a
+ // beat that landed either: we are not attached, and only a reopen
+ // fixes that. Reopen, and slow down all the same. A reopen that
+ // keeps being refused is a client that cannot currently succeed,
+ // and it must not spend the endpoint at full rate while it fails.
+ const report = slowDown();
this.reopen();
- failures = 0;
- backoff = BEAT_INTERVAL_MS;
+ if (report) {
+ log.warn('SwitchEventStream: heartbeat rejected — reopening', {
+ event: 'switch_beat_rejected',
+ status: resp.status,
+ connectionId,
+ failures,
+ backoffMs: backoff,
+ });
+ }
} else if (!resp.ok) {
fail(new Error(`HTTP ${resp.status}`));
} else {
diff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.py
index 38f759c28..c95e7be20 100644
--- a/core/switch_core/artifacts.py
+++ b/core/switch_core/artifacts.py
@@ -22,8 +22,8 @@ class ContractRange(NamedTuple):
ARTIFACT_VERSIONS: Final[dict[str, str]] = {
"switch-core": "0.22.1",
"switch-console": "0.31.3",
- "agent-runtime": "0.3.4",
- "sidecar": "1.9.6",
+ "agent-runtime": "0.4.1",
+ "sidecar": "1.9.7",
"gateway": "0.22.1",
"setup": "0.22.1",
"helm-chart": "0.22.1",