Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(client): stop two loops that retry an impossible request forever (#353)
Louis Amaudruz9994ec57d3fd
diff --git a/artifacts.yaml b/artifacts.yamlindex f04c95d03..b092b08c1 100644--- a/artifacts.yaml+++ b/artifacts.yaml@@ -82,7 +82,7 @@ artifacts:agent-runtime:description: The Switch protocol client and MCP runtime, published to a registry.- version: 0.3.4+ version: 0.4.1declared_in: console/packages/switch-agent-runtime/package.jsonsidecar:@@ -90,7 +90,7 @@ artifacts:The remote runtime Switch Console deploys to an agent host. Deployed ratherthan published, so nothing else declares its version — this file is itsonly source and the constant is generated from it.- version: 1.9.6+ version: 1.9.7# ── Published under the switch-core release ────────────────────────────────# No version of their own: `helm package` and the image build stamp them withdiff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.tsindex 3e5e4652a..ebcf6ff59 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.test.ts@@ -3,7 +3,7 @@ import * as os from 'node:os';import * as path from 'node:path';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import type { InjectionTarget } from './injection-sink';-import { type PromptInjector, RoomConnection } from './room-connection';+import { MAX_ROOM_TURN_MS, type PromptInjector, RoomConnection } from './room-connection';import { resolveSessionControl } from './session-control';import type { AgentBridgeEvent, AttachmentRef } from './switch-event-format';@@ -1454,3 +1454,225 @@ describe('repointing a restored session', () => {conn.stop();});});++/**+ * A turn that cannot end (CHOO-2274).+ *+ * Measured on a live deployment: a session reported `working` against a room it+ * no longer held, every five seconds, for 32 hours. The server rejected each+ * report — the room id it carried was null — and nothing here noticed, because+ * the only thing that ever closed a turn was the agent going idle, and that+ * signal was never coming. Three ways out now: the room being withdrawn, the+ * report being refused, and the clock.+ */+describe('a turn that cannot end', () => {+ function pushable() {+ const encoder = new TextEncoder();+ let controller!: ReadableStreamDefaultController<Uint8Array>;+ const stream = new ReadableStream<Uint8Array>({+ start(c) {+ controller = c;+ },+ });+ return { stream, push: (frame: string) => controller.enqueue(encoder.encode(frame)) };+ }++ /** A session already in a room, already mid-turn: exactly the shape the+ * stuck session was in. `spawnTurn` opens the turn as `start()` runs. */+ function workingSession(opts: { runtimeStateStatus?: number } = {}) {+ const { stream, push } = pushable();+ const fetchMock = vi.fn(async (url: string) => {+ const u = String(url);+ if (u.includes('/events')) {+ return { ok: true, status: 200, body: stream, text: async (): Promise<string> => '' };+ }+ if (u.includes('/runtime-state') && opts.runtimeStateStatus) {+ return {+ ok: false,+ status: opts.runtimeStateStatus,+ text: async (): Promise<string> => 'room_id must be a string',+ };+ }+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ vi.stubGlobal('fetch', fetchMock);+ const conn = new RoomConnection({+ creds,+ roomId: 'room-1',+ roomName: 'Room One',+ connectionId: 'conn-1',+ sessionId: 'session-1',+ sink: { acquire: () => ({ write: vi.fn() }) },+ injector,+ control,+ deeplinkScheme: 'switchdash',+ isHumanTyping: () => false,+ mediaDir,+ spawnTurn: { threadId: 'thread-1', anchorId: 'msg-1' },+ log: silentLog,+ });+ conn.start();+ return { conn, fetchMock, push };+ }++ function loggedAt(level: 'warn' | 'error', fragment: string): boolean {+ return silentLog[level].mock.calls.some((c) => String(c[0]).includes(fragment));+ }++ /** runtime-state posts only: the heartbeat keeps beating regardless, and it+ * is the reporting that must stop. */+ function reports(fetchMock: { mock: { calls: unknown[][] } }): number {+ return fetchMock.mock.calls.filter((c) => String(c[0]).includes('/runtime-state')).length;+ }++ beforeEach(() => {+ silentLog.debug.mockClear();+ silentLog.info.mockClear();+ silentLog.warn.mockClear();+ silentLog.error.mockClear();+ vi.useFakeTimers();+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllGlobals();+ });++ it('ends when the server withdraws the room', async () => {+ const { conn, fetchMock, push } = workingSession();+ await vi.advanceTimersByTimeAsync(0);+ expect(runtimeStates(fetchMock)).toContain('working');++ push(`event: subscription_changed\ndata: ${JSON.stringify({ rooms: [] })}\n\n`);+ await vi.advanceTimersByTimeAsync(0);++ expect(loggedAt('warn', 'the room was withdrawn')).toBe(true);+ expect(conn.room).toBeNull();++ // The ticker is what kept the 422s coming; nothing more may be reported.+ const settled = reports(fetchMock);+ await vi.advanceTimersByTimeAsync(60_000);+ expect(reports(fetchMock)).toBe(settled);+ conn.stop();+ });++ it('gives up when the room keeps refusing the report', async () => {+ const { conn, fetchMock } = workingSession({ runtimeStateStatus: 422 });+ await vi.advanceTimersByTimeAsync(0);++ // Five consecutive refusals: the opening push plus four ticks.+ await vi.advanceTimersByTimeAsync(4 * 5_000);+ expect(loggedAt('error', 'abandoning the turn')).toBe(true);++ const settled = reports(fetchMock);+ await vi.advanceTimersByTimeAsync(60_000);+ expect(reports(fetchMock)).toBe(settled);+ conn.stop();+ });++ it('gives up when the turn outruns the wall clock', async () => {+ const { conn, fetchMock } = workingSession();+ await vi.advanceTimersByTimeAsync(0);++ vi.setSystemTime(Date.now() + MAX_ROOM_TURN_MS);+ await vi.advanceTimersByTimeAsync(5_000);++ expect(loggedAt('error', 'outrun the cap')).toBe(true);+ // The room is told the turn is over rather than left showing "working".+ expect(runtimeStates(fetchMock).at(-1)).toBe('idle');++ const settled = reports(fetchMock);+ await vi.advanceTimersByTimeAsync(60_000);+ expect(reports(fetchMock)).toBe(settled);+ conn.stop();+ });++ it('keeps ticking while the reports land', async () => {+ const { conn, fetchMock } = workingSession();+ await vi.advanceTimersByTimeAsync(0);++ await vi.advanceTimersByTimeAsync(20_000);++ expect(loggedAt('error', 'abandoning the turn')).toBe(false);+ expect(runtimeStates(fetchMock).filter((s) => s === 'working').length).toBeGreaterThan(4);+ conn.stop();+ });+});++/**+ * A room the server refuses outright.+ *+ * The open is refused, not the room — so the connection never gets as far as a+ * room list, and re-declaring the same dead room means never connecting again.+ * Measured on a live deployment at roughly 29 errors a minute, for over a day,+ * with no path back.+ */+describe('a room the server refuses', () => {+ beforeEach(() => {+ silentLog.warn.mockClear();+ silentLog.error.mockClear();+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it('is surfaced and dropped, and the session is left room-less', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ const u = String(url);+ if (!u.includes('/events')) {+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ if (u.includes('rooms=')) {+ return {+ ok: false,+ status: 403,+ body: null,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: 'Room not found: room-gone' }),+ };+ }+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({ start() {} }),+ text: async (): Promise<string> => '',+ };+ });+ vi.stubGlobal('fetch', fetchMock);+ const refused: { roomId: string; status: number }[] = [];+ const rooms: (string | null)[] = [];+ const conn = new RoomConnection({+ creds,+ roomId: 'room-gone',+ roomName: null,+ connectionId: 'conn-1',+ sessionId: 'session-1',+ sink: { acquire: () => ({ write: vi.fn() }) },+ injector,+ control,+ deeplinkScheme: 'switchdash',+ isHumanTyping: () => false,+ mediaDir,+ onRoomChanged: (room) => rooms.push(room),+ onRoomRejected: ({ roomId, status }) => refused.push({ roomId, status }),+ log: silentLog,+ });+ conn.start();+ await flush(8);++ expect(refused).toEqual([{ roomId: 'room-gone', status: 403 }]);+ expect(rooms).toEqual([null]);+ expect(conn.room).toBeNull();+ expect(silentLog.error.mock.calls.some((c) => String(c[0]).includes('refused the room'))).toBe(+ true+ );++ const opens = fetchMock.mock.calls+ .map((c) => String(c[0]))+ .filter((u) => u.includes('/events'));+ expect(opens).toHaveLength(2);+ expect(opens[1]).not.toContain('rooms=');+ conn.stop();+ });+});diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.tsindex e5a8b8810..526e1a895 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/room-connection.ts@@ -44,6 +44,21 @@ const CONTROL_STEP_GAP_MS = 600;// While a turn is working, re-push the activity line this often with a refreshed// elapsed-time suffix (e.g. "· 15s") so a long-running step visibly ticks.const ACTIVITY_TICK_INTERVAL_MS = 5_000;+// Two bounds on a working turn, because a turn that never ends re-pushes its+// activity line every tick for as long as the process lives. One was observed+// running for 32 hours against a room the session no longer held, rejected+// every five seconds. Both give up loudly rather than keep a room reading+// "working on it" against a turn nobody is going to close.+//+// The failure cap catches the case where the report itself is refused — the+// server is answering, and answering that this report can never be accepted,+// so repeating it is the definition of a loop that cannot succeed.+const MAX_ACTIVITY_REPORT_FAILURES = 5;+// The wall clock catches everything else: a turn whose completion signal was+// lost (an interrupt fires no hook) leaves reports succeeding forever against+// work that finished long ago. Set well past any real turn, so reaching it is+// evidence of a bug rather than of a slow agent.+export const MAX_ROOM_TURN_MS = 4 * 60 * 60 * 1000;export type SwitchCredentials = { agentId: string; apiEndpoint: string; token: string };@@ -211,6 +226,14 @@ export interface RoomConnectionDeps {* `connect_to_room` call landed on.*/onRoomChanged?: (roomId: string | null) => void;+ /**+ * The server has refused a room this connection declared — it is gone, or+ * this agent may no longer subscribe to it. The stream has dropped it, so+ * anything that remembers the room has to forget it here: a room id we keep+ * is a room id we re-declare on the next connection, and the server refuses+ * the whole connection over it.+ */+ onRoomRejected?: (info: { roomId: string; status: number; detail: string }) => void;log: RoomConnectionLogger;}@@ -278,6 +301,8 @@ export class RoomConnection {private workingStartedAt = 0;/** Ticker that re-pushes the activity line with a refreshed elapsed suffix. */private activityTicker: ReturnType<typeof setInterval> | null = null;+ /** Consecutive rejected activity reports, reset by the first that lands. */+ private activityFailures = 0;private busyFallback: ReturnType<typeof setTimeout> | null = null;private humanGateTimer: ReturnType<typeof setTimeout> | null = null;private noTargetTimer: ReturnType<typeof setTimeout> | null = null;@@ -295,6 +320,10 @@ export class RoomConnection {private spawnTurn: SpawnTurn | null;/** Notified whenever the server tells us which room this session is in. */private readonly onRoomChanged: ((roomId: string | null) => void) | null;+ /** Notified when the server refuses a room we declared. */+ private readonly onRoomRejected:+ | ((info: { roomId: string; status: number; detail: string }) => void)+ | null;/*** The last room we passed to `onRoomChanged`. Deliberately not seeded from* the declared room: a connection opened declaring one is *told* the same@@ -331,6 +360,7 @@ export class RoomConnection {this.startCursor = deps.startCursor;this.spawnTurn = deps.spawnTurn ?? null;this.onRoomChanged = deps.onRoomChanged ?? null;+ this.onRoomRejected = deps.onRoomRejected ?? null;this.log = deps.log;}@@ -349,6 +379,7 @@ export class RoomConnection {startCursor: this.startCursor,onEvent: (event) => this.handleEvent(event),onRooms: (rooms) => this.adoptRoom(rooms),+ onRoomRejected: (info) => this.handleRoomRejected(info),onGap: (info) => this.handleGap(info),onEvicted: (reason) =>this.log.warn('RoomConnection: connection evicted', {@@ -440,11 +471,13 @@ export class RoomConnection {*/private adoptRoom(rooms: string[]): void {const next = rooms[0] ?? null;- // Against what we last *reported*, not what we hold: a session launched- // into a room declares it at open and the server confirms the same value,- // which is the only signal the rest of the app ever gets that the session- // is in that room.- if (next === this.reportedRoom) return;+ // Against what we last *reported*, not merely what we hold: a session+ // launched into a room declares it at open and the server confirms the+ // same value, which is the only signal the rest of the app ever gets that+ // the session is in that room. What we hold still has to agree, or a room+ // declared but never confirmed — then refused — would read as "no change"+ // and nobody would learn it had gone.+ if (next === this.reportedRoom && next === this.roomId) return;const previous = this.roomId;this.roomId = next;@@ -459,11 +492,65 @@ export class RoomConnection {roomId: next,});this.onRoomChanged?.(next);- if (next && !this.openSpawnTurn()) {+ if (next === null) {+ // The room has been taken and there is nowhere left to report. A turn+ // left open here is the 32-hour "working on it" bug: every field it+ // needs is still set, so the ticker keeps pushing runtime state at a+ // room this connection no longer covers, and the server rejects each+ // one. End it here — the same clearing stop() does — rather than let it+ // keep asking for something that cannot be granted.+ this.log.warn('RoomConnection: the room was withdrawn — ending the turn', {+ event: 'room_connection_room_withdrawn',+ sessionId: this.sessionId,+ previous,+ turnWasActive: this.roomTurnActive,+ });+ this.clearRoomTurn();+ return;+ }+ if (!this.openSpawnTurn()) {void this.postRuntimeState('idle', null).catch(() => {});}}+ /**+ * A declared room the server refuses. Terminal, not transient.+ *+ * The stream has already dropped it, which arrives here as an empty room+ * list and ends the turn. What this adds is the disclosure — at error level,+ * because a session pointed at a room that does not exist is broken and not+ * merely quiet — and the hand-off to whoever persisted the room, so a+ * restart does not resurrect it.+ */+ private handleRoomRejected(info: { roomId: string; status: number; detail: string }): void {+ this.log.error('RoomConnection: the server refused the room this session declared', {+ event: 'room_connection_room_refused',+ sessionId: this.sessionId,+ roomId: info.roomId,+ status: info.status,+ detail: info.detail,+ });+ this.onRoomRejected?.(info);+ }++ /**+ * Drop the current room turn without reporting into the room.+ *+ * For the cases where reporting is impossible or is itself what is failing:+ * the room is gone, or the report is being refused. Every caller says why at+ * warn or error level — a turn that ends without the room being told is a+ * degraded outcome and must not be silent.+ */+ private clearRoomTurn(): void {+ this.roomTurnActive = false;+ this.currentThreadId = null;+ this.currentAnchorId = null;+ this.runtimeState = 'idle';+ this.lastActivityDetail = null;+ this.activityFailures = 0;+ this.stopActivityTicker();+ }+/** Stop the loops and clear any lingering runtime-state surface. */stop(): void {if (this.stopped) return;@@ -801,7 +888,10 @@ export class RoomConnection {// the generic indicator, and idle/awaiting-input carry no activity.this.lastActivityDetail = null;if (state === 'working') {- if (!wasWorking) this.workingStartedAt = Date.now();+ if (!wasWorking) {+ this.workingStartedAt = Date.now();+ this.activityFailures = 0;+ }this.startActivityTicker();// Post the "working on it…" message with an elapsed suffix right away so// the timer ticks from the start of the turn, not only once the first@@ -848,13 +938,32 @@ export class RoomConnection {/** Push the current activity line (base + elapsed) to the bridge. */private pushActivity(): void {const detail = this.composeActivityDetail();- void this.postRuntimeState('working', this.currentThreadId, {}, detail).catch((error) => {- if (this.abort.signal.aborted) return;- this.log.warn('RoomConnection: failed to report activity', {- roomId: this.roomId,- error: String(error),- });- });+ void this.postRuntimeState('working', this.currentThreadId, {}, detail).then(+ () => {+ this.activityFailures = 0;+ },+ (error) => {+ if (this.abort.signal.aborted) return;+ this.activityFailures += 1;+ if (this.activityFailures < MAX_ACTIVITY_REPORT_FAILURES) {+ this.log.warn('RoomConnection: failed to report activity', {+ roomId: this.roomId,+ failures: this.activityFailures,+ error: String(error),+ });+ return;+ }+ this.log.error('RoomConnection: abandoning the turn — the room keeps refusing it', {+ event: 'room_connection_turn_abandoned',+ reason: 'activity_report_refused',+ sessionId: this.sessionId,+ roomId: this.roomId,+ failures: this.activityFailures,+ error: String(error),+ });+ this.clearRoomTurn();+ }+ );}private startActivityTicker(): void {@@ -864,6 +973,25 @@ export class RoomConnection {this.stopActivityTicker();return;}+ const elapsed = Date.now() - this.workingStartedAt;+ if (elapsed >= MAX_ROOM_TURN_MS) {+ // Past any turn a real agent runs, so the completion signal was lost+ // rather than late. Clear the room's "working on it" instead of+ // leaving it to age indefinitely, and say plainly that we gave up —+ // the session may well still be working, and the room will no longer+ // show it.+ this.log.error('RoomConnection: abandoning the turn — it has outrun the cap', {+ event: 'room_connection_turn_abandoned',+ reason: 'max_turn_duration',+ sessionId: this.sessionId,+ roomId: this.roomId,+ elapsedMs: elapsed,+ capMs: MAX_ROOM_TURN_MS,+ });+ this.clearRoomTurn();+ void this.postRuntimeState('idle', null).catch(() => {});+ return;+ }this.pushActivity();}, ACTIVITY_TICK_INTERVAL_MS);}diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.tsindex 82462efcd..124e0fd96 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-notification-poller.ts@@ -284,6 +284,19 @@ class SwitchNotificationPoller {switchRoomService.clearSession(ctx.sessionId);}},+ // A refused room must not survive the process. The in-memory clear+ // arrives a moment later on the room-changed path above; this is the+ // durable half, and without it every restart re-declares the same dead+ // room and is refused all over again.+ onRoomRejected: ({ roomId }) => {+ void switchRoomService.forgetRefusedRoom(ctx.sessionId, roomId).catch((error) => {+ log.warn('SwitchNotificationPoller: failed to forget a refused room', {+ sessionId: ctx.sessionId,+ roomId,+ error: String(error),+ });+ });+ },log,});this.connections.set(ctx.sessionId, connection);diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.tsindex c2eae4ef8..473482543 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.test.ts@@ -16,6 +16,7 @@ vi.mock('./switch-notification-poller', () => ({}));const dbInsert = vi.fn();+const dbDelete = vi.fn();vi.mock('@main/db/client', () => {const where = () => Promise.resolve([] as unknown[]);const from = () => ({ where });@@ -28,6 +29,10 @@ vi.mock('@main/db/client', () => {dbInsert();return { values };},+ delete: () => {+ dbDelete();+ return { where: () => Promise.resolve(undefined) };+ },},};});@@ -46,6 +51,7 @@ describe('SwitchRoomService', () => {switchRoomService.dispose();emit.mockClear();dbInsert.mockClear();+ dbDelete.mockClear();});it('records a connection and emits a change', () => {@@ -140,4 +146,22 @@ describe('SwitchRoomService', () => {switchRoomService.setSessionRoom(ctx, 'room-a', 'agent-1', 'Room A');await vi.waitFor(() => expect(dbInsert).toHaveBeenCalled());});++ it('drops the persisted room when the server refuses it', async () => {+ // The persisted row is what a restart re-declares. A room id outlives the+ // room it named, so leaving it there means the next launch opens a+ // connection the server refuses outright — for as long as the install+ // lives.+ switchRoomService.setSessionRoom(ctx, 'room-gone', 'agent-1', null);+ await switchRoomService.forgetRefusedRoom('session-1', 'room-gone');+ expect(dbDelete).toHaveBeenCalled();+ });++ it('a session ending does not drop its persisted room', async () => {+ // clearSession runs on every session exit, and the row is exactly what a+ // resumed session is restored from.+ switchRoomService.setSessionRoom(ctx, 'room-a', 'agent-1', null);+ switchRoomService.clearSession('session-1');+ expect(dbDelete).not.toHaveBeenCalled();+ });});diff --git a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.tsindex ebcc866a7..a32351a40 100644--- a/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts+++ b/console/apps/switch-console-desktop/src/main/core/switch-rooms/switch-room-service.ts@@ -186,6 +186,24 @@ class SwitchRoomService implements IDisposable {switchNotificationPoller.connect(ctx, persisted.roomId, persisted.roomName);}+ /**+ * Forget the persisted room for a session whose room the server refused.+ *+ * The persisted row is what a restart re-declares, and a room id outlives+ * the room it named: keep it and the next launch opens a connection naming a+ * room that no longer exists, which the server refuses outright. Nothing+ * else in the restore path can tell the difference, so the refusal has to be+ * spent here or it repeats for the life of the install.+ */+ async forgetRefusedRoom(sessionId: string, roomId: string): Promise<void> {+ log.error('SwitchRoomService: forgetting the persisted room — the server refused it', {+ event: 'switch_room_forgotten_after_refusal',+ sessionId,+ roomId,+ });+ await forgetRoomConnections([sessionId]);+ }+/** Session ids that had a live room connection before the last shutdown. */async listPersistedSessionIds(): Promise<string[]> {return listPersistedRoomSessionIds();diff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.tsindex 00dcdc255..2f818052f 100644--- a/console/packages/shared/src/artifacts.ts+++ b/console/packages/shared/src/artifacts.ts@@ -22,8 +22,8 @@ export interface ContractRange {export const ARTIFACT_VERSIONS = {'switch-core': '0.22.1','switch-console': '0.31.3',- 'agent-runtime': '0.3.4',- sidecar: '1.9.6',+ 'agent-runtime': '0.4.1',+ sidecar: '1.9.7',gateway: '0.22.1',setup: '0.22.1','helm-chart': '0.22.1',diff --git a/console/packages/switch-agent-runtime/package.json b/console/packages/switch-agent-runtime/package.jsonindex babf087e9..2742696ee 100644--- a/console/packages/switch-agent-runtime/package.json+++ b/console/packages/switch-agent-runtime/package.json@@ -1,6 +1,6 @@{"name": "@sandboxaq/switch-agent-runtime",- "version": "0.3.4",+ "version": "0.4.1","description": "Switch agent protocol client, and the local MCP runtime built on it","license": "Apache-2.0","author": {diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.tsindex 00dcdc255..2f818052f 100644--- a/console/packages/switch-agent-runtime/src/artifacts.ts+++ b/console/packages/switch-agent-runtime/src/artifacts.ts@@ -22,8 +22,8 @@ export interface ContractRange {export const ARTIFACT_VERSIONS = {'switch-core': '0.22.1','switch-console': '0.31.3',- 'agent-runtime': '0.3.4',- sidecar: '1.9.6',+ 'agent-runtime': '0.4.1',+ sidecar: '1.9.7',gateway: '0.22.1',setup: '0.22.1','helm-chart': '0.22.1',diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.tsnew file mode 100644index 000000000..cec1b4902--- /dev/null+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts@@ -0,0 +1,205 @@+import { afterEach, describe, expect, it, vi } from 'vitest';+import { BEAT_INTERVAL_MS, SwitchEventStream, type SwitchEventStreamDeps } from './event-stream';++/**+ * Two ways a client can retry something that can never succeed.+ *+ * Both were measured on a live deployment: a stream re-declaring a room that+ * had been deleted, refused on every open, reopening at once; and a heartbeat+ * treating "your connection is gone" as a normal answer and beating on at full+ * cadence. Neither could self-heal, and between them they produced most of the+ * error volume on that deployment.+ */++const creds = { agentId: 'agent-1', apiEndpoint: 'https://switch.test', token: 'tok' };++function silentLog() {+ return { debug: vi.fn(), warn: vi.fn(), error: vi.fn() };+}++/** A stream body that stays open, so the loop neither reconnects nor spins. */+function openForever(): ReadableStream<Uint8Array> {+ return new ReadableStream<Uint8Array>({ start() {} });+}++function urlsFor(fetchMock: { mock: { calls: unknown[][] } }, fragment: string): string[] {+ return fetchMock.mock.calls.map((c) => String(c[0])).filter((u) => u.includes(fragment));+}++function makeStream(+ fetchMock: ReturnType<typeof vi.fn>,+ deps: Partial<SwitchEventStreamDeps> & { rooms: string[] }+) {+ vi.stubGlobal('fetch', fetchMock);+ const abort = new AbortController();+ const log = silentLog();+ const stream = new SwitchEventStream({+ creds,+ connectionId: 'conn-1',+ scope: 'single',+ filter: 'all',+ onEvent: () => {},+ onGap: () => {},+ onEvicted: () => {},+ log,+ signal: abort.signal,+ ...deps,+ });+ stream.start();+ return { stream, abort, log };+}++async function flush(times = 8): Promise<void> {+ for (let i = 0; i < times; i += 1) await new Promise((r) => setTimeout(r, 0));+}++afterEach(() => {+ vi.unstubAllGlobals();+ vi.useRealTimers();+});++describe('a room the server refuses', () => {+ /** The refusal switch-core returns when a declared room no longer exists. */+ function roomGone(roomId: string) {+ return {+ ok: false,+ status: 403,+ body: null,+ text: async (): Promise<string> => JSON.stringify({ detail: `Room not found: ${roomId}` }),+ };+ }++ it('is dropped and not declared again', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events')) {+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ return String(url).includes('rooms=')+ ? roomGone('room-dead')+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const rejected: { roomId: string; status: number }[] = [];+ const reported: string[][] = [];+ const { abort, log } = makeStream(fetchMock, {+ rooms: ['room-dead'],+ onRooms: (rooms) => reported.push(rooms),+ onRoomRejected: ({ roomId, status }) => rejected.push({ roomId, status }),+ });+ await flush();++ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(2);+ expect(opens[0]).toContain('rooms=room-dead');+ expect(opens[1]).not.toContain('rooms=');+ expect(rejected).toEqual([{ roomId: 'room-dead', status: 403 }]);+ expect(reported).toEqual([[]]);+ expect(log.error).toHaveBeenCalled();+ abort.abort();+ });++ it('keeps the rooms the refusal does not name', async () => {+ const fetchMock = vi.fn(async (url: string) => {+ const u = String(url);+ if (!u.includes('/events'))+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ return u.includes('room-dead')+ ? roomGone('room-dead')+ : { ok: true, status: 200, body: openForever(), text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: ['room-dead', 'room-live'] });+ await flush();++ const opens = urlsFor(fetchMock, '/events');+ expect(opens).toHaveLength(2);+ expect(decodeURIComponent(opens[1])).toContain('rooms=room-live');+ abort.abort();+ });++ it('stays a transport error when the body names no declared room', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (!String(url).includes('/events')) {+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ return {+ ok: false,+ status: 403,+ body: null,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: 'Agent is not a member of this room' }),+ };+ });+ const rejected: string[] = [];+ const { abort } = makeStream(fetchMock, {+ rooms: ['room-live'],+ onRoomRejected: ({ roomId }) => rejected.push(roomId),+ });+ await vi.advanceTimersByTimeAsync(0);++ // One open, then the backoff — not an immediate retry, and nothing dropped.+ expect(urlsFor(fetchMock, '/events')).toHaveLength(1);+ expect(rejected).toEqual([]);+ abort.abort();+ });+});++describe('the heartbeat', () => {+ /** Beat requests made in `windowMs` of (fake) time, all of them rejected. */+ async function beatsWhileRejected(status: number, windowMs: number): Promise<number> {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events')) return new Promise(() => {});+ return { ok: false, status, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });+ await vi.advanceTimersByTimeAsync(windowMs);+ const beats = urlsFor(fetchMock, 'connection/beat').length;+ abort.abort();+ return beats;+ }++ it('backs off when the connection is rejected, instead of beating at full rate', async () => {+ const windowMs = 20 * BEAT_INTERVAL_MS;+ // At the base cadence this window holds ~20 beats. Doubling from the base+ // gives 4s, 8s, 16s… — four requests in the same window.+ expect(await beatsWhileRejected(404, windowMs)).toBeLessThanOrEqual(5);+ expect(await beatsWhileRejected(409, windowMs)).toBeLessThanOrEqual(5);+ });++ it('keeps backing off the longer the rejection lasts', async () => {+ vi.useFakeTimers();+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events')) return new Promise(() => {});+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(30_000);+ const early = urlsFor(fetchMock, 'connection/beat').length;+ await vi.advanceTimersByTimeAsync(30_000);+ const late = urlsFor(fetchMock, 'connection/beat').length - early;++ expect(late).toBeLessThan(early);+ abort.abort();+ });++ it('returns to the base cadence once a beat lands', async () => {+ vi.useFakeTimers();+ let reject = true;+ const fetchMock = vi.fn(async (url: string) => {+ if (String(url).includes('/events')) return new Promise(() => {});+ if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ const { abort } = makeStream(fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(30_000);+ reject = false;+ await vi.advanceTimersByTimeAsync(30_000);+ const recovered = urlsFor(fetchMock, 'connection/beat').length;+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);++ expect(urlsFor(fetchMock, 'connection/beat').length - recovered).toBeGreaterThanOrEqual(9);+ abort.abort();+ });+});diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.tsindex 1587358cc..eb2eba980 100644--- a/console/packages/switch-agent-runtime/src/event-stream.ts+++ b/console/packages/switch-agent-runtime/src/event-stream.ts@@ -86,6 +86,14 @@ export interface SwitchEventStreamDeps {* the agent's tool calls.*/onRooms?: (rooms: string[]) => void;+ /**+ * A room we declared that the server refuses to serve, and has therefore+ * been dropped from the declared set.+ *+ * Terminal for that room: the id outlives the room, so whatever remembers it+ * has to forget, or the next connection declares the same dead room again.+ */+ onRoomRejected?: (info: { roomId: string; status: number; detail: string }) => void;/** Fired when the server reports missed events it cannot replay. */onGap(info: { fromSequence: number; reason: string }): void;/** Fired when another stream took this connection over, or it was closed. */@@ -149,6 +157,39 @@ export class SwitchEventStream {this.deps.onRooms?.(rooms);}+ /**+ * Give up on a declared room the server refuses, keeping the connection.+ *+ * The server refuses the **whole** connection when one declared room cannot+ * be served, so re-declaring it means never connecting again: a room id+ * outlives its room, and nothing else in the open path would ever notice.+ * The room is dropped, the refusal is reported at error level, and the room+ * list goes out over the same callback the server's own updates arrive on,+ * so anything holding the room learns it is gone.+ *+ * Only rooms the body actually names are dropped. A refusal that names none+ * of them says nothing about which room is at fault — it stays a transport+ * error and keeps its backoff.+ */+ private dropRefusedRooms(status: number, body: string): boolean {+ if (status !== 403 && status !== 404) return false;+ const refused = this.rooms.filter((room) => body.includes(room));+ if (refused.length === 0) return false;+ const remaining = this.rooms.filter((room) => !refused.includes(room));+ const detail = body.slice(0, 500);+ this.deps.log.error('SwitchEventStream: the server refused a declared room — dropping it', {+ event: 'switch_stream_room_refused',+ status,+ rooms: refused,+ remaining,+ detail,+ });+ this.rooms = remaining;+ for (const roomId of refused) this.deps.onRoomRejected?.({ roomId, status, detail });+ this.deps.onRooms?.(remaining);+ return true;+ }+private async subscribe(roomId: string): Promise<void> {const resp = await this.post('connection/subscribe', {connection_id: this.deps.connectionId,@@ -223,7 +264,13 @@ export class SwitchEventStream {});if (!resp.ok || !resp.body) {- throw new Error(`HTTP ${resp.status}: ${await resp.text()}`);+ const body = await resp.text();+ // Reopening without the refused room is a different request from the+ // one that just failed, and the declared set strictly shrinks, so+ // this cannot spin: retry now rather than serving the backoff a+ // transport failure earned.+ if (this.dropRefusedRooms(resp.status, body)) continue;+ throw new Error(`HTTP ${resp.status}: ${body}`);}backoff = INITIAL_BACKOFF_MS;@@ -315,9 +362,12 @@ export class SwitchEventStream {** A 404 or 409 means we are not receiving — the connection expired, or it has* no stream attached. Both are recovered by reopening, which resumes from the- * cursor. Failing quietly here is the one thing that must not happen: a client- * that has stopped receiving while believing it is connected is exactly the- * bug this transport exists to remove.+ * cursor, and both count as a beat that did not land: the remedy is a reopen,+ * and a reopen that keeps being refused is a client that cannot succeed and+ * must not be retried at full rate forever. Failing quietly here is the one+ * thing that must not happen: a client that has stopped receiving while+ * believing it is connected is exactly the bug this transport exists to+ * remove.** While beats succeed the cadence is fixed and short — the server declares the* connection dead without them. While they fail it backs off, because a beat@@ -333,19 +383,24 @@ export class SwitchEventStream {let failures = 0;let backoff = BEAT_INTERVAL_MS;- const fail = (error: unknown): void => {+ /** Count a beat that did not land and slow the loop down. Returns whether+ * to report this one: the first, then powers of two, so a permanent+ * failure costs a handful of lines rather than one per beat. */+ const slowDown = (): boolean => {failures += 1;- // Report the first failure, then on a curve: powers of two, so a- // permanent outage costs a handful of lines rather than one per beat.- if ((failures & (failures - 1)) === 0) {- log.warn('SwitchEventStream: heartbeat failed', {- event: 'switch_beat_failed',- endpoint: this.deps.creds.apiEndpoint,- failures,- error: String(error),- });- }backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);+ return (failures & (failures - 1)) === 0;+ };++ const fail = (error: unknown): void => {+ if (!slowDown()) return;+ log.warn('SwitchEventStream: heartbeat failed', {+ event: 'switch_beat_failed',+ endpoint: this.deps.creds.apiEndpoint,+ failures,+ error: String(error),+ backoffMs: backoff,+ });};while (!signal.aborted) {@@ -355,16 +410,22 @@ export class SwitchEventStream {cursor: this.cursor,});if (resp.status === 404 || resp.status === 409) {- // Not an outage: the server answered. We are simply not attached, so- // reopen at the normal cadence rather than backing off.- log.warn('SwitchEventStream: heartbeat rejected — reopening', {- event: 'switch_beat_rejected',- status: resp.status,- connectionId,- });+ // The server answered, so this is not an outage — but it is not a+ // beat that landed either: we are not attached, and only a reopen+ // fixes that. Reopen, and slow down all the same. A reopen that+ // keeps being refused is a client that cannot currently succeed,+ // and it must not spend the endpoint at full rate while it fails.+ const report = slowDown();this.reopen();- failures = 0;- backoff = BEAT_INTERVAL_MS;+ if (report) {+ log.warn('SwitchEventStream: heartbeat rejected — reopening', {+ event: 'switch_beat_rejected',+ status: resp.status,+ connectionId,+ failures,+ backoffMs: backoff,+ });+ }} else if (!resp.ok) {fail(new Error(`HTTP ${resp.status}`));} else {diff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.pyindex 38f759c28..c95e7be20 100644--- a/core/switch_core/artifacts.py+++ b/core/switch_core/artifacts.py@@ -22,8 +22,8 @@ class ContractRange(NamedTuple):ARTIFACT_VERSIONS: Final[dict[str, str]] = {"switch-core": "0.22.1","switch-console": "0.31.3",- "agent-runtime": "0.3.4",- "sidecar": "1.9.6",+ "agent-runtime": "0.4.1",+ "sidecar": "1.9.7","gateway": "0.22.1","setup": "0.22.1","helm-chart": "0.22.1",