Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(runtime): heartbeat lockout after back-off, and self-takeover on a cancelled open (#607)
Louis Amaudruz497dc25974f3
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.tsindex fbc6ea524..a79c3cbf3 100644--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts@@ -680,6 +680,286 @@ describe('the heartbeat', () => {});});+describe('the heartbeat after a reconnect', () => {+ /**+ * A server that reattaches on every open, can drop the current stream, and+ * records when each stream was opened and each beat arrived. The server's+ * own rule is what these tests hold the client to: a new connection that+ * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.+ */+ const HEARTBEAT_TTL_MS = 6000;++ /** Either end may close a stream first; the second close is not an error here. */+ function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {+ try {+ controller.close();+ } catch {+ // Already closed by the other side.+ }+ }++ function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {+ const opens: number[] = [];+ const beats: number[] = [];+ let generation = 0;+ let current: ReadableStreamDefaultController<Uint8Array> | null = null;+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (String(url).includes('/events')) {+ opens.push(Date.now());+ generation += 1;+ const gen = generation;+ return {+ ok: true,+ status: 200,+ body: new ReadableStream<Uint8Array>({+ start(controller) {+ current = controller;+ controller.enqueue(connected(gen));+ init.signal.addEventListener('abort', () => closeQuietly(controller), {+ once: true,+ });+ },+ }),+ text: async (): Promise<string> => '',+ };+ }+ beats.push(Date.now());+ const reply = beatReply();+ if (reply === 'fail') throw new TypeError('fetch failed');+ if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ });+ /** The server ends the current stream, as it does a lapsed connection. */+ const drop = (): void => {+ if (current) closeQuietly(current);+ };+ return { fetchMock, opens, beats, drop };+ }++ it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {+ // The lockout: beats fail for a while, so the gap between them climbs to+ // its 30-second cap. The stream then reopens. The server starts its+ // 6-second clock on the new connection — and the client used to sit out+ // the rest of the 30 seconds before saying anything, lose the connection+ // for silence, and repeat that on every connection it opened afterwards.+ vi.useFakeTimers();+ let failing = true;+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ // Long enough of failed beats for the gap to reach its cap.+ await vi.advanceTimersByTimeAsync(70_000);+ failing = false;+ server.drop();+ await vi.advanceTimersByTimeAsync(40_000);++ const reopenedAt = server.opens[1];+ expect(reopenedAt).toBeDefined();+ const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);+ expect(firstBeatAfter).toBeDefined();+ expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);+ abort.abort();+ });++ it('keeps beating at the base cadence on the new connection once it lands', async () => {+ vi.useFakeTimers();+ let failing = true;+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(70_000);+ failing = false;+ server.drop();+ await vi.advanceTimersByTimeAsync(5_000);+ const settled = server.beats.length;+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);++ expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);+ abort.abort();+ });++ it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {+ // The brake the old ordering provided, which beating promptly on every+ // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The+ // wait now sits before each reopen and doubles, so a refusing server sees+ // a handful of reopens, not one every couple of seconds per agent.+ vi.useFakeTimers();+ const server = reattachingServer(() => 404);+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(60_000);++ // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.+ expect(server.opens.length).toBeLessThanOrEqual(6);+ abort.abort();+ });++ it('does not throw away a connection the stream reopened while it waited', async () => {+ // A lapsed connection's socket is closed by the server, and the stream+ // loop reopens it by itself. If that lands during the heartbeat's wait+ // after a refusal, reopening again would discard a healthy connection.+ vi.useFakeTimers();+ let refuse = true;+ const server = reattachingServer(() => (refuse ? 404 : 'ok'));+ const { abort } = makeStream(server.fetchMock, { rooms: [] });++ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);+ expect(server.beats.length).toBe(1);+ refuse = false;+ // The server ends the stream now, inside the heartbeat's wait.+ server.drop();+ await vi.advanceTimersByTimeAsync(20_000);++ // One open for the start, one for the stream's own reopen — not a third+ // from the heartbeat discarding it.+ expect(server.opens.length).toBe(2);+ abort.abort();+ });+});++describe('reopening while an open is still in flight', () => {+ /**+ * A server that keeps the incarnation the way the real one does: every open+ * it receives makes a new one, whether or not the client stays to read the+ * answer, and a reattach claiming an older one is refused as a takeover.+ * The next open can be held unanswered, the way a slow connection holds it.+ */+ function incarnationServer() {+ const state = {+ generation: 0,+ opens: 0,+ beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',+ subscribeDrops: false,+ holdNext: false,+ release: (): void => {},+ drop: (): void => {},+ };+ const body = (gen: number, signal: AbortSignal) =>+ new ReadableStream<Uint8Array>({+ start(controller) {+ const close = (): void => {+ try {+ controller.close();+ } catch {+ // already closed+ }+ };+ state.drop = close;+ controller.enqueue(connected(gen));+ signal.addEventListener('abort', close, { once: true });+ },+ });+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {+ if (String(url).includes('connection/subscribe')) {+ if (!state.subscribeDrops)+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ state.subscribeDrops = false;+ // The stream ends while the claim is still in flight.+ state.holdNext = true;+ state.drop();+ await new Promise((r) => setTimeout(r, 3_000));+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ if (!String(url).includes('/events')) {+ if (state.beat === 'slow-404-and-drop') {+ // The stream ends while this beat is in flight, and its answer comes+ // back just after the client has started reopening.+ state.beat = 'ok';+ state.holdNext = true;+ state.drop();+ await new Promise((r) => setTimeout(r, 1_200));+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ }+ if (state.beat === 404)+ return { ok: false, status: 404, text: async (): Promise<string> => '' };+ return { ok: true, status: 200, text: async (): Promise<string> => '' };+ }+ state.opens += 1;+ const claimed = new URL(String(url)).searchParams.get('expected_generation');+ if (claimed !== null && Number(claimed) < state.generation)+ return {+ ok: false,+ status: 409,+ body: null,+ text: async (): Promise<string> =>+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),+ };+ // The server acts on the open the moment it arrives.+ state.generation += 1;+ const gen = state.generation;+ const answer = {+ ok: true,+ status: 200,+ body: body(gen, init.signal),+ text: async (): Promise<string> => '',+ };+ if (!state.holdNext) return answer;+ state.holdNext = false;+ return new Promise((resolve, reject) => {+ state.release = () => resolve(answer);+ init.signal.addEventListener(+ 'abort',+ () => reject(new DOMException('aborted', 'AbortError')),+ { once: true }+ );+ });+ });+ return { state, fetchMock };+ }++ it('does not cancel an open the server has already acted on, and so does not stand down', async () => {+ // The self-takeover seen on the pilot. The server closes the stream, the+ // client starts reopening, and a beat sent just before comes back 404.+ // Reopening in answer used to cancel the open already in flight — which+ // the server had acted on, moving the connection to a new incarnation.+ // The client never read that answer, reattached claiming the old one, was+ // refused as a takeover, and stood down for good. The "other client" was+ // its own cancelled request.+ vi.useFakeTimers();+ const { state, fetchMock } = incarnationServer();+ const onEvicted = vi.fn();+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });++ await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);+ expect(state.generation).toBe(1);++ state.beat = 'slow-404-and-drop';+ // Long enough for the refused beat's wait to end while the open is held.+ await vi.advanceTimersByTimeAsync(12_000);+ state.release();+ await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);++ expect(onEvicted).not.toHaveBeenCalled();+ expect(state.opens).toBe(2);+ expect(beatGenerations(fetchMock).at(-1)).toBe(2);+ abort.abort();+ });++ it('carries out a repoint asked for mid-open once that open is answered', async () => {+ // The other way into the same hole: a room change reopens to include the+ // new room. Mid-open, it must wait for the answer rather than cancel it —+ // and must still happen, since the open in flight was built without it.+ vi.useFakeTimers();+ const { state, fetchMock } = incarnationServer();+ const onEvicted = vi.fn();+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);++ state.subscribeDrops = true;+ const repointing = stream.repoint('room-b');+ // The claim returns while the stream's reopen is held unanswered.+ await vi.advanceTimersByTimeAsync(5_000);+ state.release();+ await vi.advanceTimersByTimeAsync(5_000);+ await repointing;++ expect(onEvicted).not.toHaveBeenCalled();+ const opens = urlsFor(fetchMock, '/events');+ expect(opens.at(-1)).toContain('rooms=room-b');+ abort.abort();+ });+});+describe('a connection another client takes over', () => {it('stands down when the room it repoints to is refused, without reopening', async () => {// `repoint` claims the room *before* it reopens, so the open's own fencediff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.tsindex d25e01cfc..f71dd7eeb 100644--- a/console/packages/switch-agent-runtime/src/event-stream.ts+++ b/console/packages/switch-agent-runtime/src/event-stream.ts@@ -336,6 +336,23 @@ export class SwitchEventStream {* incarnation we have just replaced.*/private readonly fence = new ReattachFence();+ /**+ * How many times the server has named an incarnation for this client, and+ * how to cut the heartbeat's current wait short when it does.+ *+ * A fresh attach starts the server's 6-second clock on a connection that has+ * never beaten, so the heartbeat must go out promptly — whatever back-off an+ * earlier, failing connection had built up. See `beatLoop`.+ */+ private attaches = 0;+ private wakeBeat: () => void = () => {};+ /**+ * Reopens asked for, and how many of those the open now in progress was+ * built after — so a reopen asked for mid-open is carried out once that open+ * is answered, never by cancelling it. See `reopen`.+ */+ private reopensWanted = 0;+ private openCarries = 0;constructor(deps: SwitchEventStreamDeps) {this.deps = deps;@@ -404,8 +421,35 @@ export class SwitchEventStream {this.reopen();}+ /**+ * Replace the socket — but never by cancelling an open the server has not+ * answered yet.+ *+ * An open is a request the server acts on as soon as it arrives: it makes a+ * new incarnation of the connection there and then. Cancelling it+ * client-side does not undo that, it only stops us reading the answer that+ * names the new incarnation. The next open then claims the incarnation+ * before it, which the server has moved past, and is refused as a takeover+ * — terminal. The client stands down, and the "other client" it yielded to+ * was its own cancelled request.+ *+ * So a reopen asked for while an open is in flight is remembered instead,+ * and carried out when that open's `connection_state` arrives, by which time+ * we know the incarnation to claim. An open that was already built after the+ * request carries it and needs no second one.+ */private reopen(): void {- this.socketAbort?.abort();+ this.reopensWanted += 1;+ if (this.fence.admitting) this.socketAbort?.abort();+ }++ /**+ * Reopen only to get attached again — the heartbeat's reason. An open+ * already in flight does exactly that, so there is nothing to add to it; and+ * one scheduled after a dropped socket carries the request anyway.+ */+ private reattach(): void {+ if (this.fence.admitting) this.reopen();}/**@@ -663,6 +707,7 @@ export class SwitchEventStream {client_version: RUNTIME_VERSION,});this.declaredSpawnCapable = this.spawnCapable;+ this.openCarries = this.reopensWanted;if (this.declaredSpawnCapable) params.set('spawn_capable', 'true');if (this.rooms.length) params.set('rooms', this.rooms.join(','));// Reattaching, so say which incarnation we believe we still are and@@ -771,7 +816,17 @@ export class SwitchEventStream {server: frame.data.server ?? null,});this.reportRooms(frame.data.rooms);+ // A reopen asked for while this open was in flight, for something this+ // open was built too early to carry. Now that we know the incarnation+ // to claim, it is safe to replace the socket — and done before the+ // gate opens, so no beat goes out on a socket about to be dropped.+ if (this.reopensWanted > this.openCarries) {+ this.socketAbort?.abort();+ return;+ }this.fence.attached();+ this.attaches += 1;+ this.wakeBeat();this.redeclare();this.deps.onConnected?.();return;@@ -905,6 +960,35 @@ export class SwitchEventStream {const { log, signal, connectionId } = this.deps;let failures = 0;let backoff = BEAT_INTERVAL_MS;+ // Rejections in a row, which pace reopening rather than beating. Kept+ // apart from `backoff` because a fresh attach resets that and must not+ // reset this: see the 404/409 branch below.+ let rejections = 0;+ let seenAttach = this.attaches;++ /**+ * Wait `ms`, but stop waiting the moment the server names a new+ * incarnation — and do not start waiting if it already has.+ *+ * The back-off used to be served in full across a reattach. A beat that+ * had been failing left the loop sleeping for up to 30 seconds; the stream+ * reopened, the server started its 6-second clock on the new connection,+ * and the next beat arrived long after the server had closed it for+ * silence. That beat was refused, which counted as another failure, which+ * kept the wait at 30 seconds. The client could not get out of it on its+ * own: one hiccup, and it lapsed every connection it ever opened again.+ */+ const pause = async (ms: number): Promise<void> => {+ if (this.attaches !== seenAttach) return;+ await new Promise<void>((resolve) => {+ const timer = setTimeout(resolve, ms);+ this.wakeBeat = () => {+ clearTimeout(timer);+ resolve();+ };+ });+ this.wakeBeat = () => {};+ };/** Count a beat that did not land and slow the loop down. Returns whether* to report this one: the first, then powers of two, so a permanent@@ -958,6 +1042,13 @@ export class SwitchEventStream {// beat could be sent under an incarnation later than the one recorded.await Promise.race([this.fence.reached, until(signal, this.halt.signal)]);if (signal.aborted || this.halt.signal.aborted) return;+ // A connection the server has just attached has never beaten, and has+ // 6 seconds to. Whatever an earlier connection's failures built up says+ // nothing about this one, so it starts at the base cadence.+ if (this.attaches !== seenAttach) {+ seenAttach = this.attaches;+ backoff = BEAT_INTERVAL_MS;+ }const tick = await this.fence.tick(beat);if (signal.aborted || this.halt.signal.aborted) return;@@ -989,20 +1080,36 @@ export class SwitchEventStream {if (status === 404 || status === 409) {// The server answered, so this is not an outage — but it is not a// beat that landed either: we are not attached, and only a reopen- // fixes that. Reopen, and slow down all the same. A reopen that- // keeps being refused is a client that cannot currently succeed,- // and it must not spend the endpoint at full rate while it fails.- const report = slowDown();- this.reopen();- if (report) {+ // fixes that. A reopen that keeps being refused is a client that+ // cannot currently succeed, and it must not spend the endpoint at+ // full rate while it fails.+ //+ // So the wait goes *before* the reopen, and grows with each refusal+ // in a row. It used to go after — reopen at once, then sleep the+ // back-off before beating — which starved the very connection the+ // reopen had just made. Pacing the reopen instead keeps the same+ // brake on a refusing server while letting each new connection beat+ // as soon as it is attached.+ failures += 1;+ rejections += 1;+ const waitMs = Math.min(BEAT_INTERVAL_MS * 2 ** rejections, MAX_BACKOFF_MS);+ if ((failures & (failures - 1)) === 0) {log.warn('SwitchEventStream: heartbeat rejected — reopening', {event: 'switch_beat_rejected',status,connectionId,failures,- backoffMs: backoff,+ backoffMs: waitMs,});}+ await pause(waitMs);+ if (signal.aborted || this.halt.signal.aborted) return;+ // The stream may have reattached on its own while we waited — the+ // server closes a lapsed connection's socket, and the stream loop+ // reopens it. Reopening again would throw that healthy connection+ // away; beat on it instead.+ if (this.attaches === seenAttach) this.reattach();+ continue;} else if (!ok) {fail(new Error(`HTTP ${status}`));} else {@@ -1013,10 +1120,11 @@ export class SwitchEventStream {});}failures = 0;+ rejections = 0;backoff = BEAT_INTERVAL_MS;}}- await new Promise((r) => setTimeout(r, backoff));+ await pause(backoff);}}}