Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(runtime): heartbeat lockout after back-off, and self-takeover on a cancelled open (#607)

Louis Amaudruz497dc25974f3
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
index fbc6ea524..a79c3cbf3 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -680,6 +680,286 @@ describe('the heartbeat', () => {
});
});
+describe('the heartbeat after a reconnect', () => {
+ /**
+ * A server that reattaches on every open, can drop the current stream, and
+ * records when each stream was opened and each beat arrived. The server's
+ * own rule is what these tests hold the client to: a new connection that
+ * has not beaten within `HEARTBEAT_TTL_MS` is closed for silence.
+ */
+ const HEARTBEAT_TTL_MS = 6000;
+
+ /** Either end may close a stream first; the second close is not an error here. */
+ function closeQuietly(controller: ReadableStreamDefaultController<Uint8Array>): void {
+ try {
+ controller.close();
+ } catch {
+ // Already closed by the other side.
+ }
+ }
+
+ function reattachingServer(beatReply: () => 'fail' | 'ok' | 404) {
+ const opens: number[] = [];
+ const beats: number[] = [];
+ let generation = 0;
+ let current: ReadableStreamDefaultController<Uint8Array> | null = null;
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (String(url).includes('/events')) {
+ opens.push(Date.now());
+ generation += 1;
+ const gen = generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ current = controller;
+ controller.enqueue(connected(gen));
+ init.signal.addEventListener('abort', () => closeQuietly(controller), {
+ once: true,
+ });
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ beats.push(Date.now());
+ const reply = beatReply();
+ if (reply === 'fail') throw new TypeError('fetch failed');
+ if (reply === 404) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ /** The server ends the current stream, as it does a lapsed connection. */
+ const drop = (): void => {
+ if (current) closeQuietly(current);
+ };
+ return { fetchMock, opens, beats, drop };
+ }
+
+ it('beats within the server’s deadline on a new connection, however long it had been backing off', async () => {
+ // The lockout: beats fail for a while, so the gap between them climbs to
+ // its 30-second cap. The stream then reopens. The server starts its
+ // 6-second clock on the new connection — and the client used to sit out
+ // the rest of the 30 seconds before saying anything, lose the connection
+ // for silence, and repeat that on every connection it opened afterwards.
+ vi.useFakeTimers();
+ let failing = true;
+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ // Long enough of failed beats for the gap to reach its cap.
+ await vi.advanceTimersByTimeAsync(70_000);
+ failing = false;
+ server.drop();
+ await vi.advanceTimersByTimeAsync(40_000);
+
+ const reopenedAt = server.opens[1];
+ expect(reopenedAt).toBeDefined();
+ const firstBeatAfter = server.beats.find((t) => t >= reopenedAt!);
+ expect(firstBeatAfter).toBeDefined();
+ expect(firstBeatAfter! - reopenedAt!).toBeLessThan(HEARTBEAT_TTL_MS);
+ abort.abort();
+ });
+
+ it('keeps beating at the base cadence on the new connection once it lands', async () => {
+ vi.useFakeTimers();
+ let failing = true;
+ const server = reattachingServer(() => (failing ? 'fail' : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(70_000);
+ failing = false;
+ server.drop();
+ await vi.advanceTimersByTimeAsync(5_000);
+ const settled = server.beats.length;
+ await vi.advanceTimersByTimeAsync(10 * BEAT_INTERVAL_MS);
+
+ expect(server.beats.length - settled).toBeGreaterThanOrEqual(9);
+ abort.abort();
+ });
+
+ it('does not reconnect at full rate against a server that keeps refusing its beats', async () => {
+ // The brake the old ordering provided, which beating promptly on every
+ // attach would otherwise remove: refuse, reopen, attach, beat, refuse… The
+ // wait now sits before each reopen and doubles, so a refusing server sees
+ // a handful of reopens, not one every couple of seconds per agent.
+ vi.useFakeTimers();
+ const server = reattachingServer(() => 404);
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(60_000);
+
+ // Doubling from 4s: reopens near 4, 12, 28, 58 seconds.
+ expect(server.opens.length).toBeLessThanOrEqual(6);
+ abort.abort();
+ });
+
+ it('does not throw away a connection the stream reopened while it waited', async () => {
+ // A lapsed connection's socket is closed by the server, and the stream
+ // loop reopens it by itself. If that lands during the heartbeat's wait
+ // after a refusal, reopening again would discard a healthy connection.
+ vi.useFakeTimers();
+ let refuse = true;
+ const server = reattachingServer(() => (refuse ? 404 : 'ok'));
+ const { abort } = makeStream(server.fetchMock, { rooms: [] });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+ expect(server.beats.length).toBe(1);
+ refuse = false;
+ // The server ends the stream now, inside the heartbeat's wait.
+ server.drop();
+ await vi.advanceTimersByTimeAsync(20_000);
+
+ // One open for the start, one for the stream's own reopen — not a third
+ // from the heartbeat discarding it.
+ expect(server.opens.length).toBe(2);
+ abort.abort();
+ });
+});
+
+describe('reopening while an open is still in flight', () => {
+ /**
+ * A server that keeps the incarnation the way the real one does: every open
+ * it receives makes a new one, whether or not the client stays to read the
+ * answer, and a reattach claiming an older one is refused as a takeover.
+ * The next open can be held unanswered, the way a slow connection holds it.
+ */
+ function incarnationServer() {
+ const state = {
+ generation: 0,
+ opens: 0,
+ beat: 'ok' as 'ok' | 404 | 'slow-404-and-drop',
+ subscribeDrops: false,
+ holdNext: false,
+ release: (): void => {},
+ drop: (): void => {},
+ };
+ const body = (gen: number, signal: AbortSignal) =>
+ new ReadableStream<Uint8Array>({
+ start(controller) {
+ const close = (): void => {
+ try {
+ controller.close();
+ } catch {
+ // already closed
+ }
+ };
+ state.drop = close;
+ controller.enqueue(connected(gen));
+ signal.addEventListener('abort', close, { once: true });
+ },
+ });
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
+ if (String(url).includes('connection/subscribe')) {
+ if (!state.subscribeDrops)
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ state.subscribeDrops = false;
+ // The stream ends while the claim is still in flight.
+ state.holdNext = true;
+ state.drop();
+ await new Promise((r) => setTimeout(r, 3_000));
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ if (!String(url).includes('/events')) {
+ if (state.beat === 'slow-404-and-drop') {
+ // The stream ends while this beat is in flight, and its answer comes
+ // back just after the client has started reopening.
+ state.beat = 'ok';
+ state.holdNext = true;
+ state.drop();
+ await new Promise((r) => setTimeout(r, 1_200));
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ }
+ if (state.beat === 404)
+ return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ }
+ state.opens += 1;
+ const claimed = new URL(String(url)).searchParams.get('expected_generation');
+ if (claimed !== null && Number(claimed) < state.generation)
+ return {
+ ok: false,
+ status: 409,
+ body: null,
+ text: async (): Promise<string> =>
+ JSON.stringify({ detail: { code: 'taken_over', message: 'reattach refused' } }),
+ };
+ // The server acts on the open the moment it arrives.
+ state.generation += 1;
+ const gen = state.generation;
+ const answer = {
+ ok: true,
+ status: 200,
+ body: body(gen, init.signal),
+ text: async (): Promise<string> => '',
+ };
+ if (!state.holdNext) return answer;
+ state.holdNext = false;
+ return new Promise((resolve, reject) => {
+ state.release = () => resolve(answer);
+ init.signal.addEventListener(
+ 'abort',
+ () => reject(new DOMException('aborted', 'AbortError')),
+ { once: true }
+ );
+ });
+ });
+ return { state, fetchMock };
+ }
+
+ it('does not cancel an open the server has already acted on, and so does not stand down', async () => {
+ // The self-takeover seen on the pilot. The server closes the stream, the
+ // client starts reopening, and a beat sent just before comes back 404.
+ // Reopening in answer used to cancel the open already in flight — which
+ // the server had acted on, moving the connection to a new incarnation.
+ // The client never read that answer, reattached claiming the old one, was
+ // refused as a takeover, and stood down for good. The "other client" was
+ // its own cancelled request.
+ vi.useFakeTimers();
+ const { state, fetchMock } = incarnationServer();
+ const onEvicted = vi.fn();
+ const { abort } = makeStream(fetchMock, { rooms: [], onEvicted });
+
+ await vi.advanceTimersByTimeAsync(3 * BEAT_INTERVAL_MS);
+ expect(state.generation).toBe(1);
+
+ state.beat = 'slow-404-and-drop';
+ // Long enough for the refused beat's wait to end while the open is held.
+ await vi.advanceTimersByTimeAsync(12_000);
+ state.release();
+ await vi.advanceTimersByTimeAsync(5 * BEAT_INTERVAL_MS);
+
+ expect(onEvicted).not.toHaveBeenCalled();
+ expect(state.opens).toBe(2);
+ expect(beatGenerations(fetchMock).at(-1)).toBe(2);
+ abort.abort();
+ });
+
+ it('carries out a repoint asked for mid-open once that open is answered', async () => {
+ // The other way into the same hole: a room change reopens to include the
+ // new room. Mid-open, it must wait for the answer rather than cancel it —
+ // and must still happen, since the open in flight was built without it.
+ vi.useFakeTimers();
+ const { state, fetchMock } = incarnationServer();
+ const onEvicted = vi.fn();
+ const { stream, abort } = makeStream(fetchMock, { rooms: ['room-a'], onEvicted });
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS);
+
+ state.subscribeDrops = true;
+ const repointing = stream.repoint('room-b');
+ // The claim returns while the stream's reopen is held unanswered.
+ await vi.advanceTimersByTimeAsync(5_000);
+ state.release();
+ await vi.advanceTimersByTimeAsync(5_000);
+ await repointing;
+
+ expect(onEvicted).not.toHaveBeenCalled();
+ const opens = urlsFor(fetchMock, '/events');
+ expect(opens.at(-1)).toContain('rooms=room-b');
+ abort.abort();
+ });
+});
+
describe('a connection another client takes over', () => {
it('stands down when the room it repoints to is refused, without reopening', async () => {
// `repoint` claims the room *before* it reopens, so the open's own fence
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index d25e01cfc..f71dd7eeb 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -336,6 +336,23 @@ export class SwitchEventStream {
* incarnation we have just replaced.
*/
private readonly fence = new ReattachFence();
+ /**
+ * How many times the server has named an incarnation for this client, and
+ * how to cut the heartbeat's current wait short when it does.
+ *
+ * A fresh attach starts the server's 6-second clock on a connection that has
+ * never beaten, so the heartbeat must go out promptly — whatever back-off an
+ * earlier, failing connection had built up. See `beatLoop`.
+ */
+ private attaches = 0;
+ private wakeBeat: () => void = () => {};
+ /**
+ * Reopens asked for, and how many of those the open now in progress was
+ * built after — so a reopen asked for mid-open is carried out once that open
+ * is answered, never by cancelling it. See `reopen`.
+ */
+ private reopensWanted = 0;
+ private openCarries = 0;
constructor(deps: SwitchEventStreamDeps) {
this.deps = deps;
@@ -404,8 +421,35 @@ export class SwitchEventStream {
this.reopen();
}
+ /**
+ * Replace the socket — but never by cancelling an open the server has not
+ * answered yet.
+ *
+ * An open is a request the server acts on as soon as it arrives: it makes a
+ * new incarnation of the connection there and then. Cancelling it
+ * client-side does not undo that, it only stops us reading the answer that
+ * names the new incarnation. The next open then claims the incarnation
+ * before it, which the server has moved past, and is refused as a takeover
+ * — terminal. The client stands down, and the "other client" it yielded to
+ * was its own cancelled request.
+ *
+ * So a reopen asked for while an open is in flight is remembered instead,
+ * and carried out when that open's `connection_state` arrives, by which time
+ * we know the incarnation to claim. An open that was already built after the
+ * request carries it and needs no second one.
+ */
private reopen(): void {
- this.socketAbort?.abort();
+ this.reopensWanted += 1;
+ if (this.fence.admitting) this.socketAbort?.abort();
+ }
+
+ /**
+ * Reopen only to get attached again — the heartbeat's reason. An open
+ * already in flight does exactly that, so there is nothing to add to it; and
+ * one scheduled after a dropped socket carries the request anyway.
+ */
+ private reattach(): void {
+ if (this.fence.admitting) this.reopen();
}
/**
@@ -663,6 +707,7 @@ export class SwitchEventStream {
client_version: RUNTIME_VERSION,
});
this.declaredSpawnCapable = this.spawnCapable;
+ this.openCarries = this.reopensWanted;
if (this.declaredSpawnCapable) params.set('spawn_capable', 'true');
if (this.rooms.length) params.set('rooms', this.rooms.join(','));
// Reattaching, so say which incarnation we believe we still are and
@@ -771,7 +816,17 @@ export class SwitchEventStream {
server: frame.data.server ?? null,
});
this.reportRooms(frame.data.rooms);
+ // A reopen asked for while this open was in flight, for something this
+ // open was built too early to carry. Now that we know the incarnation
+ // to claim, it is safe to replace the socket — and done before the
+ // gate opens, so no beat goes out on a socket about to be dropped.
+ if (this.reopensWanted > this.openCarries) {
+ this.socketAbort?.abort();
+ return;
+ }
this.fence.attached();
+ this.attaches += 1;
+ this.wakeBeat();
this.redeclare();
this.deps.onConnected?.();
return;
@@ -905,6 +960,35 @@ export class SwitchEventStream {
const { log, signal, connectionId } = this.deps;
let failures = 0;
let backoff = BEAT_INTERVAL_MS;
+ // Rejections in a row, which pace reopening rather than beating. Kept
+ // apart from `backoff` because a fresh attach resets that and must not
+ // reset this: see the 404/409 branch below.
+ let rejections = 0;
+ let seenAttach = this.attaches;
+
+ /**
+ * Wait `ms`, but stop waiting the moment the server names a new
+ * incarnation — and do not start waiting if it already has.
+ *
+ * The back-off used to be served in full across a reattach. A beat that
+ * had been failing left the loop sleeping for up to 30 seconds; the stream
+ * reopened, the server started its 6-second clock on the new connection,
+ * and the next beat arrived long after the server had closed it for
+ * silence. That beat was refused, which counted as another failure, which
+ * kept the wait at 30 seconds. The client could not get out of it on its
+ * own: one hiccup, and it lapsed every connection it ever opened again.
+ */
+ const pause = async (ms: number): Promise<void> => {
+ if (this.attaches !== seenAttach) return;
+ await new Promise<void>((resolve) => {
+ const timer = setTimeout(resolve, ms);
+ this.wakeBeat = () => {
+ clearTimeout(timer);
+ resolve();
+ };
+ });
+ this.wakeBeat = () => {};
+ };
/** Count a beat that did not land and slow the loop down. Returns whether
* to report this one: the first, then powers of two, so a permanent
@@ -958,6 +1042,13 @@ export class SwitchEventStream {
// beat could be sent under an incarnation later than the one recorded.
await Promise.race([this.fence.reached, until(signal, this.halt.signal)]);
if (signal.aborted || this.halt.signal.aborted) return;
+ // A connection the server has just attached has never beaten, and has
+ // 6 seconds to. Whatever an earlier connection's failures built up says
+ // nothing about this one, so it starts at the base cadence.
+ if (this.attaches !== seenAttach) {
+ seenAttach = this.attaches;
+ backoff = BEAT_INTERVAL_MS;
+ }
const tick = await this.fence.tick(beat);
if (signal.aborted || this.halt.signal.aborted) return;
@@ -989,20 +1080,36 @@ export class SwitchEventStream {
if (status === 404 || status === 409) {
// The server answered, so this is not an outage — but it is not a
// beat that landed either: we are not attached, and only a reopen
- // fixes that. Reopen, and slow down all the same. A reopen that
- // keeps being refused is a client that cannot currently succeed,
- // and it must not spend the endpoint at full rate while it fails.
- const report = slowDown();
- this.reopen();
- if (report) {
+ // fixes that. A reopen that keeps being refused is a client that
+ // cannot currently succeed, and it must not spend the endpoint at
+ // full rate while it fails.
+ //
+ // So the wait goes *before* the reopen, and grows with each refusal
+ // in a row. It used to go after — reopen at once, then sleep the
+ // back-off before beating — which starved the very connection the
+ // reopen had just made. Pacing the reopen instead keeps the same
+ // brake on a refusing server while letting each new connection beat
+ // as soon as it is attached.
+ failures += 1;
+ rejections += 1;
+ const waitMs = Math.min(BEAT_INTERVAL_MS * 2 ** rejections, MAX_BACKOFF_MS);
+ if ((failures & (failures - 1)) === 0) {
log.warn('SwitchEventStream: heartbeat rejected — reopening', {
event: 'switch_beat_rejected',
status,
connectionId,
failures,
- backoffMs: backoff,
+ backoffMs: waitMs,
});
}
+ await pause(waitMs);
+ if (signal.aborted || this.halt.signal.aborted) return;
+ // The stream may have reattached on its own while we waited — the
+ // server closes a lapsed connection's socket, and the stream loop
+ // reopens it. Reopening again would throw that healthy connection
+ // away; beat on it instead.
+ if (this.attaches === seenAttach) this.reattach();
+ continue;
} else if (!ok) {
fail(new Error(`HTTP ${status}`));
} else {
@@ -1013,10 +1120,11 @@ export class SwitchEventStream {
});
}
failures = 0;
+ rejections = 0;
backoff = BEAT_INTERVAL_MS;
}
}
- await new Promise((r) => setTimeout(r, backoff));
+ await pause(backoff);
}
}
}