Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(sidecar): stop a hung-up session host blocking its room forever (#602)
Louis Amaudruz2aeeb80dce09
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 88a1baa78..210595e29 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3130,6 +3130,18 @@ published on its own.### [Unreleased]+#### Fixed+- **A session whose host hung up no longer blocks its room for good.** After a+ reset, a session host could finish and close its link to the watcher while a+ provider process it had started kept it alive. The watcher still saw it as+ running, so every room message failed and it was "started again" every five+ seconds, forever, without anyone being told. Now the host exits within five+ seconds of finishing whatever it left running; the watcher treats a host that+ hung up as gone, and stops it and what it started if it has not exited+ fifteen seconds later; and a session that still will not take a message after+ five starts in a row is left alone with its messages queued, and the room is+ told.+#### Changed- Claude Code sessions accept their agent definition in the launch spec and passit to the SDK, rather than reading `.claude/agents/<name>.md` on the host.diff --git a/console/packages/agent-providers/src/host/control.test.ts b/console/packages/agent-providers/src/host/control.test.tsindex 5de334a99..d30fe13cd 100644--- a/console/packages/agent-providers/src/host/control.test.ts+++ b/console/packages/agent-providers/src/host/control.test.ts@@ -36,8 +36,10 @@ const event = (sequence: number) => ({/** A session host child answering snapshots with its own name. */function host() {const child = new EventEmitter() as EventEmitter & {+ connected: boolean;send: (message: unknown, callback: (error: Error | null) => void) => boolean;};+ child.connected = true;child.send = (message, callback) => {callback(null);const { id } = message as { id: number };diff --git a/console/packages/agent-providers/src/host/session-channel.test.ts b/console/packages/agent-providers/src/host/session-channel.test.tsindex b2a3f4ae5..4699e0b1c 100644--- a/console/packages/agent-providers/src/host/session-channel.test.ts+++ b/console/packages/agent-providers/src/host/session-channel.test.ts@@ -6,6 +6,7 @@ import { join } from 'node:path';import { afterEach, expect, it, vi } from 'vitest';import {connectParent,+ HOST_HUNG_UP_MS,SessionHostFailedError,SessionLinks,SessionUnavailableError,@@ -19,6 +20,7 @@ function fakeChild() {};child.sent = [];(child as unknown as { connected: boolean }).connected = true;+ Object.assign(child, { pid: 4242, exitCode: null, signalCode: null });child.send = (message, callback) => {child.sent.push(message);callback(null);@@ -247,3 +249,81 @@ it('refuses a question from a host that has not said who it is, and tells who exexpect(exits).toEqual([['root', IDENTITY]]);expect(links.identity('root')).toBeNull();});++it('stops sending to a host that hung up, and waits for the next one', async () => {+ // A host that closed its pipe and did not exit — something it started still+ // running — used to read as ready for good, so every message went down a+ // closed pipe and failed, forever.+ const links = new SessionLinks();+ const hung = fakeChild();+ links.attach('root', hung as unknown as ChildProcess);+ hung.emit('message', { kind: 'ready' });+ const unanswered = links.request('root', { type: 'snapshot' }, 1000);+ await vi.waitFor(() => expect(hung.sent).toHaveLength(1));++ (hung as unknown as { connected: boolean }).connected = false;+ hung.emit('disconnect');++ await expect(unanswered).rejects.toThrow('hung up before it answered');+ expect(links.ready('root')).toBe(false);+ const next = links.request('root', { type: 'snapshot' }, 1000);+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(hung.sent).toHaveLength(1);++ const replacement = fakeChild();+ links.attach('root', replacement as unknown as ChildProcess);+ replacement.emit('message', { kind: 'ready' });+ await vi.waitFor(() => expect(replacement.sent).toHaveLength(1));+ const [request] = replacement.sent as { id: number }[];+ replacement.emit('message', { kind: 'reply', id: request!.id, ok: true, value: 'fresh' });+ expect(await next).toBe('fresh');+});++it('kills a host that hung up and stays alive, with what it started', async () => {+ vi.useFakeTimers();+ try {+ const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);+ const links = new SessionLinks();+ const hung = fakeChild();+ links.attach('root', hung as unknown as ChildProcess);+ hung.emit('disconnect');++ await vi.advanceTimersByTimeAsync(HOST_HUNG_UP_MS - 1);+ expect(kill).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1);+ // Its group: the provider it started is in it.+ expect(kill).toHaveBeenCalledWith(-4242, 'SIGKILL');+ } finally {+ vi.useRealTimers();+ }+});++it('leaves a host that hung up and then exited on its own', async () => {+ vi.useFakeTimers();+ try {+ const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);+ const links = new SessionLinks();+ const leaving = fakeChild();+ links.attach('root', leaving as unknown as ChildProcess);+ leaving.emit('disconnect');+ leaving.emit('exit', 0, null);++ await vi.advanceTimersByTimeAsync(HOST_HUNG_UP_MS * 2);+ expect(kill).not.toHaveBeenCalled();+ } finally {+ vi.useRealTimers();+ }+});++it('tells a request waiting on a host that the host stopped, rather than at its deadline', async () => {+ const links = new SessionLinks();+ const child = fakeChild();+ links.attach('root', child as unknown as ChildProcess);+ const started = Date.now();+ const waiting = links.request('root', { type: 'snapshot' }, 60000);++ child.emit('exit', 0, null);++ await expect(waiting).rejects.toThrow('stopped before it was ready');+ expect(Date.now() - started).toBeLessThan(5000);+});diff --git a/console/packages/agent-providers/src/host/session-channel.ts b/console/packages/agent-providers/src/host/session-channel.tsindex 28c9d50c4..611ee81c9 100644--- a/console/packages/agent-providers/src/host/session-channel.ts+++ b/console/packages/agent-providers/src/host/session-channel.ts@@ -5,6 +5,20 @@ import { serverEventSchema, type ServerEvent } from '@switch-console/shared/sessimport { z } from 'zod';import { LEASE_EXPIRED_EXIT_CODE } from './exit-codes';+/**+ * How long a session host that has finished may take to exit once it has+ * closed its end of the pipe. It exits by itself within this; see+ * `HOST_HUNG_UP_MS` for when its parent stops waiting.+ */+export const HOST_EXIT_GRACE_MS = 5000;++/**+ * How long a host that hung up its pipe may stay alive before its parent+ * kills it. Longer than the host's own grace, so a host that is leaving is+ * given the chance to leave on its own.+ */+export const HOST_HUNG_UP_MS = 15000;+/*** The pipe between a session host and the process that started it.*@@ -161,6 +175,8 @@ type Link = {pending: Map<number, Pending>;waiting: (() => void)[];subscribers: Set<(event: ServerEvent) => void>;+ /** How many hosts here have exited, so a waiting request can tell one did. */+ exits: number;};/**@@ -186,6 +202,7 @@ export class SessionLinks {pending: new Map(),waiting: [],subscribers: new Set(),+ exits: 0,};this.links.set(root, link);}@@ -235,12 +252,40 @@ export class SessionLinks {else pending.reject(new Error(message.error ?? 'The session host refused the request.'));}});+ // A host that closes its end of the pipe is leaving, and nothing sent+ // down the pipe reaches it again. It is gone from here from that moment,+ // not from the moment it exits: a host that hung up and stays alive —+ // something it started still running — holds its session's lock, so it+ // cannot be started again, while every message sent down the pipe fails.+ // Requests wait for the next host instead, and one that is still alive+ // after `HOST_HUNG_UP_MS` is killed with everything it started, which+ // frees the lock and lets its supervisor start it again.+ let hungUp: ReturnType<typeof setTimeout> | null = null;+ child.once('disconnect', () => {+ if (link.child !== child) return;+ link.ready = false;+ for (const [id, pending] of link.pending) {+ clearTimeout(pending.timer);+ pending.reject(new SessionUnavailableError('The session host hung up before it answered.'));+ link.pending.delete(id);+ }+ hungUp = setTimeout(() => {+ if (child.exitCode !== null || child.signalCode !== null) return;+ console.warn(+ `The session host at ${root} hung up ${HOST_HUNG_UP_MS / 1000} s ago and has not exited; stopping it and what it started.`+ );+ killHostGroup(child);+ }, HOST_HUNG_UP_MS);+ hungUp.unref?.();+ });child.once('exit', (code) => {+ if (hungUp) clearTimeout(hungUp);if (link.child !== child) return;const identity = link.identity;link.child = null;link.ready = false;link.identity = null;+ link.exits++;// A non-zero exit other than a lapsed lease is one the supervisor does// not recover from: the host has already written why.if (code !== null && code !== 0 && code !== LEASE_EXPIRED_EXIT_CODE)@@ -338,8 +383,13 @@ export class SessionLinks {async request(root: string, request: SessionRequest, timeoutMs: number): Promise<unknown> {const link = this.link(root);const deadline = Date.now() + timeoutMs;+ const exits = link.exits;while (!link.ready || !link.child) {if (link.failure !== null) throw new SessionHostFailedError(link.failure);+ // The host this request was waiting on is gone, and nothing here starts+ // another: say so now, so the caller starts it, rather than at the deadline.+ if (link.exits !== exits && !link.child)+ throw new SessionUnavailableError('The session host stopped before it was ready.');const remaining = deadline - Date.now();if (remaining <= 0) throw new SessionUnavailableError('The session host is not running.');await new Promise<void>((resolve) => {@@ -354,6 +404,10 @@ export class SessionLinks {});}const child = link.child;+ // Belt and braces for the 'disconnect' handler above: a send down a+ // closed pipe fails anyway, but saying why is clearer than the error.+ if (!child.connected)+ throw new SessionUnavailableError('The session host hung up and is on its way out.');const id = link.nextId++;return new Promise<unknown>((resolve, reject) => {const timer = setTimeout(@@ -381,6 +435,21 @@ export class SessionLinks {}}+/**+ * Kill a session host and every process in its group. A host is started+ * detached, so it leads its own group, and the provider processes it starts+ * are in it too.+ */+function killHostGroup(child: ChildProcess): void {+ if (!child.pid) return;+ try {+ if (process.platform === 'win32') child.kill('SIGKILL');+ else process.kill(-child.pid, 'SIGKILL');+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;+ }+}+export type SessionRequestHandlers = {[K in SessionRequest['type']]: (request: Extract<SessionRequest, { type: K }>diff --git a/console/packages/agent-providers/src/host/shared-daemon.ts b/console/packages/agent-providers/src/host/shared-daemon.tsindex fe9119731..b61e6f7b0 100644--- a/console/packages/agent-providers/src/host/shared-daemon.ts+++ b/console/packages/agent-providers/src/host/shared-daemon.ts@@ -8,7 +8,7 @@ import { replaceOwner } from './ownership-lock';import { ownProcessGroup } from './process-fence';import { checkProviderReadiness } from './provider-readiness';import { adapterFor } from './server';-import { SessionLinks } from './session-channel';+import { HOST_EXIT_GRACE_MS, SessionLinks } from './session-channel';import { sharedConfigSchema } from './shared-config';import { hostSessionProcess } from './shared-host';import { runSharedWatcher } from './shared-watcher';@@ -188,6 +188,15 @@ async function main(): Promise<void> {} finally {// The channel would otherwise keep this process alive after the host is done.process.disconnect();+ // Something the host started can outlive it too, and keep this process+ // alive holding the session's lock with no pipe to its parent. Exit+ // regardless: the supervisor then clears what is left of the group.+ setTimeout(() => {+ console.warn(+ `The session host finished but was still running ${HOST_EXIT_GRACE_MS / 1000} s later; exiting so its supervisor can stop what it left behind.`+ );+ process.exit();+ }, HOST_EXIT_GRACE_MS).unref();}}}diff --git a/console/packages/agent-providers/src/host/shared-watcher.test.ts b/console/packages/agent-providers/src/host/shared-watcher.test.tsindex c3e3054e0..ccde67b3d 100644--- a/console/packages/agent-providers/src/host/shared-watcher.test.ts+++ b/console/packages/agent-providers/src/host/shared-watcher.test.ts@@ -12,6 +12,7 @@ import { ensureSharedProcess, type Supervision } from './launch';import { type SessionRequest, SessionLinks } from './session-channel';import { sharedConfigSchema } from './shared-config';import {+ MAX_HOST_RESTARTS,stopSupersededSessions,runSharedWatcher,SharedWatchAssignments,@@ -1585,3 +1586,49 @@ function placementsOf(maps: Record<string, string>[]): Record<string, string> {Object.entries(maps.at(-1) ?? {}).map(([session, room]) => [room, session]));}++it('tells the room once a session keeps not taking a message, instead of starting it forever', async () => {+ // A host that is up but never takes the message — it hung up its pipe and+ // stayed alive — used to be started again every few seconds, for ever,+ // with nobody told.+ vi.useFakeTimers({ shouldAdvanceTime: true, toFake: ['setTimeout', 'clearTimeout'] });+ const root = await mkdtemp(join(tmpdir(), 'shared-watch-restart-cap-'));+ roots.push(root);+ paths.root = root;+ const config = await spawning(root);+ const hosts = sessionHosts();+ hosts.drop = true;+ const calls = switchOperations({ owner: 'ada' });+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ const error = vi.spyOn(console, 'error').mockImplementation(() => {});++ const abort = new AbortController();+ const run = runSharedWatcher(root, config, abort.signal, hosts.supervision, new WatcherControl());+ try {+ await eventually(() => streams.length === 1);+ await streams[0]!.onEvent!(addressed(1, 'room'));+ for (let second = 0; second < 120; second++) {+ if (calls.some((call) => call.name === 'send_targeted_message')) break;+ await vi.advanceTimersByTimeAsync(1000);+ }+ const told = calls.filter((call) => call.name === 'send_targeted_message');+ expect(told).toHaveLength(1);+ expect(String((told[0]!.body as { body: string }).body)).toContain(+ `after being started ${MAX_HOST_RESTARTS} times`+ );+ expect(error.mock.calls.some((call) => String(call[0]).includes('stay queued'))).toBe(true);++ // And it stops: nothing starts the host again on its own.+ const starts = vi.mocked(ensureSharedProcess).mock.calls.length;+ await vi.advanceTimersByTimeAsync(60_000);+ expect(vi.mocked(ensureSharedProcess).mock.calls.length).toBe(starts);+ // The message is still owed, for when the session can take it.+ expect((await SharedWatchAssignments.open(root)).pending().map((e) => e.messageId)).toEqual([+ 'message-1',+ ]);+ } finally {+ abort.abort();+ await run;+ vi.useRealTimers();+ }+});diff --git a/console/packages/agent-providers/src/host/shared-watcher.ts b/console/packages/agent-providers/src/host/shared-watcher.tsindex 69fd685b9..c730cc539 100644--- a/console/packages/agent-providers/src/host/shared-watcher.ts+++ b/console/packages/agent-providers/src/host/shared-watcher.ts@@ -121,6 +121,14 @@ const OWNERSHIP_RETRY_MS = 5000;/** How long a message waits for its session's host to start and take it. */const HOST_START_MS = 120000;+/**+ * How many times in a row a session's host is started again for a message it+ * would not take before the room is told it cannot be reached. Starting it+ * again is what fixes a host that stopped; one that keeps not taking the+ * message is stuck in a way starting it does not fix.+ */+export const MAX_HOST_RESTARTS = 5;+/** How often a room still waiting for an owner says so again. */const HELD_DISCLOSURE_MS = 30000;@@ -580,6 +588,7 @@ export async function runSharedWatcher(`Session ${sessionId} is running again; handing it the ${entry.queue.length} room message(s) that waited for it.`);entry.failed = null;+ entry.restarts = 0;pump(entry.config);}})@@ -679,14 +688,22 @@ export async function runSharedWatcher(* journal first and released on the host's acknowledgement, so one this* controller dies holding is routed again when it restarts.*- * `failed` is set when the host stopped on a failure it recorded: the+ * `failed` is set when the host stopped on a failure it recorded, or+ * would not take a message however often it was started again: the* messages stay queued and the host is not started again until something* changes — another room message for it, or its host coming up because- * somebody started it from Console.+ * somebody started it from Console. `restarts` counts the starts in a row+ * that did not get the first queued message taken.*/const pumps = new Map<string,- { queue: Handoff[]; running: boolean; failed: string | null; config: SharedHostConfig }+ {+ queue: Handoff[];+ running: boolean;+ failed: string | null;+ restarts: number;+ config: SharedHostConfig;+ }>();const pump = (config: SharedHostConfig) => {const sessionId = config.session.sessionId;@@ -704,6 +721,7 @@ export async function runSharedWatcher(HOST_START_MS);entry.queue.shift();+ entry.restarts = 0;pending = pending.then(() => assignments.released(event));await pending;} catch (error) {@@ -717,6 +735,15 @@ export async function runSharedWatcher(break;}if (!(error instanceof SessionUnavailableError)) throw error;+ if (entry.restarts >= MAX_HOST_RESTARTS) {+ entry.failed = `The session did not take a message after being started ${MAX_HOST_RESTARTS} times (${error.message}).`;+ console.error(+ `Session ${sessionId}: ${entry.failed} Its ${entry.queue.length} room message(s) stay queued; it is started again when the room next addresses the agent or the session is restarted from Console.`+ );+ void announce(config, entry.queue.at(-1) ?? event, entry.failed);+ break;+ }+ entry.restarts++;// Not running, or it stopped before it answered: start it again// and hand the message over once it is back.console.warn(@@ -742,7 +769,13 @@ export async function runSharedWatcher(const sessionId = config.session.sessionId;const sessionRoot = sharedSessionRoot(sessionId);if (!waiting) await assignments.park(event, false);- const entry = pumps.get(sessionId) ?? { queue: [], running: false, failed: null, config };+ const entry = pumps.get(sessionId) ?? {+ queue: [],+ running: false,+ failed: null,+ restarts: 0,+ config,+ };pumps.set(sessionId, entry);entry.config = config;const fresh = !entry.queue.some((queuedEvent) => queuedEvent.messageId === event.messageId);@@ -755,6 +788,7 @@ export async function runSharedWatcher(`Room ${event.roomId} addressed the agent again; starting session ${sessionId} again after it failed (${entry.failed}).`);entry.failed = null;+ entry.restarts = 0;}if (!links.ready(sessionRoot)) await launch(config);pump(config);