Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(sidecar): stop a hung-up session host blocking its room forever (#602)

Louis Amaudruz2aeeb80dce09
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 88a1baa78..210595e29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3130,6 +3130,18 @@ published on its own.
### [Unreleased]
+#### Fixed
+- **A session whose host hung up no longer blocks its room for good.** After a
+ reset, a session host could finish and close its link to the watcher while a
+ provider process it had started kept it alive. The watcher still saw it as
+ running, so every room message failed and it was "started again" every five
+ seconds, forever, without anyone being told. Now the host exits within five
+ seconds of finishing whatever it left running; the watcher treats a host that
+ hung up as gone, and stops it and what it started if it has not exited
+ fifteen seconds later; and a session that still will not take a message after
+ five starts in a row is left alone with its messages queued, and the room is
+ told.
+
#### Changed
- Claude Code sessions accept their agent definition in the launch spec and pass
it to the SDK, rather than reading `.claude/agents/<name>.md` on the host.
diff --git a/console/packages/agent-providers/src/host/control.test.ts b/console/packages/agent-providers/src/host/control.test.ts
index 5de334a99..d30fe13cd 100644
--- a/console/packages/agent-providers/src/host/control.test.ts
+++ b/console/packages/agent-providers/src/host/control.test.ts
@@ -36,8 +36,10 @@ const event = (sequence: number) => ({
/** A session host child answering snapshots with its own name. */
function host() {
const child = new EventEmitter() as EventEmitter & {
+ connected: boolean;
send: (message: unknown, callback: (error: Error | null) => void) => boolean;
};
+ child.connected = true;
child.send = (message, callback) => {
callback(null);
const { id } = message as { id: number };
diff --git a/console/packages/agent-providers/src/host/session-channel.test.ts b/console/packages/agent-providers/src/host/session-channel.test.ts
index b2a3f4ae5..4699e0b1c 100644
--- a/console/packages/agent-providers/src/host/session-channel.test.ts
+++ b/console/packages/agent-providers/src/host/session-channel.test.ts
@@ -6,6 +6,7 @@ import { join } from 'node:path';
import { afterEach, expect, it, vi } from 'vitest';
import {
connectParent,
+ HOST_HUNG_UP_MS,
SessionHostFailedError,
SessionLinks,
SessionUnavailableError,
@@ -19,6 +20,7 @@ function fakeChild() {
};
child.sent = [];
(child as unknown as { connected: boolean }).connected = true;
+ Object.assign(child, { pid: 4242, exitCode: null, signalCode: null });
child.send = (message, callback) => {
child.sent.push(message);
callback(null);
@@ -247,3 +249,81 @@ it('refuses a question from a host that has not said who it is, and tells who ex
expect(exits).toEqual([['root', IDENTITY]]);
expect(links.identity('root')).toBeNull();
});
+
+it('stops sending to a host that hung up, and waits for the next one', async () => {
+ // A host that closed its pipe and did not exit — something it started still
+ // running — used to read as ready for good, so every message went down a
+ // closed pipe and failed, forever.
+ const links = new SessionLinks();
+ const hung = fakeChild();
+ links.attach('root', hung as unknown as ChildProcess);
+ hung.emit('message', { kind: 'ready' });
+ const unanswered = links.request('root', { type: 'snapshot' }, 1000);
+ await vi.waitFor(() => expect(hung.sent).toHaveLength(1));
+
+ (hung as unknown as { connected: boolean }).connected = false;
+ hung.emit('disconnect');
+
+ await expect(unanswered).rejects.toThrow('hung up before it answered');
+ expect(links.ready('root')).toBe(false);
+ const next = links.request('root', { type: 'snapshot' }, 1000);
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ expect(hung.sent).toHaveLength(1);
+
+ const replacement = fakeChild();
+ links.attach('root', replacement as unknown as ChildProcess);
+ replacement.emit('message', { kind: 'ready' });
+ await vi.waitFor(() => expect(replacement.sent).toHaveLength(1));
+ const [request] = replacement.sent as { id: number }[];
+ replacement.emit('message', { kind: 'reply', id: request!.id, ok: true, value: 'fresh' });
+ expect(await next).toBe('fresh');
+});
+
+it('kills a host that hung up and stays alive, with what it started', async () => {
+ vi.useFakeTimers();
+ try {
+ const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);
+ const links = new SessionLinks();
+ const hung = fakeChild();
+ links.attach('root', hung as unknown as ChildProcess);
+ hung.emit('disconnect');
+
+ await vi.advanceTimersByTimeAsync(HOST_HUNG_UP_MS - 1);
+ expect(kill).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1);
+ // Its group: the provider it started is in it.
+ expect(kill).toHaveBeenCalledWith(-4242, 'SIGKILL');
+ } finally {
+ vi.useRealTimers();
+ }
+});
+
+it('leaves a host that hung up and then exited on its own', async () => {
+ vi.useFakeTimers();
+ try {
+ const kill = vi.spyOn(process, 'kill').mockImplementation(() => true);
+ const links = new SessionLinks();
+ const leaving = fakeChild();
+ links.attach('root', leaving as unknown as ChildProcess);
+ leaving.emit('disconnect');
+ leaving.emit('exit', 0, null);
+
+ await vi.advanceTimersByTimeAsync(HOST_HUNG_UP_MS * 2);
+ expect(kill).not.toHaveBeenCalled();
+ } finally {
+ vi.useRealTimers();
+ }
+});
+
+it('tells a request waiting on a host that the host stopped, rather than at its deadline', async () => {
+ const links = new SessionLinks();
+ const child = fakeChild();
+ links.attach('root', child as unknown as ChildProcess);
+ const started = Date.now();
+ const waiting = links.request('root', { type: 'snapshot' }, 60000);
+
+ child.emit('exit', 0, null);
+
+ await expect(waiting).rejects.toThrow('stopped before it was ready');
+ expect(Date.now() - started).toBeLessThan(5000);
+});
diff --git a/console/packages/agent-providers/src/host/session-channel.ts b/console/packages/agent-providers/src/host/session-channel.ts
index 28c9d50c4..611ee81c9 100644
--- a/console/packages/agent-providers/src/host/session-channel.ts
+++ b/console/packages/agent-providers/src/host/session-channel.ts
@@ -5,6 +5,20 @@ import { serverEventSchema, type ServerEvent } from '@switch-console/shared/sess
import { z } from 'zod';
import { LEASE_EXPIRED_EXIT_CODE } from './exit-codes';
+/**
+ * How long a session host that has finished may take to exit once it has
+ * closed its end of the pipe. It exits by itself within this; see
+ * `HOST_HUNG_UP_MS` for when its parent stops waiting.
+ */
+export const HOST_EXIT_GRACE_MS = 5000;
+
+/**
+ * How long a host that hung up its pipe may stay alive before its parent
+ * kills it. Longer than the host's own grace, so a host that is leaving is
+ * given the chance to leave on its own.
+ */
+export const HOST_HUNG_UP_MS = 15000;
+
/**
* The pipe between a session host and the process that started it.
*
@@ -161,6 +175,8 @@ type Link = {
pending: Map<number, Pending>;
waiting: (() => void)[];
subscribers: Set<(event: ServerEvent) => void>;
+ /** How many hosts here have exited, so a waiting request can tell one did. */
+ exits: number;
};
/**
@@ -186,6 +202,7 @@ export class SessionLinks {
pending: new Map(),
waiting: [],
subscribers: new Set(),
+ exits: 0,
};
this.links.set(root, link);
}
@@ -235,12 +252,40 @@ export class SessionLinks {
else pending.reject(new Error(message.error ?? 'The session host refused the request.'));
}
});
+ // A host that closes its end of the pipe is leaving, and nothing sent
+ // down the pipe reaches it again. It is gone from here from that moment,
+ // not from the moment it exits: a host that hung up and stays alive —
+ // something it started still running — holds its session's lock, so it
+ // cannot be started again, while every message sent down the pipe fails.
+ // Requests wait for the next host instead, and one that is still alive
+ // after `HOST_HUNG_UP_MS` is killed with everything it started, which
+ // frees the lock and lets its supervisor start it again.
+ let hungUp: ReturnType<typeof setTimeout> | null = null;
+ child.once('disconnect', () => {
+ if (link.child !== child) return;
+ link.ready = false;
+ for (const [id, pending] of link.pending) {
+ clearTimeout(pending.timer);
+ pending.reject(new SessionUnavailableError('The session host hung up before it answered.'));
+ link.pending.delete(id);
+ }
+ hungUp = setTimeout(() => {
+ if (child.exitCode !== null || child.signalCode !== null) return;
+ console.warn(
+ `The session host at ${root} hung up ${HOST_HUNG_UP_MS / 1000} s ago and has not exited; stopping it and what it started.`
+ );
+ killHostGroup(child);
+ }, HOST_HUNG_UP_MS);
+ hungUp.unref?.();
+ });
child.once('exit', (code) => {
+ if (hungUp) clearTimeout(hungUp);
if (link.child !== child) return;
const identity = link.identity;
link.child = null;
link.ready = false;
link.identity = null;
+ link.exits++;
// A non-zero exit other than a lapsed lease is one the supervisor does
// not recover from: the host has already written why.
if (code !== null && code !== 0 && code !== LEASE_EXPIRED_EXIT_CODE)
@@ -338,8 +383,13 @@ export class SessionLinks {
async request(root: string, request: SessionRequest, timeoutMs: number): Promise<unknown> {
const link = this.link(root);
const deadline = Date.now() + timeoutMs;
+ const exits = link.exits;
while (!link.ready || !link.child) {
if (link.failure !== null) throw new SessionHostFailedError(link.failure);
+ // The host this request was waiting on is gone, and nothing here starts
+ // another: say so now, so the caller starts it, rather than at the deadline.
+ if (link.exits !== exits && !link.child)
+ throw new SessionUnavailableError('The session host stopped before it was ready.');
const remaining = deadline - Date.now();
if (remaining <= 0) throw new SessionUnavailableError('The session host is not running.');
await new Promise<void>((resolve) => {
@@ -354,6 +404,10 @@ export class SessionLinks {
});
}
const child = link.child;
+ // Belt and braces for the 'disconnect' handler above: a send down a
+ // closed pipe fails anyway, but saying why is clearer than the error.
+ if (!child.connected)
+ throw new SessionUnavailableError('The session host hung up and is on its way out.');
const id = link.nextId++;
return new Promise<unknown>((resolve, reject) => {
const timer = setTimeout(
@@ -381,6 +435,21 @@ export class SessionLinks {
}
}
+/**
+ * Kill a session host and every process in its group. A host is started
+ * detached, so it leads its own group, and the provider processes it starts
+ * are in it too.
+ */
+function killHostGroup(child: ChildProcess): void {
+ if (!child.pid) return;
+ try {
+ if (process.platform === 'win32') child.kill('SIGKILL');
+ else process.kill(-child.pid, 'SIGKILL');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+ }
+}
+
export type SessionRequestHandlers = {
[K in SessionRequest['type']]: (
request: Extract<SessionRequest, { type: K }>
diff --git a/console/packages/agent-providers/src/host/shared-daemon.ts b/console/packages/agent-providers/src/host/shared-daemon.ts
index fe9119731..b61e6f7b0 100644
--- a/console/packages/agent-providers/src/host/shared-daemon.ts
+++ b/console/packages/agent-providers/src/host/shared-daemon.ts
@@ -8,7 +8,7 @@ import { replaceOwner } from './ownership-lock';
import { ownProcessGroup } from './process-fence';
import { checkProviderReadiness } from './provider-readiness';
import { adapterFor } from './server';
-import { SessionLinks } from './session-channel';
+import { HOST_EXIT_GRACE_MS, SessionLinks } from './session-channel';
import { sharedConfigSchema } from './shared-config';
import { hostSessionProcess } from './shared-host';
import { runSharedWatcher } from './shared-watcher';
@@ -188,6 +188,15 @@ async function main(): Promise<void> {
} finally {
// The channel would otherwise keep this process alive after the host is done.
process.disconnect();
+ // Something the host started can outlive it too, and keep this process
+ // alive holding the session's lock with no pipe to its parent. Exit
+ // regardless: the supervisor then clears what is left of the group.
+ setTimeout(() => {
+ console.warn(
+ `The session host finished but was still running ${HOST_EXIT_GRACE_MS / 1000} s later; exiting so its supervisor can stop what it left behind.`
+ );
+ process.exit();
+ }, HOST_EXIT_GRACE_MS).unref();
}
}
}
diff --git a/console/packages/agent-providers/src/host/shared-watcher.test.ts b/console/packages/agent-providers/src/host/shared-watcher.test.ts
index c3e3054e0..ccde67b3d 100644
--- a/console/packages/agent-providers/src/host/shared-watcher.test.ts
+++ b/console/packages/agent-providers/src/host/shared-watcher.test.ts
@@ -12,6 +12,7 @@ import { ensureSharedProcess, type Supervision } from './launch';
import { type SessionRequest, SessionLinks } from './session-channel';
import { sharedConfigSchema } from './shared-config';
import {
+ MAX_HOST_RESTARTS,
stopSupersededSessions,
runSharedWatcher,
SharedWatchAssignments,
@@ -1585,3 +1586,49 @@ function placementsOf(maps: Record<string, string>[]): Record<string, string> {
Object.entries(maps.at(-1) ?? {}).map(([session, room]) => [room, session])
);
}
+
+it('tells the room once a session keeps not taking a message, instead of starting it forever', async () => {
+ // A host that is up but never takes the message — it hung up its pipe and
+ // stayed alive — used to be started again every few seconds, for ever,
+ // with nobody told.
+ vi.useFakeTimers({ shouldAdvanceTime: true, toFake: ['setTimeout', 'clearTimeout'] });
+ const root = await mkdtemp(join(tmpdir(), 'shared-watch-restart-cap-'));
+ roots.push(root);
+ paths.root = root;
+ const config = await spawning(root);
+ const hosts = sessionHosts();
+ hosts.drop = true;
+ const calls = switchOperations({ owner: 'ada' });
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ const abort = new AbortController();
+ const run = runSharedWatcher(root, config, abort.signal, hosts.supervision, new WatcherControl());
+ try {
+ await eventually(() => streams.length === 1);
+ await streams[0]!.onEvent!(addressed(1, 'room'));
+ for (let second = 0; second < 120; second++) {
+ if (calls.some((call) => call.name === 'send_targeted_message')) break;
+ await vi.advanceTimersByTimeAsync(1000);
+ }
+ const told = calls.filter((call) => call.name === 'send_targeted_message');
+ expect(told).toHaveLength(1);
+ expect(String((told[0]!.body as { body: string }).body)).toContain(
+ `after being started ${MAX_HOST_RESTARTS} times`
+ );
+ expect(error.mock.calls.some((call) => String(call[0]).includes('stay queued'))).toBe(true);
+
+ // And it stops: nothing starts the host again on its own.
+ const starts = vi.mocked(ensureSharedProcess).mock.calls.length;
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(vi.mocked(ensureSharedProcess).mock.calls.length).toBe(starts);
+ // The message is still owed, for when the session can take it.
+ expect((await SharedWatchAssignments.open(root)).pending().map((e) => e.messageId)).toEqual([
+ 'message-1',
+ ]);
+ } finally {
+ abort.abort();
+ await run;
+ vi.useRealTimers();
+ }
+});
diff --git a/console/packages/agent-providers/src/host/shared-watcher.ts b/console/packages/agent-providers/src/host/shared-watcher.ts
index 69fd685b9..c730cc539 100644
--- a/console/packages/agent-providers/src/host/shared-watcher.ts
+++ b/console/packages/agent-providers/src/host/shared-watcher.ts
@@ -121,6 +121,14 @@ const OWNERSHIP_RETRY_MS = 5000;
/** How long a message waits for its session's host to start and take it. */
const HOST_START_MS = 120000;
+/**
+ * How many times in a row a session's host is started again for a message it
+ * would not take before the room is told it cannot be reached. Starting it
+ * again is what fixes a host that stopped; one that keeps not taking the
+ * message is stuck in a way starting it does not fix.
+ */
+export const MAX_HOST_RESTARTS = 5;
+
/** How often a room still waiting for an owner says so again. */
const HELD_DISCLOSURE_MS = 30000;
@@ -580,6 +588,7 @@ export async function runSharedWatcher(
`Session ${sessionId} is running again; handing it the ${entry.queue.length} room message(s) that waited for it.`
);
entry.failed = null;
+ entry.restarts = 0;
pump(entry.config);
}
})
@@ -679,14 +688,22 @@ export async function runSharedWatcher(
* journal first and released on the host's acknowledgement, so one this
* controller dies holding is routed again when it restarts.
*
- * `failed` is set when the host stopped on a failure it recorded: the
+ * `failed` is set when the host stopped on a failure it recorded, or
+ * would not take a message however often it was started again: the
* messages stay queued and the host is not started again until something
* changes — another room message for it, or its host coming up because
- * somebody started it from Console.
+ * somebody started it from Console. `restarts` counts the starts in a row
+ * that did not get the first queued message taken.
*/
const pumps = new Map<
string,
- { queue: Handoff[]; running: boolean; failed: string | null; config: SharedHostConfig }
+ {
+ queue: Handoff[];
+ running: boolean;
+ failed: string | null;
+ restarts: number;
+ config: SharedHostConfig;
+ }
>();
const pump = (config: SharedHostConfig) => {
const sessionId = config.session.sessionId;
@@ -704,6 +721,7 @@ export async function runSharedWatcher(
HOST_START_MS
);
entry.queue.shift();
+ entry.restarts = 0;
pending = pending.then(() => assignments.released(event));
await pending;
} catch (error) {
@@ -717,6 +735,15 @@ export async function runSharedWatcher(
break;
}
if (!(error instanceof SessionUnavailableError)) throw error;
+ if (entry.restarts >= MAX_HOST_RESTARTS) {
+ entry.failed = `The session did not take a message after being started ${MAX_HOST_RESTARTS} times (${error.message}).`;
+ console.error(
+ `Session ${sessionId}: ${entry.failed} Its ${entry.queue.length} room message(s) stay queued; it is started again when the room next addresses the agent or the session is restarted from Console.`
+ );
+ void announce(config, entry.queue.at(-1) ?? event, entry.failed);
+ break;
+ }
+ entry.restarts++;
// Not running, or it stopped before it answered: start it again
// and hand the message over once it is back.
console.warn(
@@ -742,7 +769,13 @@ export async function runSharedWatcher(
const sessionId = config.session.sessionId;
const sessionRoot = sharedSessionRoot(sessionId);
if (!waiting) await assignments.park(event, false);
- const entry = pumps.get(sessionId) ?? { queue: [], running: false, failed: null, config };
+ const entry = pumps.get(sessionId) ?? {
+ queue: [],
+ running: false,
+ failed: null,
+ restarts: 0,
+ config,
+ };
pumps.set(sessionId, entry);
entry.config = config;
const fresh = !entry.queue.some((queuedEvent) => queuedEvent.messageId === event.messageId);
@@ -755,6 +788,7 @@ export async function runSharedWatcher(
`Room ${event.roomId} addressed the agent again; starting session ${sessionId} again after it failed (${entry.failed}).`
);
entry.failed = null;
+ entry.restarts = 0;
}
if (!links.ready(sessionRoot)) await launch(config);
pump(config);