Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix: a watcher no longer stands down for itself, and a stuck one can be replaced (#611)

Louis Amaudruz89f5605d958a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c9e6bcbf..9835d3e3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2999,6 +2999,15 @@ The Switch protocol client and MCP runtime
### [Unreleased]
+#### Fixed
+- **A connection no longer gives itself up to itself.** A placements or room
+ subscribe request sent under one incarnation and answered after this client
+ had reopened its own stream was refused as "taken over", naming this client
+ as the new holder, and the client stood down for good with nobody else
+ anywhere near the connection. Such a refusal now only fails the request,
+ which is stated again on the next change or reconnect; a real takeover is
+ still caught by the next heartbeat.
+
### [0.7.0] - 2026-09-25
#### Changed
@@ -3173,6 +3182,13 @@ published on its own.
instead of relying on a live connection to the sidecar.
#### Fixed
+- **A stuck process can no longer block replacing a watcher.** Stopping a
+ watcher or session host sent SIGTERM and waited; one waiting on a child that
+ would never finish — a session host that hung up and stayed alive — never
+ exited, so every update to that agent failed with "The SDK host has not
+ stopped". A host asked to stop is now killed, with every process it started,
+ if it has not gone after 10 s (a supervised host) or 20 s (one being
+ replaced).
- **A session whose host hung up no longer blocks its room for good.** After a
reset, a session host could finish and close its link to the watcher while a
provider process it had started kept it alive. The watcher still saw it as
diff --git a/console/packages/agent-providers/src/host/launch.test.ts b/console/packages/agent-providers/src/host/launch.test.ts
index aee5a5658..f2555e4bc 100644
--- a/console/packages/agent-providers/src/host/launch.test.ts
+++ b/console/packages/agent-providers/src/host/launch.test.ts
@@ -306,3 +306,48 @@ it('starts the session anyway when its start cannot be recorded', async () => {
).toEqual([]);
warn.mockRestore();
});
+
+it.skipIf(process.platform === 'win32')(
+ 'kills a host that will not stop for its replacement, with the session hosts under it',
+ async () => {
+ // The case that blocked an update: an old watcher waiting forever on a
+ // session host that had hung up, so SIGTERM never finished and every
+ // replacement gave up with "has not stopped".
+ const root = await mkdtemp(join(tmpdir(), 'stop-owned-'));
+ const grandchildPid = join(root, 'grandchild.pid');
+ const stubborn = spawn(
+ process.execPath,
+ [
+ '-e',
+ `process.on('SIGTERM', () => {});
+ const child = require('node:child_process').spawn(process.execPath,
+ ['-e', 'process.on("SIGTERM", () => {}); setInterval(() => {}, 1000)'],
+ { detached: true, stdio: 'ignore' });
+ require('node:fs').writeFileSync(process.argv[1] + '/grandchild.pid', String(child.pid));
+ setInterval(() => {}, 1000);`,
+ // On its command line, which is how the stop confirms the pid is still its host.
+ root,
+ ],
+ { detached: true, stdio: 'ignore' }
+ );
+ try {
+ let pid = 0;
+ for (let attempt = 0; attempt < 100 && !pid; attempt++) {
+ pid = Number(await readFile(grandchildPid, 'utf8').catch(() => '0'));
+ if (!pid) await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ await writeFile(join(root, 'shared-owner.lock'), JSON.stringify({ pid: stubborn.pid }));
+
+ await detachedSupervision('bundle.mjs').stop(root);
+
+ expect(() => process.kill(stubborn.pid!, 0)).toThrow();
+ expect(() => process.kill(pid, 0)).toThrow();
+ } finally {
+ try {
+ process.kill(-stubborn.pid!, 'SIGKILL');
+ } catch {}
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ 40_000
+);
diff --git a/console/packages/agent-providers/src/host/launch.ts b/console/packages/agent-providers/src/host/launch.ts
index 7c667453c..d39bed82a 100644
--- a/console/packages/agent-providers/src/host/launch.ts
+++ b/console/packages/agent-providers/src/host/launch.ts
@@ -6,6 +6,7 @@ import { join } from 'node:path';
import { setTimeout as delay } from 'node:timers/promises';
import { promisify } from 'node:util';
import { replaceOwner, withOwnershipLock } from './ownership-lock';
+import { killProcessTree } from './process-fence';
import type { SessionLinks } from './session-channel';
import { recordSessionStart, type HostStartSource } from './session-start';
import { sharedConfigSchema, type SharedHostConfig } from './shared-config';
@@ -273,14 +274,31 @@ async function stopOwnedProcess(root: string, ownerPath: string): Promise<void>
'The saved PID no longer identifies this SDK host. Refusing to stop another process.'
);
process.kill(pid, 'SIGTERM');
- for (let attempt = 0; attempt < 100; attempt++) {
+ if (await exitsWithin(pid, STOP_GRACE_MS)) return;
+ // Asked, and still here: most likely waiting on a child that will never
+ // finish, such as a session host that hung up and stayed alive. Waiting
+ // longer changes nothing, and refusing leaves every update to this agent
+ // failing the same way, so what is left of the tree is killed.
+ console.warn(
+ `The SDK host at ${root} (pid ${pid}) did not stop within ${STOP_GRACE_MS / 1000} s of being asked; killing it and everything it started.`
+ );
+ await killProcessTree(pid);
+ if (await exitsWithin(pid, 5000)) return;
+ throw new Error('The SDK host has not stopped. Recovery cannot start a competing owner.');
+}
+
+/** How long a host asked to stop is given before it and what it started are killed. */
+export const STOP_GRACE_MS = 20_000;
+
+async function exitsWithin(pid: number, ms: number): Promise<boolean> {
+ for (let waited = 0; ; waited += 200) {
try {
process.kill(pid, 0);
} catch (error) {
- if ((error as NodeJS.ErrnoException).code === 'ESRCH') return;
+ if ((error as NodeJS.ErrnoException).code === 'ESRCH') return true;
throw error;
}
+ if (waited >= ms) return false;
await delay(200);
}
- throw new Error('The SDK host has not stopped. Recovery cannot start a competing owner.');
}
diff --git a/console/packages/agent-providers/src/host/process-fence.ts b/console/packages/agent-providers/src/host/process-fence.ts
index 702062853..455ee0c59 100644
--- a/console/packages/agent-providers/src/host/process-fence.ts
+++ b/console/packages/agent-providers/src/host/process-fence.ts
@@ -43,3 +43,46 @@ export async function fenceDeadOwner(pid: number, group: number | null): Promise
}
throw new Error('FENCING_REQUIRED: the previous provider process group has not exited.');
}
+
+/**
+ * Kill a process and every process descended from it, whatever group each is
+ * in.
+ *
+ * A watcher and each session host it runs are started detached, so each leads
+ * its own process group, and a provider a session host starts sits in that
+ * host's group. Killing one group therefore leaves the rest: kill a watcher's
+ * and its session hosts carry on, still holding their sessions' locks. This
+ * walks the tree instead. For a process that would not stop when asked;
+ * there is no grace here.
+ */
+export async function killProcessTree(pid: number): Promise<void> {
+ if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error('Invalid process to kill.');
+ if (process.platform === 'win32') {
+ try {
+ process.kill(pid, 'SIGKILL');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+ }
+ return;
+ }
+ const { stdout } = await execute('ps', ['-A', '-o', 'pid=,ppid=']);
+ const children = new Map<number, number[]>();
+ for (const line of stdout.split('\n')) {
+ const [child, parent] = line.trim().split(/\s+/).map(Number);
+ if (!child || parent === undefined || Number.isNaN(parent)) continue;
+ children.set(parent, [...(children.get(parent) ?? []), child]);
+ }
+ const doomed: number[] = [];
+ const visit = (next: number) => {
+ doomed.push(next);
+ for (const child of children.get(next) ?? []) visit(child);
+ };
+ visit(pid);
+ for (const target of doomed) {
+ try {
+ process.kill(target, 'SIGKILL');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error;
+ }
+ }
+}
diff --git a/console/packages/agent-providers/src/host/supervisor.test.ts b/console/packages/agent-providers/src/host/supervisor.test.ts
index db193e6eb..2258bfa7e 100644
--- a/console/packages/agent-providers/src/host/supervisor.test.ts
+++ b/console/packages/agent-providers/src/host/supervisor.test.ts
@@ -3,7 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, expect, it } from 'vitest';
import { LEASE_EXPIRED_EXIT_CODE } from './exit-codes';
-import { superviseSharedHost } from './supervisor';
+import { CHILD_STOP_GRACE_MS, superviseSharedHost } from './supervisor';
const roots: string[] = [];
afterEach(async () => {
@@ -182,3 +182,47 @@ it('does not relaunch a worker that exits with a fatal code', async () => {
JSON.parse(await readFile(join(root, 'supervisor', 'failure.json'), 'utf8')).message
).toContain('worker.log');
});
+
+it('kills a worker that will not stop when asked, and everything it started', async () => {
+ // A watcher waiting on a session host that hung up and stayed alive never
+ // exits on SIGTERM, and a replacement waiting for it used to give up.
+ const root = await fixture();
+ const grandchildPid = join(root, 'grandchild.pid');
+ const script = `
+ const fs = require('node:fs');
+ process.on('SIGTERM', () => {});
+ // In a group of its own, as a session host is: killing the worker's
+ // group alone would leave it.
+ const child = require('node:child_process').spawn(process.execPath,
+ ['-e', 'process.on("SIGTERM", () => {}); setInterval(() => {}, 1000)'],
+ { detached: true, stdio: 'ignore' });
+ fs.writeFileSync(${JSON.stringify(grandchildPid)}, String(child.pid));
+ setInterval(() => {}, 1000);
+ `;
+ const abort = new AbortController();
+ const supervising = superviseSharedHost({
+ root,
+ executable: process.execPath,
+ args: ['-e', script],
+ env: process.env,
+ signal: abort.signal,
+ build: 'test-bundle.mjs',
+ links: null,
+ });
+ let pid = 0;
+ for (let attempt = 0; attempt < 100 && !pid; attempt++) {
+ try {
+ pid = Number(await readFile(grandchildPid, 'utf8'));
+ } catch {
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ }
+ }
+ expect(pid).toBeGreaterThan(0);
+
+ const started = Date.now();
+ abort.abort();
+ await supervising;
+
+ expect(Date.now() - started).toBeGreaterThanOrEqual(CHILD_STOP_GRACE_MS - 500);
+ expect(() => process.kill(pid, 0)).toThrow();
+}, 30_000);
diff --git a/console/packages/agent-providers/src/host/supervisor.ts b/console/packages/agent-providers/src/host/supervisor.ts
index 9733df2c3..198f63c8e 100644
--- a/console/packages/agent-providers/src/host/supervisor.ts
+++ b/console/packages/agent-providers/src/host/supervisor.ts
@@ -7,7 +7,7 @@ import { setTimeout as delay } from 'node:timers/promises';
import { z } from 'zod';
import { LEASE_EXPIRED_EXIT_CODE } from './exit-codes';
import { releaseOwner, replaceOwner, withOwnershipLock } from './ownership-lock';
-import { fenceDeadOwner } from './process-fence';
+import { fenceDeadOwner, killProcessTree } from './process-fence';
import type { SessionLinks } from './session-channel';
function alive(pid: number): boolean {
@@ -32,6 +32,13 @@ async function ownerPid(path: string): Promise<number | null> {
}
}
+/**
+ * How long a supervised host asked to stop is given before it and what it
+ * started are killed. Shorter than the launcher's `STOP_GRACE_MS`, so a chain
+ * — a watcher's supervisor, the watcher, its session hosts — comes down inside
+ * the time a replacement waits for it.
+ */
+export const CHILD_STOP_GRACE_MS = 10_000;
export async function superviseSharedHost(input: {
root: string;
executable: string;
@@ -75,7 +82,24 @@ export async function superviseSharedHost(input: {
input.links?.attach(input.root, child);
const exited = once(child, 'exit');
await log.close();
- const stop = () => child.kill('SIGTERM');
+ // Asked first. A host that has not gone `CHILD_STOP_GRACE_MS` later is
+ // waiting on something that will not finish — a session host that hung
+ // up and stayed alive, say — and is killed with everything it started,
+ // so that stopping this supervisor cannot hang on it.
+ let escalation: ReturnType<typeof setTimeout> | null = null;
+ const stop = () => {
+ child.kill('SIGTERM');
+ escalation ??= setTimeout(() => {
+ if (child.exitCode !== null || child.signalCode !== null || !child.pid) return;
+ console.warn(
+ `The shared host at ${input.root} did not stop within ${CHILD_STOP_GRACE_MS / 1000} s of being asked; killing it and everything it started.`
+ );
+ killProcessTree(child.pid).catch((error: unknown) =>
+ console.error(`Could not kill the shared host at ${input.root}: ${String(error)}`)
+ );
+ }, CHILD_STOP_GRACE_MS);
+ escalation.unref?.();
+ };
input.signal.addEventListener('abort', stop, { once: true });
if (input.signal.aborted) stop();
let code: number | null;
@@ -84,6 +108,7 @@ export async function superviseSharedHost(input: {
[code, signal] = await exited;
} finally {
input.signal.removeEventListener('abort', stop);
+ if (escalation) clearTimeout(escalation);
}
if (child.pid) {
await fenceDeadOwner(child.pid, child.pid);
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
index eabecc239..fbc6ea524 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -1426,3 +1426,95 @@ it('states placements on the attached incarnation, and raises on a refusal', asy
abort.abort();
}
});
+
+it('does not give its connection up to itself over a request answered across its own reopen', async () => {
+ // Seen on a real host: placements stated under incarnation 4, the stream
+ // reopened (to 5) before the answer came, and the answer — "taken over:
+ // reopened since 4, now at 5" — named this very client as the new holder.
+ // It stood down for good, with nobody else anywhere near the connection.
+ const opened: ReadableStreamDefaultController<Uint8Array>[] = [];
+ let answerPlacements: (response: Response) => void = () => {};
+ const fetchMock = vi.fn(async (url: string) => {
+ if (url.includes('/events')) {
+ const generation = 4 + opened.length;
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ opened.push(controller);
+ controller.enqueue(connected(generation));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ }
+ if (url.includes('connection/placements'))
+ return new Promise<Response>((resolve) => {
+ answerPlacements = resolve;
+ });
+ return Response.json({});
+ });
+ const evicted: Eviction[] = [];
+ const onConnected = vi.fn();
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: [],
+ scope: 'all',
+ onConnected,
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ try {
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(1));
+ const stating = stream.replacePlacements({ session: 'room' });
+ await vi.waitFor(() => expect(urlsFor(fetchMock, 'connection/placements')).toHaveLength(1));
+
+ opened[0]!.close();
+ await vi.waitFor(() => expect(onConnected).toHaveBeenCalledTimes(2), { timeout: 3000 });
+ answerPlacements(
+ Response.json(
+ {
+ detail: {
+ code: 'taken_over',
+ message: 'connection conn-1 has been reopened since incarnation 4 and is now at 5',
+ },
+ },
+ { status: 409 }
+ )
+ );
+
+ await expect(stating).rejects.toThrow('409');
+ expect(evicted).toEqual([]);
+ } finally {
+ abort.abort();
+ }
+});
+
+it('still stands down when a request is refused as taken over on the incarnation it holds', async () => {
+ const fetchMock = vi.fn(async (url: string) => {
+ if (url.includes('/events'))
+ return new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(connected(4));
+ },
+ }),
+ { headers: { 'content-type': 'text/event-stream' } }
+ );
+ if (url.includes('connection/placements'))
+ return Response.json(
+ { detail: { code: 'taken_over', message: 'another client holds it' } },
+ { status: 409 }
+ );
+ return Response.json({});
+ });
+ const evicted: Eviction[] = [];
+ const { stream, abort } = makeStream(fetchMock, {
+ rooms: [],
+ scope: 'all',
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+ try {
+ await expect(stream.replacePlacements({ session: 'room' })).rejects.toThrow('409');
+ expect(evicted.map((eviction) => eviction.code)).toEqual([EVICTION_TAKEN_OVER]);
+ } finally {
+ abort.abort();
+ }
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index 2303aea1f..d25e01cfc 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -474,6 +474,25 @@ export class SwitchEventStream {
});
}
+ /**
+ * Whether a takeover refusal for a request sent under `sent` describes a
+ * reopen of our own.
+ *
+ * A request carries the incarnation it was sent under, and the server
+ * refuses it as taken over once the connection has moved past that. But
+ * this client reopens its own stream — after an eviction, a refused beat, a
+ * repoint — and a request in flight across that reopen is refused the same
+ * way, naming as the new holder the incarnation this client now is. Standing
+ * down on that gives the connection up to ourselves, for good. Heartbeats
+ * already discard such answers through the fence; this is the same test for
+ * the requests that do not go through it. A real takeover after our reopen
+ * is still caught: the next beat, under the incarnation we now hold, is
+ * refused, and stands down.
+ */
+ private supersededOurselves(sent: number | null): boolean {
+ return sent !== null && this.generation !== null && this.generation !== sent;
+ }
+
/**
* Give the connection up to the client that now holds it.
*
@@ -504,6 +523,7 @@ export class SwitchEventStream {
// the frame, so waiting costs such a client nothing.
await Promise.race([this.fence.reached, until(this.deps.signal, this.halt.signal)]);
if (this.halt.signal.aborted) return;
+ const generation = this.generation;
const resp = await this.post('connection/subscribe', {
connection_id: this.deps.connectionId,
room_id: roomId,
@@ -513,11 +533,15 @@ export class SwitchEventStream {
// displaced would rewrite the winner's rooms — and evict whoever holds
// the room it asks for — and being refused the open afterwards would
// come too late to undo any of it.
- generation: this.generation,
+ generation,
});
if (!resp.ok) {
const body = await resp.text();
if (resp.status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
+ if (this.supersededOurselves(generation))
+ throw new Error(
+ `subscribe to ${roomId} was answered for an incarnation this connection has since reopened past; ask again`
+ );
// Not this connection's client any more. Terminal, like every other
// door onto a takeover: there is nothing to repoint.
this.standDown();
@@ -551,14 +575,20 @@ export class SwitchEventStream {
]).finally(() => clearTimeout(timer));
if (!attached || this.halt.signal.aborted)
throw new Error('the connection to Switch is not open, so it cannot take placements');
+ const generation = this.generation;
const resp = await this.post('connection/placements', {
connection_id: this.deps.connectionId,
placements,
- generation: this.generation,
+ generation,
});
if (resp.ok) return;
const body = await resp.text();
- if (resp.status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) this.standDown();
+ if (
+ resp.status === 409 &&
+ refusalCode(body) === EVICTION_TAKEN_OVER &&
+ !this.supersededOurselves(generation)
+ )
+ this.standDown();
throw new PlacementsRefusedError(resp.status, body.slice(0, 500));
}