Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(console): read remote watcher status from the host, so it cannot go stale (#608)
Louis Amaudruz0b556cd99814
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8b210c6ab..e1299949f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1369,6 +1369,15 @@ version of their own to them without also giving them a release of their own.bundle is uploaded only when an agent is started, restarted or updated.#### Fixed+- **A remote agent's status can no longer go stale.** Console used to learn+ whether a remote agent was connected only from what its sidecar pushed down a+ long-lived connection, and trusted that connection until it said it had+ closed. One that died without saying so, as when every sidecar is replaced+ while its host's SSH connection is rebuilt, left the last thing it carried+ on screen for good: agents that were answering in Slack showed "Connection+ failed" and "The agent's room watcher is not running". Console now reads each+ host every five seconds, in one command for every agent on it, and the+ sidebar and the agent's page are both built from that one read.- **Agents come up after Console starts without their page being opened.**Console now brings each host's agents up side by side, so a slow or wedgedSSH connection holds back only that host, never local agents or other hosts,@@ -3149,6 +3158,11 @@ published on its own.### [Unreleased]+#### Added+- **The room watcher records its connection to Switch in `health.json`** beside+ its other state, each time it changes, so Console can read it from the host+ instead of relying on a live connection to the sidecar.+#### Fixed- **A session whose host hung up no longer blocks its room for good.** After areset, a session host could finish and close its link to the watcher while adiff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.test.tsindex 3fa1585df..3463c89fb 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.test.ts@@ -5,8 +5,8 @@ import {type ConnectionHealthDeps,ConnectionHealthMonitor,type LinkedAgent,- type RemoteHealthSource,} from './connection-health-monitor';+import type { HostWatcherStatus } from './host-watchers';const NOW = Date.parse('2026-09-24T12:00:00.000Z');@@ -38,31 +38,26 @@ function localWatcher(initial: WatcherHealth) {};}-/** A sidecar's control connection, as the monitor uses it. */-function sidecar(initial: WatcherHealth) {- let current = initial;- const listeners = new Set<(health: WatcherHealth) => void>();- const closers = new Set<(error: Error) => void>();- const client: RemoteHealthSource = {- health: async () => current,- onHealth: async (listener) => {- listeners.add(listener);- return () => listeners.delete(listener);- },- onClose: (listener) => {- closers.add(listener);- return () => closers.delete(listener);- },- };+/** A remote watcher as its host's files describe it, with what it wrote about itself. */+function onHost(+ written: WatcherHealth | null,+ overrides: Partial<HostWatcherStatus> = {}+): HostWatcherStatus {return {- client,- report(next: WatcherHealth) {- current = next;- for (const listener of listeners) listener(next);- },- close(error: Error) {- for (const closer of closers) closer(error);- },+ agentId: 'switch-remote',+ root: '/state/sdk-watchers/remote',+ running: true,+ build: '/state/sdk-host/shared-host-abc.mjs',+ enabled: true,+ spawn: true,+ stoodDown: false,+ supervisorPid: 100,+ workerPid: 101,+ workerAlive: true,+ health: written && { ...written, pid: 101, updatedAt: new Date(NOW).toISOString() },+ failure: null,+ takenOver: null,+ ...overrides,};}@@ -82,13 +77,12 @@ function monitorWith(overrides: Partial<ConnectionHealthDeps>) {local: () => {throw new Error('no local watcher in this test');},- remote: () => Promise.reject(new Error('no sidecar in this test')),- remoteStatus: async () => null,+ remoteWatcher: () => Promise.reject(new Error('no remote agent in this test')),emit: (_serverId, snapshot) => pushed.push(snapshot),redact: (text) => text.replaceAll('secret', '[redacted]'),logError: vi.fn(),now: () => NOW,- retryMs: 10_000,+ pollMs: 5_000,...overrides,};const monitor = new ConnectionHealthMonitor(deps);@@ -153,75 +147,131 @@ it('shows a stream that just dropped as connecting, then failed once the grace h]);});-it('reads a remote agent through its sidecar and follows what the sidecar pushes', async () => {- const remote = sidecar(health({ state: 'connecting', since: new Date(NOW).toISOString() }));+it('reads a remote agent from what its watcher wrote on its host, and reads it again', async () => {+ vi.useFakeTimers();+ const host = vi+ .fn<(agentId: string) => Promise<HostWatcherStatus | null>>()+ .mockResolvedValueOnce(+ onHost(health({ state: 'connecting', since: new Date(NOW).toISOString() }))+ )+ .mockResolvedValue(onHost(health({ placements: { 'session-9': 'room-9' } })));const { monitor, pushed } = monitorWith({linkedAgents: async () => [agent('remote')],isRemote: async () => true,- remote: async (agentId) => {- expect(agentId).toBe('remote');- return remote.client;- },+ remoteWatcher: host,});expect((await monitor.snapshot('server')).agents).toEqual([{ agentId: 'remote', state: 'connecting', detail: null },]);- remote.report(health({ placements: { 'session-9': 'room-9' } }));- await vi.waitFor(() =>- expect(pushed.at(-1)).toEqual({- agents: [{ agentId: 'remote', state: 'connected', detail: null }],- placements: { 'session-9': 'room-9' },- })- );+ expect(host).toHaveBeenCalledWith('remote');++ await vi.advanceTimersByTimeAsync(5_000);+ expect(pushed.at(-1)).toEqual({+ agents: [{ agentId: 'remote', state: 'connected', detail: null }],+ placements: { 'session-9': 'room-9' },+ });+});++it('never keeps showing what a replaced watcher last said', async () => {+ // The bug this replaces: a push connection to a sidecar that was swapped+ // out died without saying so, and its last words — "not running", as it+ // stopped — stayed on screen for good.+ vi.useFakeTimers();+ const host = vi+ .fn<(agentId: string) => Promise<HostWatcherStatus | null>>()+ .mockResolvedValueOnce(+ onHost(health({ state: 'not-running', since: new Date(NOW - 60_000).toISOString() }), {+ workerAlive: false,+ failure: null,+ })+ )+ .mockResolvedValue(+ onHost(null, {+ workerPid: 202,+ health: { ...health({}), pid: 202, updatedAt: new Date(NOW).toISOString() },+ })+ );+ const { monitor, pushed } = monitorWith({+ linkedAgents: async () => [agent('remote')],+ isRemote: async () => true,+ remoteWatcher: host,+ });+ await monitor.snapshot('server');+ await vi.advanceTimersByTimeAsync(5_000);+ expect(pushed.at(-1)!.agents).toEqual([{ agentId: 'remote', state: 'connected', detail: null }]);});-it('says a sidecar cannot be reached, and why, and connects again once it can', async () => {+it('does not take a file a previous watcher left as this watcher’s word', async () => {vi.useFakeTimers();- const remote = sidecar(health({}));- const reach = vi- .fn<(agentId: string) => Promise<RemoteHealthSource>>()- .mockRejectedValueOnce(new Error("The agent's sidecar is not running on its host."))- .mockResolvedValue(remote.client);+ let now = NOW;+ // Written by pid 101; the watcher alive now is 202 and has written nothing.+ const stale = onHost(health({}), { workerPid: 202 });const { monitor, pushed } = monitorWith({linkedAgents: async () => [agent('remote')],isRemote: async () => true,- remote: reach,- remoteStatus: async () => ({ takenOver: null, failure: 'out of memory' }),+ remoteWatcher: async () => stale,+ now: () => now,});expect((await monitor.snapshot('server')).agents).toEqual([+ { agentId: 'remote', state: 'connecting', detail: null },+ ]);+ now = NOW + 20_000;+ await vi.advanceTimersByTimeAsync(20_000);+ expect(pushed.at(-1)!.agents).toEqual([{agentId: 'remote',- state: 'unreachable',+ state: 'failed',detail:- "The agent's sidecar is not running on its host. The sidecar last stopped with: out of memory",+ "The agent's room watcher is running but does not report its connection to Switch. Update its sidecar to this Console's build.",},]);+});- await vi.advanceTimersByTimeAsync(10_000);- expect(pushed.at(-1)!.agents).toEqual([{ agentId: 'remote', state: 'connected', detail: null }]);-- // The connection drops: unreachable again, not a made-up "disconnected".- remote.close(new Error('The connection to the agent sidecar closed.'));- await vi.advanceTimersByTimeAsync(0);- expect(pushed.at(-1)!.agents).toMatchObject([- { agentId: 'remote', state: 'unreachable', detail: expect.stringContaining('closed') },- ]);- // Asking again (a refetch) tries the sidecar at once rather than waiting.+it('says the host cannot be read, and reads it again on the next round', async () => {+ vi.useFakeTimers();+ const host = vi+ .fn<(agentId: string) => Promise<HostWatcherStatus | null>>()+ .mockRejectedValueOnce(new Error('SSH exec channel open timed out after 15000ms'))+ .mockResolvedValue(onHost(health({})));+ const { monitor, pushed } = monitorWith({+ linkedAgents: async () => [agent('remote')],+ isRemote: async () => true,+ remoteWatcher: host,+ });expect((await monitor.snapshot('server')).agents).toEqual([- { agentId: 'remote', state: 'connected', detail: null },+ {+ agentId: 'remote',+ state: 'unreachable',+ detail:+ "Could not read the agent's state on its host: SSH exec channel open timed out after 15000ms",+ },]);- expect(reach).toHaveBeenCalledTimes(3);+ await vi.advanceTimersByTimeAsync(5_000);+ expect(pushed.at(-1)!.agents).toEqual([{ agentId: 'remote', state: 'connected', detail: null }]);});-it('names a takeover the unreachable sidecar stood down for', async () => {+it('names a takeover, the failure a stopped watcher recorded, and a host with no watcher', async () => {+ const statuses: Record<string, HostWatcherStatus | null> = {+ displaced: onHost(null, {+ stoodDown: true,+ takenOver: { at: new Date(NOW).toISOString(), reason: 'another client' },+ }),+ crashed: onHost(null, { workerAlive: false, running: false, failure: 'out of memory' }),+ missing: null,+ };const { monitor } = monitorWith({- linkedAgents: async () => [agent('remote')],+ linkedAgents: async () => [agent('displaced'), agent('crashed'), agent('missing')],isRemote: async () => true,- remote: () => Promise.reject(new Error('not running')),- remoteStatus: async () => ({ takenOver: { reason: 'another client' }, failure: null }),+ remoteWatcher: async (agentId) => statuses[agentId]!,});expect((await monitor.snapshot('server')).agents).toEqual([- { agentId: 'remote', state: 'taken-over', detail: 'another client' },+ { agentId: 'displaced', state: 'taken-over', detail: 'another client' },+ { agentId: 'crashed', state: 'failed', detail: 'out of memory' },+ {+ agentId: 'missing',+ state: 'failed',+ detail: 'No room watcher has been set up for this agent on its host.',+ },]);});diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.tsindex 2c631be01..de450adef 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health-monitor.ts@@ -1,10 +1,12 @@-import type { WatcherHealth } from '@switch-console/agent-providers';+import { type WatcherHealth, watcherHealthFileSchema } from '@switch-console/agent-providers';import {+ CONNECTION_GRACE_MS,type AgentConnectionHealth,classifyWatcher,type RoomHealthSnapshot,type WatcherReport,} from '@shared/core/switch-rooms/connection-health';+import type { HostWatcherStatus } from './host-watchers';/** A local agent's room watcher, running inside Console. */export type LocalHealthSource = {@@ -12,13 +14,6 @@ export type LocalHealthSource = {onHealth(listener: (health: WatcherHealth) => void): () => void;};-/** A remote agent's room watcher, reached through its sidecar's control port. */-export type RemoteHealthSource = {- health(): Promise<WatcherHealth>;- onHealth(listener: (health: WatcherHealth) => void): Promise<() => void>;- onClose(listener: (error: Error) => void): () => void;-};-export type LinkedAgent = {id: string;serverId: string;@@ -34,20 +29,18 @@ export type ConnectionHealthDeps = {/** Agents whose room connection a person stopped from Console. */stoppedAgentIds: () => Promise<string[]>;local: (switchAgentId: string) => LocalHealthSource;- remote: (agentId: string) => Promise<RemoteHealthSource>;/**- * What the sidecar's files on its host say about why it is not answering:- * a takeover it stood down for, or the failure it recorded.+ * What a remote agent's watcher is doing, from its host's files: one read+ * shared by every agent on the host. Null when the host has no watcher for+ * the agent; raises when the host cannot be read.*/- remoteStatus: (- agentId: string- ) => Promise<{ takenOver: { reason: string } | null; failure: string | null } | null>;+ remoteWatcher: (agentId: string) => Promise<HostWatcherStatus | null>;emit: (serverId: string, snapshot: RoomHealthSnapshot) => void;redact: (text: string) => string;logError: (message: string, context: Record<string, unknown>) => void;now: () => number;- /** How long after failing to reach a sidecar it is tried again. */- retryMs: number;+ /** How often a remote agent's host is read again. */+ pollMs: number;};type Source = {@@ -71,10 +64,11 @@ const message = (error: unknown) => (error instanceof Error ? error.message : St* Each linked agent's room connection, from the agent's own room watcher.** The watcher holds the agent's one connection to Switch and decides which- * session attends which room, so it is asked directly: in-process for a local- * agent, through the sidecar's control port for a remote one. Every change it- * reports is pushed on as the server's whole snapshot. A sidecar that cannot- * be reached is reported as such, and tried again every `retryMs`.+ * session attends which room, so its own account is what is shown: in-process+ * for a local agent, and for a remote one from the state it writes on its+ * host, read every `pollMs` with the rest of that host's watchers. Every+ * change is pushed on as the server's whole snapshot. A host that cannot be+ * read is reported as such, and read again on the next round.*/export class ConnectionHealthMonitor {private readonly entries = new Map<string, Entry>();@@ -153,50 +147,45 @@ export class ConnectionHealthMonitor {return source;}+ /**+ * A remote agent's watcher, read from its host every `pollMs`.+ *+ * Asked rather than listened to: a connection to the sidecar that dies+ * without saying so — across an SSH reconnect, say — would otherwise leave+ * the last thing it carried on screen for good. A read that fails says so,+ * and the next one tries again.+ */private remoteSource(agent: LinkedAgent): Source {let disposed = false;- let cleanup: (() => void)[] = [];let timer: ReturnType<typeof setTimeout> | null = null;- const release = () => {- for (const stop of cleanup) stop();- cleanup = [];+ let silentSince: number | null = null;+ const poll = async () => {if (timer) clearTimeout(timer);timer = null;- };- const lost = async (error: unknown) => {- release();- if (disposed) return;- let detail = message(error);- let takenOver: string | null = null;- try {- const status = await this.deps.remoteStatus(agent.id);- takenOver = status?.takenOver?.reason ?? null;- if (status?.failure) detail = `${detail} The sidecar last stopped with: ${status.failure}`;- } catch (statusError) {- detail = `${detail} Its state on the host could not be read either: ${message(statusError)}`;- }- if (disposed) return;- source.health = null;- source.unreachable = { detail, takenOver };- timer = setTimeout(() => void connect(), this.deps.retryMs);- this.changed(agent.serverId);- };- const connect = async () => {- release();if (disposed) return;try {- const client = await this.deps.remote(agent.id);- const update = (health: WatcherHealth) => {- if (disposed) return;- source.health = health;+ const status = await this.deps.remoteWatcher(agent.id);+ if (disposed) return;+ const read = healthFromHost(status, silentSince, this.deps.now());+ silentSince = read.silentSince;+ if (source.unreachable || !sameHealth(source.health, read.health)) {+ source.health = read.health;source.unreachable = null;this.changed(agent.serverId);- };- cleanup.push(await client.onHealth(update));- cleanup.push(client.onClose((error) => void lost(error)));- update(await client.health());+ }} catch (error) {- await lost(error);+ if (disposed) return;+ const detail = `Could not read the agent's state on its host: ${message(error)}`;+ if (source.health || source.unreachable?.detail !== detail) {+ source.health = null;+ source.unreachable = { detail, takenOver: null };+ this.changed(agent.serverId);+ }+ } finally {+ if (!disposed) {+ timer = setTimeout(() => void poll(), this.deps.pollMs);+ timer.unref?.();+ }}};const source: Source = {@@ -206,15 +195,15 @@ export class ConnectionHealthMonitor {unreachable: null,ready: Promise.resolve(),retry: () => {- source.ready = connect();+ source.ready = poll();return source.ready;},dispose: () => {disposed = true;- release();+ if (timer) clearTimeout(timer);},};- source.ready = connect();+ source.ready = poll();return source;}@@ -280,3 +269,75 @@ export class ConnectionHealthMonitor {return { agents, placements };}}++const sameHealth = (a: WatcherHealth | null, b: WatcherHealth): boolean =>+ a !== null && JSON.stringify(a) === JSON.stringify(b);++/**+ * A remote watcher's health, from what its host's files say.+ *+ * The watcher writes its own connection state beside its other files; that is+ * taken only from the process alive now, so a file a previous watcher left+ * does not speak for this one. A watcher that is alive but has written+ * nothing — just started, or a sidecar from before it wrote the file — is+ * given the usual grace from when it was first seen silent, and then named as+ * silent rather than guessed at. `silentSince` is carried from one read to the+ * next by the caller.+ */+export function healthFromHost(+ status: HostWatcherStatus | null,+ silentSince: number | null,+ now: number+): { health: WatcherHealth; silentSince: number | null } {+ const at = (since: number) => new Date(since).toISOString();+ if (!status)+ return {+ health: {+ state: 'not-running',+ detail: 'No room watcher has been set up for this agent on its host.',+ since: at(0),+ placements: {},+ },+ silentSince: null,+ };+ if (status.takenOver || status.stoodDown)+ return {+ health: {+ state: 'taken-over',+ detail: status.takenOver?.reason ?? "Another client holds this agent's connection.",+ since: status.takenOver?.at || at(0),+ placements: {},+ },+ silentSince: null,+ };+ const written = watcherHealthFileSchema.safeParse(status.health);+ if (status.workerAlive && written.success && written.data.pid === status.workerPid) {+ const { pid: _pid, updatedAt: _updatedAt, ...health } = written.data;+ return { health, silentSince: null };+ }+ const since = silentSince ?? now;+ if (!status.workerAlive)+ return {+ health: {+ state: 'not-running',+ // A recorded failure is the reason; with none, it may just be+ // restarting, and the grace below decides.+ detail: status.failure,+ since: at(since),+ placements: {},+ },+ silentSince: since,+ };+ return {+ health: {+ state: 'not-running',+ detail:+ now - since < CONNECTION_GRACE_MS+ ? null+ : "The agent's room watcher is running but does not report its connection to Switch. Update its sidecar to this Console's build.",+ since: at(since),+ placements: {},+ },+ silentSince: since,+ };+}diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health.tsindex 838023f74..b1181c476 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/connection-health.ts@@ -7,12 +7,11 @@ import { redactSecrets } from '@main/lib/file-logger';import { log } from '@main/lib/logger';import { roomHealthChangedChannel } from '@shared/core/switch-rooms/switchRoomEvents';import { ConnectionHealthMonitor } from './connection-health-monitor';-import { remoteWatcherStatus } from './diagnostics';+import { hostWatcherStatus } from './host-watcher-snapshot';import { localWatcherControl } from './local-host';-import { sidecarControl } from './sidecar-control';-/** How often a sidecar that could not be reached is tried again. */-const SIDECAR_RETRY_MS = 10_000;+/** How often each remote agent's host is read for its watchers' state. */+const REMOTE_POLL_MS = 5_000;const monitor = new ConnectionHealthMonitor({linkedAgents: async (serverId) => {@@ -33,13 +32,12 @@ const monitor = new ConnectionHealthMonitor({isRemote: async (agent) => !!(await getAgentLocation(agent)).sshHost,stoppedAgentIds: listStoppedControllerAgentIds,local: localWatcherControl,- remote: sidecarControl,- remoteStatus: remoteWatcherStatus,+ remoteWatcher: hostWatcherStatus,emit: (serverId, snapshot) => events.emit(roomHealthChangedChannel, snapshot, serverId),redact: redactSecrets,logError: (message, context) => log.error(message, context),now: () => Date.now(),- retryMs: SIDECAR_RETRY_MS,+ pollMs: REMOTE_POLL_MS,});/**diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.test.tsindex 8965944a7..4cbaa2496 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.test.ts@@ -1,5 +1,5 @@import { beforeEach, expect, it, vi } from 'vitest';-const mocks = vi.hoisted(() => ({ exec: vi.fn(), sessions: vi.fn() }));+const mocks = vi.hoisted(() => ({ exec: vi.fn(), sessions: vi.fn(), watcher: vi.fn() }));vi.mock('electron', () => ({ app: {} }));vi.mock('@main/core/agent-runtime/impl/resolve-sidecar-bundle', () => ({resolveSharedHostBundlePath: () => import.meta.filename,@@ -20,11 +20,13 @@ vi.mock('@main/core/switch-servers/servers-store', () => ({getServer: async () => ({ id: 'server' }),}));vi.mock('./host-sessions', () => ({ listHostSessions: mocks.sessions }));+vi.mock('./host-watcher-snapshot', () => ({ hostWatcherStatus: mocks.watcher }));const { sharedAgentDiagnostics, sharedAgentLogs } = await import('./diagnostics');beforeEach(() => {vi.resetAllMocks();mocks.exec.mockResolvedValue({ stdout: '[]' });mocks.sessions.mockResolvedValue([]);+ mocks.watcher.mockResolvedValue(null);});it('keeps host management available while explicitly reporting a session-list failure', async () => {mocks.sessions.mockRejectedValue(new Error('Server unreachable'));@@ -48,25 +50,36 @@ it('redacts secrets from logs before returning them to the renderer', async () =expect(log).toContain('Watcher started');expect(mocks.exec.mock.calls[0][1].slice(-2)).toEqual(['remote-agent', 'logs']);});-it('redacts watcher failures and propagates SSH failures', async () => {- mocks.exec.mockResolvedValueOnce({- stdout: JSON.stringify([- {- running: false,- enabled: true,- pid: null,- supervisorPid: null,- buildHash: null,- failure: 'token=example-placeholder',- takenOver: { at: '2026-01-01T00:00:00.000Z', reason: 'token=example-placeholder' },- },- ]),+it('reads a remote watcher from its host’s snapshot, redacted, and propagates SSH failures', async () => {+ mocks.watcher.mockResolvedValueOnce({+ agentId: 'remote-agent',+ root: '/state/sdk-watchers/remote',+ running: false,+ build: `/state/sdk-host/shared-host-${'a'.repeat(64)}.mjs`,+ enabled: true,+ spawn: true,+ stoodDown: true,+ supervisorPid: 100,+ workerPid: 101,+ workerAlive: false,+ health: null,+ failure: 'token=example-placeholder',+ takenOver: { at: '2026-01-01T00:00:00.000Z', reason: 'token=example-placeholder' },});const [watcher] = (await sharedAgentDiagnostics('agent')).watchers;+ expect(watcher).toMatchObject({+ running: false,+ enabled: true,+ pid: null,+ supervisorPid: null,+ buildHash: 'a'.repeat(64),+ });expect(watcher.failure).toContain('[REDACTED]');// Whatever the server said when it evicted us reaches the panel too, so it// goes through the same redaction as any other reported text.expect(watcher.takenOver?.reason).toContain('[REDACTED]');- mocks.exec.mockRejectedValueOnce(new Error('SSH unavailable'));+ // The panel no longer asks the host separately: it shares the snapshot.+ expect(mocks.exec).not.toHaveBeenCalled();+ mocks.watcher.mockRejectedValueOnce(new Error('SSH unavailable'));await expect(sharedAgentDiagnostics('agent')).rejects.toThrow('SSH unavailable');});diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.tsindex fec2b7801..b2909b602 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/diagnostics.ts@@ -9,6 +9,7 @@ import { LocalExecutionContext } from '@main/core/execution-context/local-executimport { getServer } from '@main/core/switch-servers/servers-store';import { redactSecrets } from '@main/lib/file-logger';import { listHostSessions } from './host-sessions';+import { hostWatcherStatus } from './host-watcher-snapshot';import { inspectWatchers } from './watcher-inspection';const watcherSchema = z.object({@@ -37,12 +38,36 @@ async function agentHost(agentId: string) {return { agent, location, ctx };}+/**+ * A remote watcher as the panel shows it, from the same read of its host the+ * connection state comes from, so the two cannot disagree.+ */+async function remoteWatchers(agentId: string): Promise<z.infer<typeof watcherSchema>[]> {+ const status = await hostWatcherStatus(agentId);+ if (!status) return [];+ return [+ {+ running: status.workerAlive,+ enabled: status.enabled ?? false,+ failure: status.workerAlive ? null : status.failure,+ pid: status.workerAlive ? status.workerPid : null,+ supervisorPid: status.running ? status.supervisorPid : null,+ buildHash: status.build?.match(/shared-host-([a-f0-9]{64})\.mjs$/)?.[1] ?? null,+ takenOver: status.takenOver,+ },+ ];+}+export async function sharedAgentDiagnostics(agentId: string) {const { agent, location, ctx } = await agentHost(agentId);const server = await getServer(agent.serverId!);if (!server) throw new Error('The agent’s Switch server is missing.');- const [host, bundle, remote] = await Promise.all([- ctx.exec('node', ['-e', inspectWatchers, agent.switchAgentId!, 'status']),+ const [watchers, bundle, remote] = await Promise.all([+ location.sshHost+ ? remoteWatchers(agentId)+ : ctx+ .exec('node', ['-e', inspectWatchers, agent.switchAgentId!, 'status'])+ .then((host) => watcherSchema.array().parse(JSON.parse(host.stdout))),readFile(resolveSharedHostBundlePath()),listHostSessions(agentId).then((sessions) => ({ sessions, error: null }),@@ -53,16 +78,13 @@ export async function sharedAgentDiagnostics(agentId: string) {workingDir: location.dir,transport: location.sshHost ? ('ssh' as const) : ('local' as const),availableBuildHash: createHash('sha256').update(bundle).digest('hex'),- watchers: watcherSchema- .array()- .parse(JSON.parse(host.stdout))- .map((watcher) => ({- ...watcher,- failure: watcher.failure ? redactSecrets(watcher.failure) : null,- takenOver: watcher.takenOver- ? { ...watcher.takenOver, reason: redactSecrets(watcher.takenOver.reason) }- : null,- })),+ watchers: watchers.map((watcher) => ({+ ...watcher,+ failure: watcher.failure ? redactSecrets(watcher.failure) : null,+ takenOver: watcher.takenOver+ ? { ...watcher.takenOver, reason: redactSecrets(watcher.takenOver.reason) }+ : null,+ })),sessions: remote.sessions?.filter((session) => session.agentId === agent.switchAgentId) ?? null,sessionError: remote.error,};@@ -73,13 +95,3 @@ export async function sharedAgentLogs(agentId: string): Promise<string> {const result = await ctx.exec('node', ['-e', inspectWatchers, agent.switchAgentId!, 'logs']);return redactSecrets(z.string().parse(JSON.parse(result.stdout)));}--/**- * The watcher's state as its files on the host record it: why a sidecar that- * cannot be reached stopped, or that it stood down for another client.- */-export async function remoteWatcherStatus(agentId: string) {- const { agent, ctx } = await agentHost(agentId);- const host = await ctx.exec('node', ['-e', inspectWatchers, agent.switchAgentId!, 'status']);- return watcherSchema.array().parse(JSON.parse(host.stdout))[0] ?? null;-}diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watcher-snapshot.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watcher-snapshot.tsnew file mode 100644index 000000000..e46321fbe--- /dev/null+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watcher-snapshot.ts@@ -0,0 +1,51 @@+import { getRemoteAgentLocation } from '@main/core/agents/agent-location';+import { connectRemoteAgent } from '@main/core/agents/connect-remote-agent';+import { getAgentById } from '@main/core/agents/getAgentById';+import { type HostWatcherStatus, listHostWatchers } from './host-watchers';++/**+ * How long one host's watcher state is reused. Shorter than the poll that+ * reads it, so each round reads the host afresh, and long enough that the+ * status panel and the sidebar asking in the same moment share one read.+ */+export const HOST_WATCHERS_TTL_MS = 3000;++const hostReads = new Map<+ string,+ { at: number; watchers: Promise<Map<string, HostWatcherStatus>> }+>();++/** Drop every cached read. For tests. */+export function clearHostWatcherSnapshots(): void {+ hostReads.clear();+}++/**+ * What a remote agent's watcher is doing, from one read of its whole host.+ *+ * Everything Console shows about a remote watcher comes from here — whether+ * the process is alive, which build, what it recorded when it stopped, and+ * its connection to Switch as it wrote it itself — so the sidebar and the+ * agent's page cannot disagree, and nothing depends on a connection to the+ * sidecar staying up. Callers on the same host share the read in flight.+ *+ * Null when the host has no watcher for this agent. A read that fails is not+ * cached, so the next caller tries again.+ */+export async function hostWatcherStatus(agentId: string): Promise<HostWatcherStatus | null> {+ const agent = await getAgentById(agentId);+ if (!agent?.switchAgentId) throw new Error('This agent is not linked to Switch.');+ const location = await getRemoteAgentLocation(agent);+ if (!location) throw new Error('This agent does not run on an SSH host.');+ const hostKey = location.sshHost;+ let cached = hostReads.get(hostKey);+ if (!cached || Date.now() - cached.at >= HOST_WATCHERS_TTL_MS) {+ const watchers = connectRemoteAgent(agent).then(({ ctx }) => listHostWatchers(ctx));+ watchers.catch(() => {+ if (hostReads.get(hostKey)?.watchers === watchers) hostReads.delete(hostKey);+ });+ cached = { at: Date.now(), watchers };+ hostReads.set(hostKey, cached);+ }+ return (await cached.watchers).get(agent.switchAgentId) ?? null;+}diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.test.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.test.tsindex 0ae2f8195..35824b3ba 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.test.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.test.ts@@ -12,8 +12,18 @@* most of what is asserted here.*/+import { execFile } from 'node:child_process';+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { promisify } from 'node:util';import { expect, it, vi } from 'vitest';-import { listHostWatchers, watcherIsCurrent, type HostWatcherStatus } from './host-watchers';+import {+ listHostWatchers,+ watcherIsCurrent,+ WATCHER_STATUS_SCRIPT,+ type HostWatcherStatus,+} from './host-watchers';const ENTRYPOINT = '/home/u/.local/state/switch/sdk-host/shared-host-abc123.mjs';const BUNDLE_FILE = 'shared-host-abc123.mjs';@@ -28,6 +38,12 @@ function status(overrides: Partial<HostWatcherStatus> = {}): HostWatcherStatus {enabled: true,spawn: true,stoodDown: false,+ supervisorPid: 100,+ workerPid: 101,+ workerAlive: true,+ health: null,+ failure: null,+ takenOver: null,...overrides,};}@@ -58,24 +74,15 @@ it('does not skip when the flags we want differ from the ones it has', () => {it('reads every watcher on the host in one command, keyed by agent', async () => {const exec = vi.fn(async () => ({stdout: JSON.stringify([- {- agentId: 'switch-a1',- root: '/r/1',- running: true,- build: ENTRYPOINT,- enabled: true,- spawn: true,- stoodDown: false,- },- {+ status({ agentId: 'switch-a1', root: '/r/1' }),+ status({agentId: 'switch-a2',root: '/r/2',running: false,build: null,enabled: null,spawn: null,- stoodDown: false,- },+ }),]),stderr: '',exitCode: 0,@@ -97,3 +104,55 @@ it('treats a host with no watchers as nothing to skip', async () => {expect(found.size).toBe(0);expect(watcherIsCurrent(found.get('switch-a1'), want)).toBe(false);});++it('reports each watcher’s process, what it wrote about itself, and why it stopped', async () => {+ const base = await mkdtemp(join(tmpdir(), 'host-watchers-'));+ try {+ const write = async (root: string, file: string, body: unknown) => {+ await mkdir(join(base, root, file.includes('/') ? file.split('/')[0]! : ''), {+ recursive: true,+ });+ await writeFile(join(base, root, file), JSON.stringify(body));+ };+ const healthFile = { state: 'connected', detail: null, since: 'x', placements: {} };+ // Alive: this test's own process stands in for the watcher.+ await write('alive', 'config.json', { session: { agentId: 'switch-alive' } });+ await write('alive', 'shared-owner.lock', { pid: process.pid });+ await write('alive', 'health.json', { ...healthFile, pid: process.pid, updatedAt: 'y' });+ // Stopped, with the reason it recorded and no process behind its lock.+ await write('dead', 'config.json', { session: { agentId: 'switch-dead' } });+ await write('dead', 'shared-owner.lock', { pid: 2 ** 22 + 12345 });+ await write('dead', 'supervisor/failure.json', { message: 'out of memory' });+ // Stood down for another client.+ await write('displaced', 'config.json', { session: { agentId: 'switch-displaced' } });+ await write('displaced', 'taken-over.json', { at: 'z', reason: 'another client' });+ // A staging directory an interrupted launch left: not a watcher.+ await write('.launch-x', 'config.json', { session: { agentId: 'switch-alive' } });++ const { stdout } = await promisify(execFile)(process.execPath, [+ '-e',+ WATCHER_STATUS_SCRIPT,+ base,+ ]);+ const found = new Map(+ (JSON.parse(stdout) as HostWatcherStatus[]).map((entry) => [entry.agentId, entry])+ );++ expect([...found.keys()].sort()).toEqual(['switch-alive', 'switch-dead', 'switch-displaced']);+ expect(found.get('switch-alive')).toMatchObject({+ workerPid: process.pid,+ workerAlive: true,+ health: { state: 'connected', pid: process.pid },+ });+ expect(found.get('switch-dead')).toMatchObject({+ workerAlive: false,+ failure: 'out of memory',+ });+ expect(found.get('switch-displaced')).toMatchObject({+ stoodDown: true,+ takenOver: { at: 'z', reason: 'another client' },+ });+ } finally {+ await rm(base, { recursive: true, force: true });+ }+});diff --git a/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.ts b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.tsindex 5aaa8a3d5..b1124e4ed 100644--- a/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.ts+++ b/console/apps/switch-console-desktop/src/main/core/sdk-host/host-watchers.ts@@ -1,6 +1,7 @@import { z } from 'zod';import type { IExecutionContext } from '@main/core/execution-context/types';import { READ_JSON } from './remote-json';+import { IS_STATE_ROOT } from './state-roots';/*** What every watcher on one host is currently doing — one command, whatever@@ -23,7 +24,7 @@ import { READ_JSON } from './remote-json';* `watcherIsCurrent` for that, so the decision lives in one place and can be* read without the script.*/-export const WATCHER_STATUS_SCRIPT = String.raw`${READ_JSON}+export const WATCHER_STATUS_SCRIPT = String.raw`${READ_JSON}${IS_STATE_ROOT}const fs = require('node:fs');const os = require('node:os');const path = require('node:path');@@ -33,7 +34,7 @@ const alive = (pid) => {try { process.kill(pid, 0); return true; } catch { return false; }};let names = [];-try { names = fs.readdirSync(base); } catch {}+try { names = fs.readdirSync(base).filter(isStateRoot); } catch {}const found = [];for (const name of names) {const root = path.join(base, name);@@ -46,6 +47,15 @@ for (const name of names) {try { watch = readJson(path.join(root, 'watch.json')); } catch (e) { if (e.code !== 'ENOENT') throw e }let stoodDown = false;try { fs.statSync(path.join(root, 'taken-over.json')); stoodDown = true; } catch (e) { if (e.code !== 'ENOENT') throw e }+ let takenOver = null;+ try { takenOver = readJson(path.join(root, 'taken-over.json')); } catch (e) { if (e.code !== 'ENOENT') throw e }+ let worker = null;+ try { worker = readJson(path.join(root, 'shared-owner.lock')); } catch (e) { if (e.code !== 'ENOENT') throw e }+ let health = null;+ try { health = readJson(path.join(root, 'health.json')); } catch (e) { if (e.code !== 'ENOENT') throw e }+ let failure = null;+ try { failure = readJson(path.join(root, 'supervisor', 'failure.json')); } catch (e) { if (e.code !== 'ENOENT') throw e }+ const workerPid = worker && Number.isSafeInteger(worker.pid) && worker.pid > 0 ? worker.pid : null;found.push({agentId: config.session.agentId,root,@@ -54,6 +64,14 @@ for (const name of names) {enabled: watch ? watch.enabled === true : null,spawn: watch ? watch.spawn === true : null,stoodDown,+ supervisorPid: owner && Number.isSafeInteger(owner.pid) && owner.pid > 0 ? owner.pid : null,+ workerPid,+ workerAlive: workerPid !== null && alive(workerPid),+ health,+ failure: failure && typeof failure.message === 'string' ? failure.message : null,+ takenOver: takenOver && typeof takenOver.reason === 'string'+ ? { at: String(takenOver.at ?? ''), reason: takenOver.reason }+ : null,});}process.stdout.write(JSON.stringify(found));@@ -68,6 +86,18 @@ const statusSchema = z.array(enabled: z.boolean().nullable(),spawn: z.boolean().nullable(),stoodDown: z.boolean(),+ supervisorPid: z.number().int().positive().nullable(),+ /** The watcher process itself, from `shared-owner.lock`, and whether it is alive. */+ workerPid: z.number().int().positive().nullable(),+ workerAlive: z.boolean(),+ /**+ * What the watcher last wrote about its connection to Switch, or null+ * from a sidecar that does not write it. Unparsed here: see+ * `watcherHealthFileSchema`, applied where it is read.+ */+ health: z.unknown(),+ failure: z.string().nullable(),+ takenOver: z.object({ at: z.string(), reason: z.string() }).nullable(),}));diff --git a/console/packages/agent-providers/src/host/shared-daemon.ts b/console/packages/agent-providers/src/host/shared-daemon.tsindex b61e6f7b0..7e289e491 100644--- a/console/packages/agent-providers/src/host/shared-daemon.ts+++ b/console/packages/agent-providers/src/host/shared-daemon.ts@@ -13,6 +13,7 @@ import { sharedConfigSchema } from './shared-config';import { hostSessionProcess } from './shared-host';import { runSharedWatcher } from './shared-watcher';import { superviseSharedHost } from './supervisor';+import { recordWatcherHealth } from './watcher-health-file';import { WatcherControl } from './watcher-tools';const [root, configPath, mode] = process.argv.slice(2);@@ -120,6 +121,9 @@ async function main(): Promise<void> {const ensure = ensureSessions(supervision);// Console's "Reconnect to room" reaches the watcher through the control port.const control = new WatcherControl();+ // Console reads the watcher's connection state from this file, with the+ // rest of the host's watcher state, rather than from the control port.+ const stopRecording = recordWatcherHealth(resolve(root), control);// A watcher that stops (disabled, stood down after a takeover, or// signalled) takes the process with it: the control port and every// session host go too, so the supervisor sees a clean exit and does not@@ -132,6 +136,7 @@ async function main(): Promise<void> {serveControl(resolve(root), links, ensure, control, stop.signal),]);} finally {+ stopRecording();await supervision.close();}} else if (process.platform !== 'win32' && (await ownProcessGroup()) === null) {diff --git a/console/packages/agent-providers/src/host/watcher-health-file.test.ts b/console/packages/agent-providers/src/host/watcher-health-file.test.tsnew file mode 100644index 000000000..b367ff86c--- /dev/null+++ b/console/packages/agent-providers/src/host/watcher-health-file.test.ts@@ -0,0 +1,44 @@+import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, expect, it, vi } from 'vitest';+import { recordWatcherHealth, WATCHER_HEALTH_FILE } from './watcher-health-file';+import { WatcherControl } from './watcher-tools';++const roots: string[] = [];+afterEach(async () => {+ for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true });+});++async function written(root: string) {+ return JSON.parse(await readFile(join(root, WATCHER_HEALTH_FILE), 'utf8'));+}++it('keeps the watcher’s own account of its connection on the host, as it changes', async () => {+ const root = await mkdtemp(join(tmpdir(), 'watcher-health-'));+ roots.push(root);+ const control = new WatcherControl();+ const stop = recordWatcherHealth(root, control);++ await vi.waitFor(async () =>+ expect(await written(root)).toMatchObject({ state: 'not-running', pid: process.pid })+ );++ control.report({ state: 'connecting' });+ control.report({ state: 'connected', placements: { session: 'room' } });+ await vi.waitFor(async () =>+ expect(await written(root)).toMatchObject({+ state: 'connected',+ detail: null,+ placements: { session: 'room' },+ pid: process.pid,+ })+ );+ // Renamed into place, so nothing half-written is ever left beside it.+ expect(await readdir(root)).toEqual([WATCHER_HEALTH_FILE]);++ stop();+ control.report({ state: 'disconnected', detail: 'gone' });+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect((await written(root)).state).toBe('connected');+});diff --git a/console/packages/agent-providers/src/host/watcher-health-file.ts b/console/packages/agent-providers/src/host/watcher-health-file.tsnew file mode 100644index 000000000..bc92e179b--- /dev/null+++ b/console/packages/agent-providers/src/host/watcher-health-file.ts@@ -0,0 +1,58 @@+import { randomUUID } from 'node:crypto';+import { rename, writeFile } from 'node:fs/promises';+import { join } from 'node:path';+import { z } from 'zod';+import { type WatcherControl, watcherHealthSchema } from './watcher-tools';++/**+ * Where a watcher keeps its own account of its connection to Switch, beside+ * its other state.+ *+ * Console reads it with the rest of the host's watcher state, in one command+ * for every agent on the host, instead of holding a connection to each sidecar+ * and waiting to be told. A connection can die without saying so — across an+ * SSH reconnect, say — and then the last thing it carried stays on screen for+ * good. A file read every few seconds cannot go stale that way.+ */+export const WATCHER_HEALTH_FILE = 'health.json';++/**+ * What the file holds: the watcher's health, the process that wrote it, and+ * when. A reader compares `pid` with the watcher process it sees alive, so a+ * file a previous watcher left behind is not read as this one's word.+ */+export const watcherHealthFileSchema = watcherHealthSchema.extend({+ pid: z.number().int().positive(),+ updatedAt: z.string(),+});++export type WatcherHealthFile = z.infer<typeof watcherHealthFileSchema>;++/**+ * Keep `health.json` in `root` in step with what the watcher reports to+ * `control`, from now until the returned function is called. Written whole,+ * under a temporary name and renamed into place, so a reader never sees half+ * of it; and in order, so an older state never lands after a newer one.+ */+export function recordWatcherHealth(root: string, control: WatcherControl): () => void {+ const path = join(root, WATCHER_HEALTH_FILE);+ let writing: Promise<void> = Promise.resolve();+ const write = () => {+ const body: WatcherHealthFile = {+ ...control.health(),+ pid: process.pid,+ updatedAt: new Date().toISOString(),+ };+ writing = writing+ .then(async () => {+ const temporary = `${path}.${randomUUID()}`;+ await writeFile(temporary, JSON.stringify(body), { mode: 0o600 });+ await rename(temporary, path);+ })+ .catch((error: unknown) => {+ console.error(`Could not record the room watcher's health in ${path}: ${String(error)}`);+ });+ };+ write();+ return control.onHealth(write);+}diff --git a/console/packages/agent-providers/src/index.ts b/console/packages/agent-providers/src/index.tsindex 67de28a7a..f535c15d8 100644--- a/console/packages/agent-providers/src/index.ts+++ b/console/packages/agent-providers/src/index.ts@@ -60,6 +60,12 @@ export type { Supervision } from './host/launch';export { hostStartSources, type HostStartSource } from './host/session-start';export { runSharedWatcher } from './host/shared-watcher';export { clearTakenOver, readTakenOver, type TakenOver } from './host/taken-over';+export {+ recordWatcherHealth,+ WATCHER_HEALTH_FILE,+ type WatcherHealthFile,+ watcherHealthFileSchema,+} from './host/watcher-health-file';export {readWatchFlags,WATCH_FLAGS_FILE,