Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(console): recover cleanly after sleep — watcher self-takeover, silent SSH death, one row indicator (#621)

Louis Amaudruz74d03bb1c3de
diff --git a/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.test.ts b/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.test.ts
index a4c0de41c..90e61895a 100644
--- a/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.test.ts
@@ -148,6 +148,19 @@ describe('SshExecutionContext.exec', () => {
expect(isTransportFailure(error)).toBe(true);
});
+ it('wraps ssh2 "Not connected" as a transport failure', async () => {
+ // A dead transport, not a failed command: read as an ordinary failure, a
+ // CLI lookup that hit it reported the CLI as not installed.
+ const ctx = new SshExecutionContext(
+ makeProxy((_command, cb) => {
+ cb(new Error('Not connected'));
+ })
+ );
+
+ const error = await ctx.exec('which', ['claude']).catch((e: unknown) => e);
+ expect(isTransportFailure(error)).toBe(true);
+ });
+
it('wraps a missing connection as a transport failure', async () => {
const proxy = {
getRemoteShellProfile: () => Promise.reject(new Error('SSH connection is not available')),
diff --git a/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.ts b/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.ts
index a74b8d534..a06760800 100644
--- a/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.ts
+++ b/console/apps/switch-console-desktop/src/main/core/execution-context/ssh-execution-context.ts
@@ -7,6 +7,7 @@ import {
import {
isSshChannelOpenFailure,
isSshChannelTimeout,
+ isSshTransportGone,
} from '@main/core/ssh/lifecycle/ssh-channel-open-failure';
import type { SshClientProxy } from '@main/core/ssh/lifecycle/ssh-client-proxy';
import { quoteShellArg } from '@main/utils/shellEscape';
@@ -26,6 +27,10 @@ function toTransportError(error: unknown): TransportError {
function isTransportShaped(error: unknown): boolean {
if (isSshChannelOpenFailure(error) || isSshChannelTimeout(error)) return true;
+ // ssh2's "Not connected": the transport is gone. Unrecognised, it fell
+ // through as an ordinary command failure, and a CLI lookup that failed this
+ // way reported the CLI as not installed.
+ if (isSshTransportGone(error)) return true;
return error instanceof Error && error.message.includes('SSH connection is not available');
}
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-channel-open-failure.ts b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-channel-open-failure.ts
index 328629938..6fe931fa5 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-channel-open-failure.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-channel-open-failure.ts
@@ -48,3 +48,16 @@ export class SshChannelTimeoutError extends Error {
export function isSshChannelTimeout(error: unknown): boolean {
return error instanceof SshChannelTimeoutError;
}
+
+/**
+ * ssh2's answer to any channel open on a client whose transport is gone:
+ * thrown synchronously by `exec`, `forwardOut` and `sftp` once the socket can
+ * no longer be read. Unlike a refusal or a slow open it is never transient —
+ * the client cannot recover from it, and its own `destroy()` is a no-op by
+ * then, so it may never emit `close` either. Seen on an IAP tunnel whose
+ * stdout ended without the process exiting in error (2026-09-30, dev-vm).
+ */
+export function isSshTransportGone(error: unknown): boolean {
+ const message = error instanceof Error ? error.message : String(error);
+ return message === 'Not connected' || message === 'No response from server';
+}
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.test.ts b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.test.ts
index b503c0d5e..9d17f079d 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.test.ts
@@ -286,6 +286,40 @@ describe('SshClientProxy channel open timeouts', () => {
}
});
+ it('settles an exec that throws synchronously once, with no invented timeout after it', async () => {
+ // ssh2 throws "Not connected" from exec on a dead transport. The open
+ // timer used to outlive that throw and fire 15 s later: a second callback,
+ // and a fake "timed out" fed into the wedge count.
+ vi.useFakeTimers();
+ try {
+ const reporter = { reportChannelError: vi.fn(), reportChannelSuccess: vi.fn() };
+ const client = {
+ exec: vi.fn(() => {
+ throw new Error('Not connected');
+ }),
+ };
+ const proxy = new SshClientProxy('ssh-1', reporter);
+ proxy.update(client as never);
+
+ const callback = vi.fn();
+ proxy.exec('true', callback);
+ expect(callback).toHaveBeenCalledTimes(1);
+ expect(String(callback.mock.calls[0]![0])).toContain('Not connected');
+ // Reported, so the manager can drop the dead transport at once.
+ expect(reporter.reportChannelError).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(20_000);
+ expect(callback).toHaveBeenCalledTimes(1);
+ expect(reporter.reportChannelError).toHaveBeenCalledTimes(1);
+
+ // The slot was freed: four more execs are all attempted.
+ for (let i = 0; i < 4; i += 1) proxy.exec('true', vi.fn());
+ expect(client.exec).toHaveBeenCalledTimes(5);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
it('rejects a forwardOut whose open is never answered', async () => {
vi.useFakeTimers();
try {
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.ts b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.ts
index 2cbd4b2f7..fdb80bcc7 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-client-proxy.ts
@@ -228,9 +228,12 @@ export class SshClientProxy {
this.client.exec(command, wrappedCallback);
}
} catch (err) {
- // e.g. connection not available — surface via callback, don't leak the slot.
- release();
- userCallback?.(err as Error, undefined as never);
+ // e.g. connection not available. Settled through the wrapper, so its
+ // open-timeout is cancelled and the failure is reported once: called
+ // around it, the timer outlived the throw and fired a second,
+ // invented "timed out" 15 s later, into the callback again and into
+ // the wedge count. The wrapper's callback frees the slot.
+ wrappedCallback(err as Error & { code: number }, undefined as never);
}
};
@@ -257,14 +260,22 @@ export class SshClientProxy {
try {
this.client.forwardOut('127.0.0.1', 0, '127.0.0.1', dstPort, wrapped);
} catch (err) {
- reject(err instanceof Error ? err : new Error(String(err)));
+ // Through the wrapper, not around it: see exec.
+ const error = err instanceof Error ? err : new Error(String(err));
+ wrapped(error as Error & { code: number }, undefined as never);
}
});
}
sftp(callback: ClientSFTPCallback): void {
const wrapped = this.withChannelOpenTimeout('sftp', callback as unknown as ClientCallback);
- this.client.sftp(wrapped as unknown as ClientSFTPCallback);
+ try {
+ this.client.sftp(wrapped as unknown as ClientSFTPCallback);
+ } catch (err) {
+ // Through the wrapper, not around it: see exec.
+ const error = err instanceof Error ? err : new Error(String(err));
+ wrapped(error as Error & { code: number }, undefined as never);
+ }
}
private reportChannelResult(err: Error | undefined): void {
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.test.ts b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.test.ts
index 74ea2b203..c73d07416 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.test.ts
@@ -827,4 +827,128 @@ describe('SshConnectionManager', () => {
vi.useRealTimers();
}
});
+
+ /**
+ * A transport that dies without ssh2 ever saying `close`: its read side
+ * ended (an IAP tunnel's stdout, on dev-vm), and from then on ssh2 refuses
+ * every channel with "Not connected" while `destroy()` and `end()` do
+ * nothing. Every recovery that waited on `close` waited forever — the wedge
+ * rebuild fired ~260 times a minute for hours, and the host was never
+ * reported down.
+ */
+ describe('a connection that dies without closing', () => {
+ class SilentlyDyingClient extends PassThrough {
+ ready = true;
+ connect() {
+ queueMicrotask(() => {
+ if (this.ready) this.emit('ready');
+ });
+ }
+ destroy() {
+ return this;
+ }
+ end() {
+ return this;
+ }
+ }
+
+ async function setUp() {
+ const { SshConnectionManager } = await import('./ssh-connection-manager');
+ const clients: SilentlyDyingClient[] = [];
+ const cleanups: number[] = [];
+ const manager = new SshConnectionManager({
+ createClient: () => {
+ const client = new SilentlyDyingClient();
+ clients.push(client);
+ return client as unknown as Client;
+ },
+ });
+ manager.register('ssh-1', async () => ({
+ config: { sock: new PassThrough(), username: 'alice' },
+ cleanup: () => cleanups.push(clients.length),
+ debugLogs: [],
+ }));
+ const events: string[] = [];
+ manager.on('connection-event', (event) => events.push(event.type));
+ const proxy = await manager.connect('ssh-1');
+ expect(proxy.isConnected).toBe(true);
+ return { manager, clients, cleanups, events, proxy };
+ }
+
+ async function expectRecovered(
+ ctx: Awaited<ReturnType<typeof setUp>>,
+ what: string
+ ): Promise<void> {
+ expect(ctx.proxy.isConnected, `${what}: dead client still serves`).toBe(false);
+ expect(ctx.events).toContain('disconnected');
+ expect(ctx.events).toContain('reconnecting');
+ // The transport is torn down (for a ProxyCommand, its process killed).
+ expect(ctx.cleanups).toEqual([1]);
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(ctx.proxy.isConnected).toBe(true);
+ expect(ctx.proxy.client).toBe(ctx.clients[1]);
+ // These clients never close, so the graceful disconnect runs to its timeout.
+ const disconnected = ctx.manager.disconnect('ssh-1');
+ await vi.advanceTimersByTimeAsync(5_000);
+ await disconnected;
+ }
+
+ it('reconnects when the transport ends, without waiting for close', async () => {
+ vi.useFakeTimers();
+ try {
+ const ctx = await setUp();
+ ctx.clients[0]!.emit('end');
+ await expectRecovered(ctx, 'end');
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('reconnects on the first "Not connected", which is never transient', async () => {
+ vi.useFakeTimers();
+ try {
+ const ctx = await setUp();
+ ctx.manager.reportChannelError('ssh-1', new Error('Not connected'));
+ await expectRecovered(ctx, 'Not connected');
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('completes a wedge rebuild whose client never emits close', async () => {
+ vi.useFakeTimers();
+ try {
+ const ctx = await setUp();
+ const wedgeError = { reason: 2, message: 'channel open failure' };
+ for (let i = 0; i < 3; i += 1) ctx.manager.reportChannelError('ssh-1', wedgeError);
+ await expectRecovered(ctx, 'wedge');
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('completes a resume refresh whose client never emits close', async () => {
+ vi.useFakeTimers();
+ try {
+ const ctx = await setUp();
+ ctx.manager.handleSystemResume();
+ await expectRecovered(ctx, 'resume');
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('does not act twice when the dead client closes after all', async () => {
+ vi.useFakeTimers();
+ try {
+ const ctx = await setUp();
+ ctx.clients[0]!.emit('end');
+ ctx.clients[0]!.emit('close');
+ expect(ctx.events.filter((e) => e === 'reconnecting')).toHaveLength(1);
+ await expectRecovered(ctx, 'late close');
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+ });
});
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
index cfb82cad8..f5a9a67e0 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
@@ -3,7 +3,11 @@ import ssh2, { type Client, type ConnectConfig } from 'ssh2';
import type { ConnectionState, SshHealthState } from '@shared/core/ssh/ssh';
import type { SshConnectionEvent } from '@shared/core/ssh/sshEvents';
import type { SshConnectResult } from '../connect/resolve-ssh-connect-config';
-import { isSshChannelOpenFailure, isSshChannelTimeout } from './ssh-channel-open-failure';
+import {
+ isSshChannelOpenFailure,
+ isSshChannelTimeout,
+ isSshTransportGone,
+} from './ssh-channel-open-failure';
import { SshClientProxy } from './ssh-client-proxy';
const { Client: Ssh2Client } = ssh2;
@@ -118,6 +122,13 @@ interface ConnectionRecord {
* transport (terminal + agent runtime providers) always get their signal.
*/
hasConnectedBefore: boolean;
+ /**
+ * Declare the live client dead and recover, without waiting for it to say
+ * so. Set while a client backs the proxy. A transport can die without ssh2
+ * ever emitting `close` — and then `destroy()` emits nothing either — so any
+ * recovery that waits on that event waits forever.
+ */
+ abandon?: (reason: string) => void;
}
// ─── Implementation ──────────────────────────────────────────────────────────
@@ -313,6 +324,12 @@ export class SshConnectionManager extends EventEmitter {
* events never surface — so the manager force-rebuilds it.
*/
reportChannelError(connectionId: string, error: unknown): void {
+ if (isSshTransportGone(error)) {
+ // Not a symptom to count towards a wedge: the client has already lost
+ // its transport and will refuse every open from now on. One is proof.
+ this.records.get(connectionId)?.abandon?.('channel open refused: the transport is gone');
+ return;
+ }
if (!isSshChannelOpenFailure(error) && !isSshChannelTimeout(error)) return;
const record = this.record(connectionId);
@@ -347,9 +364,11 @@ export class SshConnectionManager extends EventEmitter {
}
);
record.channelFailureStreak = 0;
- // Destroying the client fires its close handler, which invalidates the
- // proxy and schedules the normal auto-reconnect path.
- record.client.destroy();
+ // Not `destroy()` and wait for its close handler: a dead transport may
+ // never emit `close`, and this rebuild then repeated every few seconds
+ // for hours without ever reconnecting (dev-vm, 2026-09-30).
+ if (record.abandon) record.abandon('wedged: consecutive channel failures');
+ else record.client.destroy();
}
}
@@ -445,7 +464,10 @@ export class SshConnectionManager extends EventEmitter {
this.deps.log.warn('SshConnectionManager: system resumed — refreshing live connection', {
connectionId: record.id,
});
- record.client?.destroy();
+ // Abandoned rather than destroyed-and-awaited, for the same reason as
+ // the wedge rebuild: the reconnect must not hinge on a `close` event.
+ if (record.abandon) record.abandon('system resumed');
+ else record.client?.destroy();
} else if (record.state === 'reconnecting') {
this.deps.log.warn('SshConnectionManager: system resumed — retrying reconnect now', {
connectionId: record.id,
@@ -545,6 +567,40 @@ export class SshConnectionManager extends EventEmitter {
this.deps.publishEvent({ type: 'disconnected', connectionId: id });
};
+ /**
+ * Tear the live client down ourselves and schedule the reconnect — the
+ * same outcome as its `close` handler, reached without needing `close`.
+ * Once the proxy no longer points at this client, a `close` or `error`
+ * that straggles in from it later finds nothing to act on.
+ */
+ const abandon = (reason: string) => {
+ if (!(proxy.isConnected && proxy.client === client)) return;
+ this.deps.log.warn('SshConnectionManager: abandoning dead connection', {
+ event: 'ssh_connection_abandoned',
+ connectionId: id,
+ reason,
+ });
+ if (record.abandon === abandon) record.abandon = undefined;
+ proxy.invalidate();
+ emitDisconnectedOnce();
+ cleanupOnce();
+ try {
+ client.destroy();
+ } catch {
+ // A client in this state may throw on teardown; it is discarded anyway.
+ }
+ const isCurrent = record.generation === generation;
+ if (!isCurrent) return;
+ this.setState(record, 'disconnected');
+ if (!record.intentional) this.scheduleReconnect(record);
+ };
+
+ // The transport's read side ended. ssh2 surfaces that as `end`, and only
+ // emits `close` once the socket is fully torn down — which a half-open
+ // ProxyCommand pipe may never be. From here every channel open throws
+ // "Not connected", so this is the moment the connection is gone.
+ client.on('end', () => abandon('the transport ended'));
+
client.on('error', (error: Error) => {
// A remote host being down is an expected condition, not a fault in the
// app: laptops sleep, VPNs drop, SSO tokens expire. The host's
@@ -630,6 +686,7 @@ export class SshConnectionManager extends EventEmitter {
}
proxy.update(client);
+ record.abandon = abandon;
record.channelFailureStreak = 0;
record.hasConnectedBefore = true;
this.clearHealthState(record);
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.test.ts b/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.test.ts
index 17cd645d0..901cd7bfe 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.test.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.test.ts
@@ -176,6 +176,23 @@ describe('childToDuplex', () => {
]);
});
+ it('closes the duplex when the proxy process exits cleanly', async () => {
+ // A clean exit used to leave the pipe half-open: read side ended, never
+ // destroyed, so ssh2 never emitted `close` and the connection was never
+ // seen to die (an IAP tunnel on dev-vm, for hours).
+ const child = fakeChild();
+ const duplex = childToDuplex(child);
+ const closed = once(duplex, 'close');
+ const errors: unknown[] = [];
+ duplex.on('error', (error) => errors.push(error));
+
+ child.emit('close', 0, null);
+
+ await closed;
+ expect(duplex.destroyed).toBe(true);
+ expect(errors).toEqual([]);
+ });
+
it('ignores stdin EPIPE but destroys on other stdin errors', async () => {
const epipeChild = fakeChild();
const epipeDuplex = childToDuplex(epipeChild);
diff --git a/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.ts b/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.ts
index cb508292f..9345f6e94 100644
--- a/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.ts
+++ b/console/apps/switch-console-desktop/src/main/core/ssh/transport/transports.ts
@@ -93,7 +93,15 @@ export function childToDuplex(child: SshChild): Duplex {
});
child.once('error', (error) => duplex.destroy(error));
child.once('close', (code, signal) => {
- if (duplex.destroyed || code === 0) return;
+ if (duplex.destroyed) return;
+ // A clean exit still ends the transport. Left alone, the pipe stayed
+ // half-open — read side ended, never destroyed — so ssh2 refused every
+ // channel with "Not connected" and never emitted `close` (an IAP tunnel
+ // on dev-vm, 2026-09-30, for hours).
+ if (code === 0) {
+ duplex.destroy();
+ return;
+ }
const reason = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`;
duplex.destroy(new Error(`Proxy process exited with ${reason}`));
});
diff --git a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-trouble-indicator.tsx b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-trouble-indicator.tsx
index e1cd53a5b..e79707a8a 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-trouble-indicator.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/remote-hosts/host-trouble-indicator.tsx
@@ -15,7 +15,7 @@
* as broken because Codex is absent.
*/
-import { PlugZap, Wrench } from 'lucide-react';
+import { ServerOff, Wrench } from 'lucide-react';
import { observer } from 'mobx-react-lite';
import { useEffect } from 'react';
import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/tooltip';
@@ -55,7 +55,7 @@ export const HostTroubleIndicator = observer(function HostTroubleIndicator({
return (
<Tooltip>
<TooltipTrigger>
- <PlugZap className={ICON} aria-label="Host unavailable" />
+ <ServerOff className={ICON} aria-label="Host unavailable" />
</TooltipTrigger>
<TooltipContent>
{status.kind === 'auth-failed'
diff --git a/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-item.tsx b/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-item.tsx
index 1f529a096..76a362dd8 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-item.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-item.tsx
@@ -33,6 +33,7 @@ import {
import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/tooltip';
import { cn } from '@renderer/utils/utils';
import type { Agent } from '@shared/core/agents/agents';
+import { AgentStatusSlot } from './agent-status-slot';
import { DiscoveryFailureIndicator } from './discovery-failure-indicator';
import { SidebarItemMiniButton, SidebarMenuAction, SidebarMenuRow } from './sidebar-primitives';
import { agentExpandKey, depthIndent } from './sidebar-store';
@@ -136,15 +137,13 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({
) : (
<Bot className="h-3 w-3 shrink-0 text-foreground-muted" />
))}
- {location.data?.sshHost != null && (
+ {/* A down host takes this slot over: the status below draws it
+ as the server, disconnected, so the row carries one icon
+ for the host rather than a server and a warning. */}
+ {location.data?.sshHost != null && !hostUnreachable && (
<Tooltip>
<TooltipTrigger>
- <Server
- className={cn(
- 'h-3.5 w-3.5 shrink-0',
- hostUnreachable ? 'text-foreground-warning' : 'text-foreground-muted'
- )}
- />
+ <Server className="h-3.5 w-3.5 shrink-0 text-foreground-muted" />
</TooltipTrigger>
<TooltipContent>
Runs remotely on {location.data.sshHost}
@@ -152,52 +151,54 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({
</TooltipContent>
</Tooltip>
)}
- {/* Unreachable host, or one missing something this agent needs.
- Shared with the room-grouped rows so the two trees cannot
- disagree about the same agent (CHOO-1682/1809). */}
- <HostTroubleIndicator sshHost={sshHost} agentId={agent.providerId ?? null} />
- <DiscoveryFailureIndicator agentId={agent.id} label={label} />
- {agent.providerId && (
- <ProviderIssueIndicator
- providerId={agent.providerId}
- sshHost={sshHost}
- hostReachable={!hostUnreachable}
- onOpen={open}
- />
- )}
- {locationViewKind(location) === 'ready' &&
- hasSessionError(agent.locationId) &&
- (hasDiscardableSessionError(agent.locationId) ? (
- <Tooltip>
- <TooltipTrigger
- render={
- <SidebarItemMiniButton
- type="button"
- aria-label={`Dismiss failed session for ${label}`}
- onClick={(e) => {
- e.stopPropagation();
- getSessionManagerStore(agent.locationId)?.discardFailedCreations();
- }}
- >
- <TriangleAlert className="h-3.5 w-3.5 shrink-0 text-foreground-destructive" />
- </SidebarItemMiniButton>
- }
- />
- <TooltipContent>
- A session failed to connect — click to dismiss
- </TooltipContent>
- </Tooltip>
- ) : (
- <Tooltip>
- <TooltipTrigger>
- <TriangleAlert className="h-3.5 w-3.5 shrink-0 text-foreground-destructive" />
- </TooltipTrigger>
- <TooltipContent>A session failed to connect</TooltipContent>
- </Tooltip>
- ))}
+ <AgentStatusSlot>
+ {/* Unreachable host, or one missing something this agent needs.
+ Shared with the room-grouped rows so the two trees cannot
+ disagree about the same agent (CHOO-1682/1809). */}
+ <HostTroubleIndicator sshHost={sshHost} agentId={agent.providerId ?? null} />
+ <AgentConnectionIndicator agent={agent} />
+ <DiscoveryFailureIndicator agentId={agent.id} label={label} />
+ {agent.providerId && (
+ <ProviderIssueIndicator
+ providerId={agent.providerId}
+ sshHost={sshHost}
+ hostReachable={!hostUnreachable}
+ onOpen={open}
+ />
+ )}
+ {locationViewKind(location) === 'ready' &&
+ hasSessionError(agent.locationId) &&
+ (hasDiscardableSessionError(agent.locationId) ? (
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <SidebarItemMiniButton
+ type="button"
+ aria-label={`Dismiss failed session for ${label}`}
+ onClick={(e) => {
+ e.stopPropagation();
+ getSessionManagerStore(agent.locationId)?.discardFailedCreations();
+ }}
+ >
+ <TriangleAlert className="h-3.5 w-3.5 shrink-0 text-foreground-destructive" />
+ </SidebarItemMiniButton>
+ }
+ />
+ <TooltipContent>
+ A session failed to connect — click to dismiss
+ </TooltipContent>
+ </Tooltip>
+ ) : (
+ <Tooltip>
+ <TooltipTrigger>
+ <TriangleAlert className="h-3.5 w-3.5 shrink-0 text-foreground-destructive" />
+ </TooltipTrigger>
+ <TooltipContent>A session failed to connect</TooltipContent>
+ </Tooltip>
+ ))}
+ </AgentStatusSlot>
</span>
</SidebarMenuAction>
- <AgentConnectionIndicator agent={agent} />
</div>
<Tooltip>
<TooltipTrigger
diff --git a/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-status-slot.tsx b/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-status-slot.tsx
new file mode 100644
index 000000000..69509485a
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/features/sidebar/agent-status-slot.tsx
@@ -0,0 +1,22 @@
+import type { ReactNode } from 'react';
+
+/**
+ * At most one warning on an agent's row: the first of its children that
+ * renders anything.
+ *
+ * Every indicator is its own check, and each used to draw its own icon, so one
+ * cause showed up as several — a host whose link had died read as "sessions
+ * could not load", "CLI not installed" and "watcher disconnected" side by
+ * side, all of them consequences of the one outage. Children go in order of
+ * cause: the host first, because when it is down nothing else on the row can
+ * be judged, then the agent's connection, then what depends on it. Each still
+ * decides for itself whether it has anything to say (rendering nothing when
+ * not), so the order here is the only thing that ranks them.
+ */
+export function AgentStatusSlot({ children }: { children: ReactNode }) {
+ return (
+ <span className="inline-flex shrink-0 items-center [&>*:not(:first-child)]:hidden">
+ {children}
+ </span>
+ );
+}
diff --git a/console/apps/switch-console-desktop/src/renderer/features/sidebar/room-agent-row.tsx b/console/apps/switch-console-desktop/src/renderer/features/sidebar/room-agent-row.tsx
index aa62c7c82..facbd1d44 100644
--- a/console/apps/switch-console-desktop/src/renderer/features/sidebar/room-agent-row.tsx
+++ b/console/apps/switch-console-desktop/src/renderer/features/sidebar/room-agent-row.tsx
@@ -24,6 +24,7 @@ import {
import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/tooltip';
import { cn } from '@renderer/utils/utils';
import type { Agent } from '@shared/core/agents/agents';
+import { AgentStatusSlot } from './agent-status-slot';
import { DiscoveryFailureIndicator } from './discovery-failure-indicator';
import { SidebarItemMiniButton, SidebarMenuAction, SidebarMenuRow } from './sidebar-primitives';
import { depthIndent, roomAgentGroupKey } from './sidebar-store';
@@ -136,27 +137,34 @@ export const RoomAgentRow = observer(function RoomAgentRow({
) : (
<Bot className="h-3 w-3 shrink-0 text-foreground-muted" />
))}
- {/* Same agent, same host problem — this row used to show
- nothing, so whether you saw it depended on which grouping
- the sidebar happened to be in. */}
- <HostTroubleIndicator
- sshHost={location.data?.sshHost ?? null}
- agentId={agent.providerId ?? null}
- />
- <DiscoveryFailureIndicator agentId={agent.id} label={label} />
- {agent.providerId && (
- <ProviderIssueIndicator
- providerId={agent.providerId}
+ <AgentStatusSlot>
+ {/* Same agent, same host problem — this row used to show
+ nothing, so whether you saw it depended on which grouping
+ the sidebar happened to be in. */}
+ <HostTroubleIndicator
sshHost={location.data?.sshHost ?? null}
- hostReachable={!hostReachabilityStore.isBlocked(location.data?.sshHost ?? null)}
- onOpen={() =>
- navigate('location', { locationId: agent.locationId, agentName: agent.name })
- }
+ agentId={agent.providerId ?? null}
/>
- )}
+ <AgentConnectionIndicator agent={agent} />
+ <DiscoveryFailureIndicator agentId={agent.id} label={label} />
+ {agent.providerId && (
+ <ProviderIssueIndicator
+ providerId={agent.providerId}
+ sshHost={location.data?.sshHost ?? null}
+ hostReachable={
+ !hostReachabilityStore.isBlocked(location.data?.sshHost ?? null)
+ }
+ onOpen={() =>
+ navigate('location', {
+ locationId: agent.locationId,
+ agentName: agent.name,
+ })
+ }
+ />
+ )}
+ </AgentStatusSlot>
</span>
</SidebarMenuAction>
- <AgentConnectionIndicator agent={agent} />
</div>
<Tooltip>
<TooltipTrigger
diff --git a/console/apps/switch-console-desktop/src/renderer/tests/browser/agent-status-slot.test.tsx b/console/apps/switch-console-desktop/src/renderer/tests/browser/agent-status-slot.test.tsx
new file mode 100644
index 000000000..80f1974b5
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/renderer/tests/browser/agent-status-slot.test.tsx
@@ -0,0 +1,64 @@
+import { act } from 'react';
+import { createRoot, type Root } from 'react-dom/client';
+import { afterEach, describe, expect, it } from 'vitest';
+import '@renderer/index.css';
+import { AgentStatusSlot } from '@renderer/features/sidebar/agent-status-slot';
+
+let container: HTMLDivElement | null = null;
+let root: Root | null = null;
+
+afterEach(async () => {
+ if (root) await act(async () => root!.unmount());
+ container?.remove();
+ container = null;
+ root = null;
+});
+
+/** An indicator with nothing to say, the way each one answers for a healthy agent. */
+function Silent() {
+ return null;
+}
+
+async function render(node: React.ReactNode): Promise<HTMLElement> {
+ container = document.createElement('div');
+ document.body.appendChild(container);
+ root = createRoot(container);
+ await act(async () => root!.render(node));
+ return container;
+}
+
+function visible(el: HTMLElement, id: string): boolean {
+ const node = el.querySelector<HTMLElement>(`[data-testid="${id}"]`);
+ return !!node && getComputedStyle(node).display !== 'none';
+}
+
+describe('AgentStatusSlot', () => {
+ it('shows only the first indicator that has something to say', async () => {
+ // A dead host used to put three warnings on one row, each a consequence of
+ // the outage the first one already named.
+ const el = await render(
+ <AgentStatusSlot>
+ <Silent />
+ <span data-testid="host">host</span>
+ <span data-testid="connection">connection</span>
+ <span data-testid="cli">cli</span>
+ </AgentStatusSlot>
+ );
+
+ expect(visible(el, 'host')).toBe(true);
+ expect(visible(el, 'connection')).toBe(false);
+ expect(visible(el, 'cli')).toBe(false);
+ });
+
+ it('falls through to the next cause when the first has nothing to report', async () => {
+ const el = await render(
+ <AgentStatusSlot>
+ <Silent />
+ <Silent />
+ <span data-testid="cli">cli</span>
+ </AgentStatusSlot>
+ );
+
+ expect(visible(el, 'cli')).toBe(true);
+ });
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.test.ts b/console/packages/switch-agent-runtime/src/event-stream.test.ts
index a79c3cbf3..00f6c5df6 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.test.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.test.ts
@@ -656,12 +656,14 @@ describe('the heartbeat', () => {
it('returns to the base cadence once a beat lands', async () => {
vi.useFakeTimers();
let reject = true;
- const fetchMock = vi.fn(async (url: string) => {
+ // A socket that ends when the client drops it: each refusal reopens, and
+ // the heartbeat is held from the reopen until the new stream is named.
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal }) => {
if (String(url).includes('/events'))
return {
ok: true,
status: 200,
- body: openForever(),
+ body: openUntilAborted(init),
text: async (): Promise<string> => '',
};
if (reject) return { ok: false, status: 404, text: async (): Promise<string> => '' };
@@ -1798,3 +1800,101 @@ it('still stands down when a request is refused as taken over on the incarnation
abort.abort();
}
});
+
+describe('the heartbeat across its own reopen after a refusal', () => {
+ /**
+ * What took every in-Console watcher off the air when a laptop woke
+ * (2026-10-01, four agents within ten seconds). The connection had lapsed
+ * while the machine slept, so the first beat after waking was refused with a
+ * 404. The heartbeat asked the stream to reopen and went straight back to
+ * its gate — which it had found open a moment before the stream loop closed
+ * it for the reopen. One beat went out under the lapsed incarnation right
+ * behind the open that replaced it. The server, holding the new incarnation,
+ * answered that beat `taken_over`, and the client believed it: it stood down
+ * over a connection it had just made itself. In the server log the open and
+ * the stale beat arrived 11 ms apart, every time.
+ */
+ function lapsingServer() {
+ let generation = 0;
+ let lapsed = false;
+ const fetchMock = vi.fn(async (url: string, init: { signal: AbortSignal; body?: string }) => {
+ if (String(url).includes('/events')) {
+ generation += 1;
+ lapsed = false;
+ const gen = generation;
+ return {
+ ok: true,
+ status: 200,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(connected(gen));
+ init.signal.addEventListener(
+ 'abort',
+ () => {
+ try {
+ controller.error(new DOMException('The operation was aborted.', 'AbortError'));
+ } catch {
+ // Already closed.
+ }
+ },
+ { once: true }
+ );
+ },
+ }),
+ text: async (): Promise<string> => '',
+ };
+ }
+ if (lapsed) return { ok: false, status: 404, text: async (): Promise<string> => '' };
+ const claimed = (JSON.parse(init.body ?? '{}') as { generation: number | null }).generation;
+ if (claimed !== null && claimed !== generation) {
+ return {
+ ok: false,
+ status: 409,
+ text: async (): Promise<string> =>
+ JSON.stringify({
+ detail: {
+ code: 'taken_over',
+ message: `connection conn-1 was reopened since incarnation ${claimed} and is now at ${generation}`,
+ },
+ }),
+ };
+ }
+ return { ok: true, status: 200, text: async (): Promise<string> => '' };
+ });
+ /** The server sweeps the connection for silence; its socket stays up
+ * client-side, the way a socket does across a sleep. */
+ const lapse = (): void => {
+ lapsed = true;
+ };
+ return { fetchMock, lapse };
+ }
+
+ it('sends no beat under the lapsed incarnation once it has asked for a reopen', async () => {
+ vi.useFakeTimers();
+ const server = lapsingServer();
+ const evicted: Eviction[] = [];
+ const { abort } = makeStream(server.fetchMock, {
+ rooms: [],
+ onEvicted: (eviction) => evicted.push(eviction),
+ });
+
+ await vi.advanceTimersByTimeAsync(BEAT_INTERVAL_MS / 2);
+ server.lapse();
+ await vi.advanceTimersByTimeAsync(30_000);
+
+ // The beat refused with a 404 is the last one under the lapsed
+ // incarnation: what follows it on the wire is the reopen, not another beat
+ // that the server could only take for someone else's.
+ const sent = server.fetchMock.mock.calls.map((call) =>
+ String(call[0]).includes('/events')
+ ? 'open'
+ : `beat@${(JSON.parse((call[1] as { body: string }).body) as { generation: number }).generation}`
+ );
+ const reopen = sent.indexOf('open', 1);
+ expect(reopen).toBeGreaterThan(0);
+ expect(sent.slice(1, reopen).filter((s) => s === 'beat@1')).toHaveLength(2);
+ expect(sent.slice(reopen + 1).every((s) => s === 'beat@2')).toBe(true);
+ expect(evicted).toEqual([]);
+ abort.abort();
+ });
+});
diff --git a/console/packages/switch-agent-runtime/src/event-stream.ts b/console/packages/switch-agent-runtime/src/event-stream.ts
index f71dd7eeb..5cea207f0 100644
--- a/console/packages/switch-agent-runtime/src/event-stream.ts
+++ b/console/packages/switch-agent-runtime/src/event-stream.ts
@@ -440,7 +440,15 @@ export class SwitchEventStream {
*/
private reopen(): void {
this.reopensWanted += 1;
- if (this.fence.admitting) this.socketAbort?.abort();
+ if (!this.fence.admitting) return;
+ // Held from this instant, not from when the stream loop gets round to its
+ // own `detaching`. Whoever asked for the reopen — the heartbeat, after a
+ // refusal — goes straight back to the gate, and could otherwise find it
+ // still open and send one more beat under the incarnation this reopen is
+ // about to replace. Arriving just behind the open, that beat is refused as
+ // a takeover and the client stands down over a connection it made itself.
+ this.fence.closeAdmission();
+ this.socketAbort?.abort();
}
/**
@@ -1042,6 +1050,11 @@ export class SwitchEventStream {
// beat could be sent under an incarnation later than the one recorded.
await Promise.race([this.fence.reached, until(signal, this.halt.signal)]);
if (signal.aborted || this.halt.signal.aborted) return;
+ // The gate this pass awaited may already have been closed by the time
+ // the await resumed: a reopen closes it synchronously, and the resolved
+ // promise we were handed is the old one. Checked here, with nothing
+ // awaited before the tick, so no beat leaves while admission is shut.
+ if (!this.fence.admitting) continue;
// A connection the server has just attached has never beaten, and has
// 6 seconds to. Whatever an earlier connection's failures built up says
// nothing about this one, so it starts at the base cadence.