Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(switchdash): model remote-host reachability as first-class state (CHOO-1682) (#82)
Louis Amaudruz496b5b9360a4
diff --git a/dash/AGENTS.md b/dash/AGENTS.mdindex a5952d706..a80a046f3 100644--- a/dash/AGENTS.md+++ b/dash/AGENTS.md@@ -248,7 +248,18 @@ pnpm run test- Migration tests run in the `migrations` project via `pnpm run test:migrations`.- Fixture generation runs in the `fixtures` project via `pnpm run db:fixtures`.- Renderer browser tests run in the `browser` project using Playwright-backed- `@vitest/browser-playwright`.+ `@vitest/browser-playwright`. **Agents: skip these — do not run or try to fix+ them.** They need a real Chromium and its system libraries (e.g. `libnspr4`),+ which dev VMs generally lack, so they fail for environment reasons unrelated+ to your change. CI is the gate for them. Run the node projects instead:++```bash+pnpm vitest run --project node --project main-db+```++ If `pnpm run test` reports failures only from the `browser` project (a+ Playwright launch error rather than an assertion), treat the run as green and+ say so — do not install browsers or chase the failure.- Main-process tests are colocated in `src/main/core/**/*.test.ts`.- Renderer unit tests live under `src/renderer/tests/`.- Renderer browser tests live under `src/renderer/tests/browser/`.diff --git a/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sql b/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sqlnew file mode 100644index 000000000..f411245a3--- /dev/null+++ b/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sql@@ -0,0 +1,9 @@+CREATE TABLE `remote_host_reachability` (+ `ssh_host` text PRIMARY KEY NOT NULL,+ `status` text DEFAULT 'unknown' NOT NULL,+ `last_error` text,+ `last_checked_at` text,+ `last_reachable_at` text,+ `consecutive_failures` integer DEFAULT 0 NOT NULL,+ `updated_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL+);diff --git a/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.json b/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.jsonnew file mode 100644index 000000000..55c5735ea--- /dev/null+++ b/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.json@@ -0,0 +1,773 @@+{+ "version": "6",+ "dialect": "sqlite",+ "id": "24ca41c8-3ae9-4b92-976a-08a6a6d4ab4c",+ "prevId": "4744931c-4640-42c4-9ac3-6b1b0946a0b4",+ "tables": {+ "agents": {+ "name": "agents",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "location_id": {+ "name": "location_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "provider_id": {+ "name": "provider_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "switch_agent_id": {+ "name": "switch_agent_id",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "api_endpoint": {+ "name": "api_endpoint",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "server_id": {+ "name": "server_id",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "auto_approve": {+ "name": "auto_approve",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": false+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_agents_location_id": {+ "name": "idx_agents_location_id",+ "columns": ["location_id"],+ "isUnique": false+ },+ "idx_agents_server_id": {+ "name": "idx_agents_server_id",+ "columns": ["server_id"],+ "isUnique": false+ }+ },+ "foreignKeys": {+ "agents_location_id_locations_id_fk": {+ "name": "agents_location_id_locations_id_fk",+ "tableFrom": "agents",+ "tableTo": "locations",+ "columnsFrom": ["location_id"],+ "columnsTo": ["id"],+ "onDelete": "no action",+ "onUpdate": "no action"+ },+ "agents_server_id_switch_servers_id_fk": {+ "name": "agents_server_id_switch_servers_id_fk",+ "tableFrom": "agents",+ "tableTo": "switch_servers",+ "columnsFrom": ["server_id"],+ "columnsTo": ["id"],+ "onDelete": "set null",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "app_secrets": {+ "name": "app_secrets",+ "columns": {+ "key": {+ "name": "key",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "secret": {+ "name": "secret",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ }+ },+ "indexes": {+ "idx_app_secrets_key": {+ "name": "idx_app_secrets_key",+ "columns": ["key"],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "app_settings": {+ "name": "app_settings",+ "columns": {+ "key": {+ "name": "key",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "value": {+ "name": "value",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "updated_at": {+ "name": "updated_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_app_settings_key": {+ "name": "idx_app_settings_key",+ "columns": ["key"],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "kv": {+ "name": "kv",+ "columns": {+ "key": {+ "name": "key",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "value": {+ "name": "value",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "updated_at": {+ "name": "updated_at",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_kv_key": {+ "name": "idx_kv_key",+ "columns": ["key"],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "location_settings": {+ "name": "location_settings",+ "columns": {+ "location_id": {+ "name": "location_id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "base_settings_json": {+ "name": "base_settings_json",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'{}'"+ },+ "shareable_settings_json": {+ "name": "shareable_settings_json",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'{}'"+ },+ "legacy_config_migrated_at": {+ "name": "legacy_config_migrated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {},+ "foreignKeys": {+ "location_settings_location_id_locations_id_fk": {+ "name": "location_settings_location_id_locations_id_fk",+ "tableFrom": "location_settings",+ "tableTo": "locations",+ "columnsFrom": ["location_id"],+ "columnsTo": ["id"],+ "onDelete": "cascade",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "locations": {+ "name": "locations",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "ssh_host": {+ "name": "ssh_host",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "''"+ },+ "dir": {+ "name": "dir",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_locations_host_dir": {+ "name": "idx_locations_host_dir",+ "columns": ["ssh_host", "dir"],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "messages": {+ "name": "messages",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "session_id": {+ "name": "session_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "content": {+ "name": "content",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "sender": {+ "name": "sender",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "timestamp": {+ "name": "timestamp",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "metadata": {+ "name": "metadata",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ }+ },+ "indexes": {+ "idx_messages_session_id": {+ "name": "idx_messages_session_id",+ "columns": ["session_id"],+ "isUnique": false+ },+ "idx_messages_timestamp": {+ "name": "idx_messages_timestamp",+ "columns": ["timestamp"],+ "isUnique": false+ }+ },+ "foreignKeys": {+ "messages_session_id_sessions_id_fk": {+ "name": "messages_session_id_sessions_id_fk",+ "tableFrom": "messages",+ "tableTo": "sessions",+ "columnsFrom": ["session_id"],+ "columnsTo": ["id"],+ "onDelete": "cascade",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "remote_host_reachability": {+ "name": "remote_host_reachability",+ "columns": {+ "ssh_host": {+ "name": "ssh_host",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'unknown'"+ },+ "last_error": {+ "name": "last_error",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_checked_at": {+ "name": "last_checked_at",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_reachable_at": {+ "name": "last_reachable_at",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "consecutive_failures": {+ "name": "consecutive_failures",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {},+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "remote_hosts": {+ "name": "remote_hosts",+ "columns": {+ "ssh_host": {+ "name": "ssh_host",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {},+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "sessions": {+ "name": "sessions",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "agent_id": {+ "name": "agent_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "title": {+ "name": "title",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "config": {+ "name": "config",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "shell_id": {+ "name": "shell_id",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "'system'"+ },+ "status": {+ "name": "status",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "agent_session_id": {+ "name": "agent_session_id",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "agent_status": {+ "name": "agent_status",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "agent_status_seen": {+ "name": "agent_status_seen",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false,+ "default": 1+ },+ "is_initial_session": {+ "name": "is_initial_session",+ "type": "integer",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "is_pinned": {+ "name": "is_pinned",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": 0+ },+ "archived_at": {+ "name": "archived_at",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "last_interacted_at": {+ "name": "last_interacted_at",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "status_changed_at": {+ "name": "status_changed_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_sessions_agent_id": {+ "name": "idx_sessions_agent_id",+ "columns": ["agent_id"],+ "isUnique": false+ }+ },+ "foreignKeys": {+ "sessions_agent_id_agents_id_fk": {+ "name": "sessions_agent_id_agents_id_fk",+ "tableFrom": "sessions",+ "tableTo": "agents",+ "columnsFrom": ["agent_id"],+ "columnsTo": ["id"],+ "onDelete": "cascade",+ "onUpdate": "no action"+ }+ },+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ },+ "switch_servers": {+ "name": "switch_servers",+ "columns": {+ "id": {+ "name": "id",+ "type": "text",+ "primaryKey": true,+ "notNull": true,+ "autoincrement": false+ },+ "name": {+ "name": "name",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "gateway_url": {+ "name": "gateway_url",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "api_url": {+ "name": "api_url",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false+ },+ "managed": {+ "name": "managed",+ "type": "integer",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": false+ },+ "management_kind": {+ "name": "management_kind",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "ssh_host": {+ "name": "ssh_host",+ "type": "text",+ "primaryKey": false,+ "notNull": false,+ "autoincrement": false+ },+ "created_at": {+ "name": "created_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ },+ "updated_at": {+ "name": "updated_at",+ "type": "text",+ "primaryKey": false,+ "notNull": true,+ "autoincrement": false,+ "default": "CURRENT_TIMESTAMP"+ }+ },+ "indexes": {+ "idx_switch_servers_gateway_url": {+ "name": "idx_switch_servers_gateway_url",+ "columns": ["gateway_url"],+ "isUnique": true+ }+ },+ "foreignKeys": {},+ "compositePrimaryKeys": {},+ "uniqueConstraints": {},+ "checkConstraints": {}+ }+ },+ "views": {},+ "enums": {},+ "_meta": {+ "schemas": {},+ "tables": {},+ "columns": {}+ },+ "internal": {+ "indexes": {}+ }+}diff --git a/dash/apps/switchdash-desktop/drizzle/meta/_journal.json b/dash/apps/switchdash-desktop/drizzle/meta/_journal.jsonindex 6ae857a92..bf8cac704 100644--- a/dash/apps/switchdash-desktop/drizzle/meta/_journal.json+++ b/dash/apps/switchdash-desktop/drizzle/meta/_journal.json@@ -302,6 +302,13 @@"when": 1785164441913,"tag": "0042_drop_agent_definition_name","breakpoints": true+ },+ {+ "idx": 43,+ "version": "6",+ "when": 1785224611278,+ "tag": "0043_furry_maggott",+ "breakpoints": true}]}diff --git a/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts b/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.tsindex 2a582bdca..ce3925e5d 100644--- a/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts+++ b/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts@@ -4,6 +4,9 @@ import { probeAgentSidecar } from '@main/core/agent-runtime/impl/ensure-agent-siimport type { SidecarEndpoint } from '@main/core/agent-runtime/impl/remote-sidecar-launcher';import { httpGetJsonOverChannel } from '@main/core/agent-runtime/impl/sidecar-http';import { sshConnectionIdForHost } from '@main/core/locations/location-transport';+import type { HostReachabilityChange } from '@main/core/remote-hosts/host-reachability-service';+import { HostUnreachableError } from '@main/core/remote-hosts/host-reachability-service';+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';import { sessionHooks } from '@main/core/sessions/session-hooks';import { sessionService } from '@main/core/sessions/session-service';import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';@@ -108,6 +111,23 @@ class RemoteSessionReconciler {sessionHooks.on('session:remote-terminated', ({ terminatedSessionId }) => {void this.handleRemoteTerminated(terminatedSessionId);});++ // A host coming back is the signal to resume the agents that were paused on+ // it — without this they would idle until their next 2s tick, which is+ // harmless, but ticking immediately makes recovery feel instant and keeps+ // the paused/resumed transition symmetric.+ hostReachabilityService.on('change', ({ current }: HostReachabilityChange) => {+ if (current.status === 'reachable') void this.onHostReachable(current.sshHost);+ });+ }++ /** Tick every agent whose location sits on a host that just became reachable. */+ private async onHostReachable(sshHost: string): Promise<void> {+ for (const agentId of [...this.timers.keys()]) {+ const agent = await getAgentById(agentId);+ const location = agent ? await getRemoteAgentLocation(agent) : null;+ if (location?.sshHost === sshHost) void this.tick(agentId);+ }}private async handleRemoteTerminated(sessionId: string): Promise<void> {@@ -227,7 +247,17 @@ class RemoteSessionReconciler {}),]);} catch (error) {- log.warn('RemoteSessionReconciler: reconcile failed', { agentId, error: String(error) });+ // A host that went unreachable mid-pass is already reported once, loudly,+ // by the reachability service. Repeating it here every 2s per agent is the+ // log flood CHOO-1682 is about.+ if (error instanceof HostUnreachableError) {+ log.debug('RemoteSessionReconciler: host unreachable — pass skipped', {+ agentId,+ sshHost: error.reachability.sshHost,+ });+ } else {+ log.warn('RemoteSessionReconciler: reconcile failed', { agentId, error: String(error) });+ }} finally {if (deadline) clearTimeout(deadline);this.inFlight.delete(agentId);@@ -250,6 +280,12 @@ class RemoteSessionReconciler {const sidecarKey = `${location.id}::${agent.name ?? agent.id}`;if (!this.claimSidecar(agentId, sidecarKey)) return;+ // The host is known to be down: skip the pass entirely rather than+ // attempting a connect every 2s per agent. Recovery is driven by the+ // reachability service's single backoff probe, and `onHostReachable` ticks+ // this agent the moment the host comes back, so nothing is lost by waiting.+ if (hostReachabilityService.isBlocked(location.sshHost)) return;+// The manager is mid-backoff rebuilding the transport: let it finish// rather than racing it with connect attempts every tick. Disconnected and// suspended states DO proceed — connectRemoteAgent's connect() is whatdiff --git a/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts b/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.tsindex d0114487d..a64fb7ad0 100644--- a/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts+++ b/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts@@ -1,5 +1,6 @@import { err, ok, type Result } from '@switchdash/shared';import type { IDisposable } from '@switchdash/shared';+import { HostUnreachableError } from '@main/core/remote-hosts/host-reachability-service';import { HookCore, type Hookable } from '@main/lib/hookable';import { LifecycleMap } from '@main/lib/lifecycle-map';import { log } from '@main/lib/logger';@@ -51,10 +52,13 @@ class LocationManager implements Hookable<LocationManagerHooks>, IDisposable {return ok(provider);} catch (e) {const initError = toLifecycleError(e);- log.error('LocationManager: error during location initialization', {- locationId: location.id,- ...initError,- });+ // A location on a host that is down cannot open, and that is an+ // expected condition the host's reachability state already reports —+ // logging it as an app error made every outage look like a defect.+ const message = 'LocationManager: error during location initialization';+ const metadata = { locationId: location.id, ...initError };+ if (e instanceof HostUnreachableError) log.warn(message, metadata);+ else log.error(message, metadata);return err(initError);}});diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.tsindex ed937db95..85e4f16c5 100644--- a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts@@ -23,10 +23,12 @@ import {getRemoteSwitchSetupService,type RemoteSwitchSetupService,} from '@main/core/switch-setup/remote-switch-setup';+import { hostBlockedReason, type HostReachability } from '@shared/core/remote-hosts/reachability';import type { ConnectionState, SshHealthState } from '@shared/core/ssh/ssh';import { createRPCController } from '@shared/lib/ipc/rpc';import type { SwitchAgentConfig } from '@shared/switch-agents';import { listSshConfigHosts } from './list-ssh-config-hosts';+import { hostReachabilityService } from './production-host-reachability';import { listRemoteHosts, removeRemoteHost, upsertRemoteHost, type RemoteHost } from './store';/** A single dependency's status on a remote host, enriched for the UI. */@@ -89,15 +91,16 @@ async function probeGhAuth(sshHost: string): Promise<GhAuthStatus> {}}+/**+ * Ad-hoc reachability check. Routed through the reachability service rather+ * than connecting directly, so a manual "Test connection" and the background+ * probe share one code path — and a successful test immediately un-pauses the+ * host-dependent work that was gated on the old state.+ */async function testConnection(sshHost: string): Promise<TestConnectionResult> {- try {- const proxy = await ensureSshConnected(sshConnectionIdForHost(sshHost), sshHost);- const ctx = new SshExecutionContext(proxy);- await ctx.exec('uname', ['-s']);- return { ok: true };- } catch (error) {- return { ok: false, message: error instanceof Error ? error.message : String(error) };- }+ const reachability = await hostReachabilityService.checkNow(sshHost);+ if (reachability.status === 'reachable') return { ok: true };+ return { ok: false, message: reachability.lastError ?? hostBlockedReason(reachability) };}async function probeDeps(sshHost: string): Promise<RemoteDependencyView[]> {@@ -241,13 +244,36 @@ export const remoteHostsController = createRPCController({getConnectionStatus: (sshHost: string): Promise<HostConnectionStatus> =>Promise.resolve(hostConnectionStatus(sshHost)),+ /** Modeled reachability for one host — the state the UI gates its display on. */+ getReachability: (sshHost: string): Promise<HostReachability> =>+ Promise.resolve(hostReachabilityService.get(sshHost)),++ /** Every host the reachability model knows about, for the initial UI hydrate. */+ listReachability: (): Promise<HostReachability[]> =>+ Promise.resolve(hostReachabilityService.getAll()),++ /**+ * Probe now, bypassing the backoff — the "Retry connection" button. On+ * success this clears the blocked state, which resumes every paused+ * host-dependent path (reconciler included) rather than only fixing the+ * screen the user is looking at.+ */+ retryHost: (sshHost: string): Promise<HostReachability> =>+ hostReachabilityService.checkNow(sshHost),+/*** Force a full transport rebuild for the host's pooled connection — the* manual recovery path for a wedged or given-up connection. Returns the* post-rebuild status.*/reconnectHost: async (sshHost: string): Promise<HostConnectionStatus> => {- await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);+ try {+ await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);+ } catch (error) {+ hostReachabilityService.reportFailure(sshHost, error);+ throw error;+ }+ hostReachabilityService.reportSuccess(sshHost);return hostConnectionStatus(sshHost);},diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.tsnew file mode 100644index 000000000..63384d7f4--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.ts@@ -0,0 +1,196 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';+import type { HostReachability } from '@shared/core/remote-hosts/reachability';+import {+ HostReachabilityService,+ HostUnreachableError,+ probeDelayFor,+} from './host-reachability-service';++vi.mock('./reachability-store', () => ({+ listPersistedReachability: vi.fn(async () => []),+ savePersistedReachability: vi.fn(async () => {}),+ hydrate: (record: unknown) => ({+ ...(record as object),+ nextProbeAt: null,+ probing: false,+ }),+}));++class AuthError extends Error {}++function makeService(probe: (sshHost: string) => Promise<void>) {+ const published: HostReachability[] = [];+ const timers: Array<{ fn: () => void; ms: number }> = [];+ const service = new HostReachabilityService({+ probe,+ isAuthError: (error) => error instanceof AuthError,+ publish: (reachability) => published.push(reachability),+ log: { info: () => {}, warn: () => {} },+ setTimer: (fn, ms) => {+ timers.push({ fn, ms });+ return timers.length as unknown as NodeJS.Timeout;+ },+ clearTimer: () => {},+ now: () => 1_700_000_000_000,+ });+ return { service, published, timers };+}++/** Flush the microtask queue so fire-and-forget transitions settle. */+const settle = () => new Promise((resolve) => setImmediate(resolve));++describe('HostReachabilityService', () => {+ let service: HostReachabilityService;++ beforeEach(() => {+ vi.clearAllMocks();+ });++ it('treats a never-probed host as unknown and lets work through', () => {+ ({ service } = makeService(async () => {}));+ expect(service.get('vm').status).toBe('unknown');+ expect(service.isBlocked('vm')).toBe(false);+ expect(() => service.requireReachable('vm')).not.toThrow();+ });++ it('blocks work and throws the modeled reason once a host is unreachable', async () => {+ ({ service } = makeService(async () => {+ throw new Error('Connection lost before handshake');+ }));++ await service.checkNow('vm');++ expect(service.get('vm').status).toBe('unreachable');+ expect(service.isBlocked('vm')).toBe(true);+ expect(() => service.requireReachable('vm')).toThrow(HostUnreachableError);+ expect(() => service.requireReachable('vm')).toThrow(/Connection lost before handshake/);+ });++ it('does not auto-probe an auth failure — a rejected key never self-heals', async () => {+ const { service: svc, timers } = makeService(async () => {+ throw new AuthError('All configured authentication methods failed');+ });++ await svc.checkNow('vm');++ expect(svc.get('vm').status).toBe('suspended');+ expect(timers).toHaveLength(0);+ });++ it('backs off between probes instead of retrying at a fixed interval', async () => {+ const { service: svc, timers } = makeService(async () => {+ throw new Error('unreachable');+ });++ await svc.checkNow('vm');+ expect(timers.at(-1)?.ms).toBe(1_000);++ await svc.checkNow('vm');+ expect(timers.at(-1)?.ms).toBe(5_000);++ await svc.checkNow('vm');+ expect(timers.at(-1)?.ms).toBe(15_000);+ });++ it('probes tightly at first, then caps at five minutes', () => {+ expect(probeDelayFor(1)).toBe(1_000);+ expect(probeDelayFor(4)).toBe(30_000);+ expect(probeDelayFor(6)).toBe(300_000);+ expect(probeDelayFor(50)).toBe(300_000);+ });++ it('clears the blocked state and emits a change when the host recovers', async () => {+ let reachable = false;+ const { service: svc } = makeService(async () => {+ if (!reachable) throw new Error('down');+ });+ const changes: string[] = [];+ svc.on('change', ({ current }) => changes.push(current.status));++ await svc.checkNow('vm');+ expect(svc.isBlocked('vm')).toBe(true);++ reachable = true;+ await svc.checkNow('vm');++ expect(svc.get('vm').status).toBe('reachable');+ expect(svc.isBlocked('vm')).toBe(false);+ expect(svc.get('vm').consecutiveFailures).toBe(0);+ expect(svc.get('vm').lastError).toBeNull();+ expect(changes).toEqual(['unreachable', 'reachable']);+ });++ it('coalesces concurrent probes onto one round trip', async () => {+ const probe = vi.fn(async () => {});+ const { service: svc } = makeService(probe);++ await Promise.all([svc.checkNow('vm'), svc.checkNow('vm'), svc.checkNow('vm')]);++ expect(probe).toHaveBeenCalledTimes(1);+ });++ it('probes each host independently', async () => {+ const { service: svc } = makeService(async (sshHost) => {+ if (sshHost === 'bad') throw new Error('down');+ });++ await svc.checkNow('good');+ await svc.checkNow('bad');++ expect(svc.isBlocked('good')).toBe(false);+ expect(svc.isBlocked('bad')).toBe(true);+ });++ it('lets ordinary traffic keep the record fresh without a probe', async () => {+ const probe = vi.fn(async () => {+ throw new Error('down');+ });+ const { service: svc } = makeService(probe);++ await svc.checkNow('vm');+ expect(svc.isBlocked('vm')).toBe(true);++ svc.reportSuccess('vm');+ await settle();++ expect(svc.get('vm').status).toBe('reachable');+ expect(probe).toHaveBeenCalledTimes(1);+ });++ it('reports a transport failure from real work as unreachable', async () => {+ const { service: svc } = makeService(async () => {});++ svc.reportFailure('vm', new Error('Connection lost before handshake'));+ await settle();++ expect(svc.get('vm').status).toBe('unreachable');+ expect(svc.get('vm').lastError).toBe('Connection lost before handshake');+ });++ it('folds a live SSH connection into the host model', async () => {+ const { service: svc } = makeService(async () => {+ throw new Error('down');+ });++ await svc.checkNow('vm');+ svc.handleSshConnectionEvent('vm', { type: 'connected', connectionId: 'agent-ssh:vm' });+ await settle();++ expect(svc.get('vm').status).toBe('reachable');+ });++ it('re-probes blocked hosts after a system resume', async () => {+ const probe = vi.fn(async () => {+ throw new Error('down');+ });+ const { service: svc } = makeService(probe);++ await svc.checkNow('vm');+ probe.mockClear();++ svc.handleSystemResume();+ await settle();++ expect(probe).toHaveBeenCalledWith('vm');+ });+});diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.tsnew file mode 100644index 000000000..1689a48c9--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.ts@@ -0,0 +1,365 @@+import { EventEmitter } from 'node:events';+import {+ hostBlockedReason,+ type HostReachability,+ type HostReachabilityStatus,+ isHostBlocked,+ unknownHostReachability,+} from '@shared/core/remote-hosts/reachability';+import type { SshConnectionEvent } from '@shared/core/ssh/sshEvents';+import {+ hydrate,+ listPersistedReachability,+ savePersistedReachability,+ type PersistedHostReachability,+} from './reachability-store';++/**+ * Backoff schedule between probes of an unreachable host, capped at five+ * minutes and repeated.+ *+ * The early steps stay tight because the common cause is a credential the user+ * is actively re-establishing (VPN, `aws sso login`) and they expect the app to+ * notice quickly; the long tail keeps a host that is genuinely gone from+ * costing anything. Manual retry short-circuits the whole schedule, so the cap+ * only governs the unattended case.+ */+const PROBE_BACKOFF_MS = [1_000, 5_000, 15_000, 30_000, 60_000, 300_000];++export function probeDelayFor(consecutiveFailures: number): number {+ const step = Math.min(Math.max(consecutiveFailures, 1), PROBE_BACKOFF_MS.length);+ return PROBE_BACKOFF_MS[step - 1]!;+}++/** Probes a host's transport. Resolves on success, throws on failure. */+export type HostProbe = (sshHost: string) => Promise<void>;++/** Distinguishes a rejected credential (no point retrying) from a dead network. */+export type AuthErrorPredicate = (error: unknown) => boolean;++export type HostReachabilityLog = {+ info: (message: string, metadata?: Record<string, unknown>) => void;+ warn: (message: string, metadata?: Record<string, unknown>) => void;+};++export type HostReachabilityServiceDeps = {+ probe: HostProbe;+ isAuthError: AuthErrorPredicate;+ publish: (reachability: HostReachability) => void;+ log: HostReachabilityLog;+ /** Injected so tests can drive the backoff without real timers. */+ setTimer?: (fn: () => void, ms: number) => NodeJS.Timeout;+ clearTimer?: (timer: NodeJS.Timeout) => void;+ now?: () => number;+};++type HostEntry = {+ reachability: HostReachability;+ timer?: NodeJS.Timeout;+ /** In-flight probe, so concurrent callers coalesce onto one round trip. */+ inFlight?: Promise<HostReachability>;+};++/**+ * Raised when host-dependent work is attempted against a host known to be+ * unreachable. Carries the reachability record so callers (and the UI) can+ * render the modeled state rather than re-deriving it from a transport error.+ */+export class HostUnreachableError extends Error {+ readonly reachability: HostReachability;++ constructor(reachability: HostReachability) {+ super(hostBlockedReason(reachability));+ this.name = 'HostUnreachableError';+ this.reachability = reachability;+ }+}++export type HostReachabilityChange = {+ previous: HostReachabilityStatus;+ current: HostReachability;+};++/**+ * The single source of truth for whether a remote SSH host can be reached.+ *+ * One probe loop per host, shared by every consumer: ten agents on a dead host+ * produce one probe per backoff step, not ten reconnect storms. Consumers ask+ * `isBlocked`/`requireReachable` before doing host-dependent work and subscribe+ * to `change` to resume when a host comes back.+ */+export class HostReachabilityService extends EventEmitter {+ private readonly deps: Required<HostReachabilityServiceDeps>;+ private readonly hosts = new Map<string, HostEntry>();+ private loaded = false;++ constructor(deps: HostReachabilityServiceDeps) {+ super();+ this.deps = {+ setTimer: (fn, ms) => setTimeout(fn, ms),+ clearTimer: (timer) => clearTimeout(timer),+ now: () => Date.now(),+ ...deps,+ };+ }++ /**+ * Load persisted state so the app boots knowing which hosts were down. Hosts+ * that were unreachable at shutdown get a probe scheduled rather than being+ * hammered by every consumer at once on startup.+ */+ async initialize(): Promise<void> {+ if (this.loaded) return;+ this.loaded = true;+ const persisted = await listPersistedReachability();+ for (const record of persisted) {+ const reachability = hydrate(record);+ this.hosts.set(record.sshHost, { reachability });+ if (isHostBlocked(reachability) && reachability.status !== 'suspended') {+ this.scheduleProbe(record.sshHost);+ }+ }+ this.deps.log.info('HostReachabilityService: restored persisted host reachability', {+ hosts: persisted.length,+ });+ }++ get(sshHost: string): HostReachability {+ return this.entry(sshHost).reachability;+ }++ getAll(): HostReachability[] {+ return [...this.hosts.values()].map((entry) => entry.reachability);+ }++ /** True when host-dependent work should be held back for this host. */+ isBlocked(sshHost: string): boolean {+ return isHostBlocked(this.get(sshHost));+ }++ /**+ * Gate for host-dependent work. Throws `HostUnreachableError` immediately —+ * with the modeled reason — instead of letting the caller discover the+ * failure through a 20-second SSH timeout.+ */+ requireReachable(sshHost: string): void {+ const reachability = this.get(sshHost);+ if (isHostBlocked(reachability)) throw new HostUnreachableError(reachability);+ }++ /**+ * Record that a host was just reached successfully by real work (not a+ * probe). Lets ordinary traffic keep the record fresh so a busy host is+ * never probed redundantly.+ */+ reportSuccess(sshHost: string): void {+ const entry = this.entry(sshHost);+ if (entry.reachability.status === 'reachable') return;+ this.cancelProbe(entry);+ void this.transition(sshHost, {+ status: 'reachable',+ lastError: null,+ consecutiveFailures: 0,+ });+ }++ /**+ * Record that host-dependent work failed at the transport level. Moves the+ * host into `unreachable` (or `suspended` for auth) and starts the shared+ * backoff probe, so the failing caller does not have to retry on its own.+ */+ reportFailure(sshHost: string, error: unknown): void {+ const message = error instanceof Error ? error.message : String(error);+ const entry = this.entry(sshHost);+ const status: HostReachabilityStatus = this.deps.isAuthError(error)+ ? 'suspended'+ : 'unreachable';+ void this.transition(sshHost, {+ status,+ lastError: message,+ consecutiveFailures: entry.reachability.consecutiveFailures + 1,+ }).then(() => {+ if (status === 'unreachable') this.scheduleProbe(sshHost);+ });+ }++ /**+ * Probe now, bypassing the backoff — the "Retry connection" path. Also used+ * to validate a host before it is adopted (e.g. the add-agent modal), so+ * validation and background recovery share one code path.+ */+ async checkNow(sshHost: string): Promise<HostReachability> {+ const entry = this.entry(sshHost);+ if (entry.inFlight) return entry.inFlight;+ this.cancelProbe(entry);++ const run = (async (): Promise<HostReachability> => {+ this.setProbing(sshHost, true);+ try {+ await this.deps.probe(sshHost);+ return await this.transition(sshHost, {+ status: 'reachable',+ lastError: null,+ consecutiveFailures: 0,+ });+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ const status: HostReachabilityStatus = this.deps.isAuthError(error)+ ? 'suspended'+ : 'unreachable';+ const next = await this.transition(sshHost, {+ status,+ lastError: message,+ consecutiveFailures: this.entry(sshHost).reachability.consecutiveFailures + 1,+ });+ if (status === 'unreachable') this.scheduleProbe(sshHost);+ return next;+ } finally {+ this.entry(sshHost).inFlight = undefined;+ this.setProbing(sshHost, false);+ }+ })();++ entry.inFlight = run;+ return run;+ }++ /**+ * Fold the pooled connection's own lifecycle into the host model, so a host+ * that recovers (or dies) through ordinary session traffic updates the shared+ * state without waiting for the next probe.+ */+ handleSshConnectionEvent(sshHost: string, event: SshConnectionEvent): void {+ if (event.type === 'connected' || event.type === 'reconnected') {+ this.reportSuccess(sshHost);+ return;+ }+ if (event.type === 'reconnect-failed') {+ void this.transition(sshHost, {+ status: 'suspended',+ lastError: this.get(sshHost).lastError ?? 'SSH authentication failed',+ consecutiveFailures: this.get(sshHost).consecutiveFailures + 1,+ });+ return;+ }+ if (event.type === 'error') {+ this.reportFailure(sshHost, new Error(event.errorMessage));+ }+ }++ /** After a system resume every host is suspect — re-probe the blocked ones. */+ handleSystemResume(): void {+ for (const [sshHost, entry] of this.hosts) {+ if (isHostBlocked(entry.reachability)) void this.checkNow(sshHost);+ }+ }++ /** Stop all timers (app shutdown / tests). */+ dispose(): void {+ for (const entry of this.hosts.values()) this.cancelProbe(entry);+ }++ private entry(sshHost: string): HostEntry {+ let entry = this.hosts.get(sshHost);+ if (!entry) {+ entry = { reachability: unknownHostReachability(sshHost) };+ this.hosts.set(sshHost, entry);+ }+ return entry;+ }++ private setProbing(sshHost: string, probing: boolean): void {+ const entry = this.entry(sshHost);+ if (entry.reachability.probing === probing) return;+ entry.reachability = { ...entry.reachability, probing };+ this.deps.publish(entry.reachability);+ }++ private async transition(+ sshHost: string,+ patch: Pick<HostReachability, 'status' | 'lastError' | 'consecutiveFailures'>+ ): Promise<HostReachability> {+ const entry = this.entry(sshHost);+ const previous = entry.reachability.status;+ const checkedAt = new Date(this.deps.now()).toISOString();++ const next: HostReachability = {+ ...entry.reachability,+ ...patch,+ sshHost,+ lastCheckedAt: checkedAt,+ lastReachableAt:+ patch.status === 'reachable' ? checkedAt : entry.reachability.lastReachableAt,+ nextProbeAt: patch.status === 'reachable' ? null : entry.reachability.nextProbeAt,+ };+ entry.reachability = next;++ // Log only on a status change: an unreachable host must be loud once, not+ // once per retry — the log spam was half of what CHOO-1682 reports.+ if (previous !== next.status) {+ const metadata = { sshHost, previous, status: next.status, error: next.lastError };+ if (next.status === 'reachable') {+ this.deps.log.info('HostReachabilityService: host reachable', metadata);+ } else {+ this.deps.log.warn('HostReachabilityService: host state changed', metadata);+ }+ }++ this.deps.publish(next);+ if (previous !== next.status) {+ this.emit('change', { previous, current: next } satisfies HostReachabilityChange);+ }++ await this.persist(next);+ return next;+ }++ private async persist(reachability: HostReachability): Promise<void> {+ const record: PersistedHostReachability = {+ sshHost: reachability.sshHost,+ status: reachability.status,+ lastError: reachability.lastError,+ lastCheckedAt: reachability.lastCheckedAt,+ lastReachableAt: reachability.lastReachableAt,+ consecutiveFailures: reachability.consecutiveFailures,+ };+ try {+ await savePersistedReachability(record);+ } catch (error) {+ // Persistence is a convenience (surviving restart); losing it must not+ // take down the in-memory model every consumer depends on.+ this.deps.log.warn('HostReachabilityService: failed to persist reachability', {+ sshHost: reachability.sshHost,+ error: error instanceof Error ? error.message : String(error),+ });+ }+ }++ private scheduleProbe(sshHost: string): void {+ const entry = this.entry(sshHost);+ this.cancelProbe(entry);+ if (entry.reachability.status === 'suspended') return;++ const delayMs = probeDelayFor(entry.reachability.consecutiveFailures);+ entry.reachability = {+ ...entry.reachability,+ nextProbeAt: new Date(this.deps.now() + delayMs).toISOString(),+ };+ this.deps.publish(entry.reachability);++ entry.timer = this.deps.setTimer(() => {+ entry.timer = undefined;+ void this.checkNow(sshHost);+ }, delayMs);+ }++ private cancelProbe(entry: HostEntry): void {+ if (entry.timer) {+ this.deps.clearTimer(entry.timer);+ entry.timer = undefined;+ }+ if (entry.reachability.nextProbeAt !== null) {+ entry.reachability = { ...entry.reachability, nextProbeAt: null };+ }+ }+}diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.tsnew file mode 100644index 000000000..5d0f446bc--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.ts@@ -0,0 +1,44 @@+import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';+import type { SshConnectionManagerEvent } from '@main/core/ssh/lifecycle/ssh-connection-manager';+import { hostReachabilityService } from './production-host-reachability';++/** Reverse of `sshConnectionIdForHost`; null for ids the pool didn't mint for a host. */+function hostForConnectionId(connectionId: string): string | null {+ const prefix = 'agent-ssh:';+ return connectionId.startsWith(prefix) ? connectionId.slice(prefix.length) : null;+}++/**+ * Load persisted host reachability and bind the pooled SSH connection's+ * lifecycle into it, so state stays fresh from ordinary traffic rather than+ * only from probes.+ */+export async function initializeHostReachability(): Promise<void> {+ await hostReachabilityService.initialize();++ sshConnectionManager.on('connection-event', (event: SshConnectionManagerEvent) => {+ const sshHost = hostForConnectionId(event.connectionId);+ if (!sshHost) return;+ if (event.type === 'connected' || event.type === 'reconnected') {+ hostReachabilityService.handleSshConnectionEvent(sshHost, {+ type: event.type,+ connectionId: event.connectionId,+ });+ return;+ }+ if (event.type === 'reconnect-failed') {+ hostReachabilityService.handleSshConnectionEvent(sshHost, {+ type: 'reconnect-failed',+ connectionId: event.connectionId,+ });+ return;+ }+ if (event.type === 'error') {+ hostReachabilityService.handleSshConnectionEvent(sshHost, {+ type: 'error',+ connectionId: event.connectionId,+ errorMessage: event.error.message,+ });+ }+ });+}diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.tsnew file mode 100644index 000000000..d0a91e6d5--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.ts@@ -0,0 +1,26 @@+import { SshExecutionContext } from '@main/core/execution-context/ssh-execution-context';+import { sshConnectionIdForHost } from '@main/core/locations/location-transport';+import { forceSshReconnect } from '@main/core/ssh/connect/connect-agent-ssh';+import { SshAuthError } from '@main/core/ssh/lifecycle/ssh-connection-manager';+import { events } from '@main/lib/events';+import { log } from '@main/lib/logger';+import { hostReachabilityEventChannel } from '@shared/core/remote-hosts/reachability';+import { HostReachabilityService } from './host-reachability-service';++/**+ * The reachability probe: rebuild the host's transport and run a trivial+ * command. `forceSshReconnect` rather than `ensureSshConnected` because a probe+ * must answer "can I reach this host *now*", not "is there a pooled connection+ * object" — a wedged connection would otherwise report success.+ */+async function probeHost(sshHost: string): Promise<void> {+ const proxy = await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);+ await new SshExecutionContext(proxy).exec('uname', ['-s']);+}++export const hostReachabilityService = new HostReachabilityService({+ probe: probeHost,+ isAuthError: (error) => error instanceof SshAuthError,+ publish: (reachability) => events.emit(hostReachabilityEventChannel, reachability),+ log,+});diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.tsnew file mode 100644index 000000000..ec036736c--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.ts@@ -0,0 +1,77 @@+import { eq } from 'drizzle-orm';+import { db } from '@main/db/client';+import { remoteHostReachability, type RemoteHostReachabilityRow } from '@main/db/schema';+import {+ type HostReachability,+ type HostReachabilityStatus,+ unknownHostReachability,+} from '@shared/core/remote-hosts/reachability';++/** Persisted shape of a reachability record — everything except live probe state. */+export type PersistedHostReachability = Omit<HostReachability, 'nextProbeAt' | 'probing'>;++const STATUSES: HostReachabilityStatus[] = ['unknown', 'reachable', 'unreachable', 'suspended'];++function toStatus(raw: string, sshHost: string): HostReachabilityStatus {+ const status = STATUSES.find((s) => s === raw);+ if (!status) {+ throw new Error(`Unknown reachability status '${raw}' persisted for host ${sshHost}`);+ }+ return status;+}++function toRecord(row: RemoteHostReachabilityRow): PersistedHostReachability {+ return {+ sshHost: row.sshHost,+ status: toStatus(row.status, row.sshHost),+ lastError: row.lastError,+ lastCheckedAt: row.lastCheckedAt,+ lastReachableAt: row.lastReachableAt,+ consecutiveFailures: row.consecutiveFailures,+ };+}++export async function listPersistedReachability(): Promise<PersistedHostReachability[]> {+ const rows = await db.select().from(remoteHostReachability);+ return rows.map(toRecord);+}++export async function getPersistedReachability(+ sshHost: string+): Promise<PersistedHostReachability | null> {+ const [row] = await db+ .select()+ .from(remoteHostReachability)+ .where(eq(remoteHostReachability.sshHost, sshHost))+ .limit(1);+ return row ? toRecord(row) : null;+}++export async function savePersistedReachability(record: PersistedHostReachability): Promise<void> {+ const values = {+ sshHost: record.sshHost,+ status: record.status,+ lastError: record.lastError,+ lastCheckedAt: record.lastCheckedAt,+ lastReachableAt: record.lastReachableAt,+ consecutiveFailures: record.consecutiveFailures,+ updatedAt: new Date().toISOString(),+ };+ await db+ .insert(remoteHostReachability)+ .values(values)+ .onConflictDoUpdate({ target: remoteHostReachability.sshHost, set: values });+}++export async function deletePersistedReachability(sshHost: string): Promise<void> {+ await db.delete(remoteHostReachability).where(eq(remoteHostReachability.sshHost, sshHost));+}++/** A persisted record widened back to the full live shape (no probe scheduled yet). */+export function hydrate(record: PersistedHostReachability): HostReachability {+ return { ...record, nextProbeAt: null, probing: false };+}++export function emptyFor(sshHost: string): HostReachability {+ return unknownHostReachability(sshHost);+}diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.tsindex ccd10ea25..574c60d80 100644--- a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts@@ -1,30 +1,40 @@+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';import type { SshClientProxy } from '@main/core/ssh/lifecycle/ssh-client-proxy';-import { resolveSshConnectConfig } from './resolve-ssh-connect-config';+import { registerSshResolver } from './ssh-resolver';/**- * Register the per-id config resolver (the host's `~/.ssh/config` alias →- * ssh2 ConnectConfig) and establish the pooled connection. The resolver is- * reused for auto-reconnect; `connect` coalesces concurrent calls and reuses a- * live connection, so calling this repeatedly for the same host is safe.+ * Establish (or reuse) the pooled connection to a host, gated on the host's+ * modeled reachability.*- * host/port/username in the transient config are fallbacks: the real values- * (and identity/agent) are resolved from the alias via `ssh -G`. Auth goes- * through the SSH agent, matching switchdash's remote stack — switchdash stores no- * credentials of its own.+ * This is the single funnel every remote capability goes through — exec+ * contexts, SFTP, terminals, PTYs, sidecar deploys, dependency probes, the+ * managed server — so gating here is what centralizes reachability for all of+ * them (CHOO-1682). A host known to be unreachable fails fast with the modeled+ * reason instead of each caller rediscovering it through a 20s SSH timeout, and+ * every real connect outcome feeds the shared host state so ordinary traffic+ * keeps it fresh without extra probes.*/export async function ensureSshConnected(connectionId: string,sshHost: string): Promise<SshClientProxy> {+ hostReachabilityService.requireReachable(sshHost);registerSshResolver(connectionId, sshHost);- return sshConnectionManager.connect(connectionId);+ try {+ const proxy = await sshConnectionManager.connect(connectionId);+ hostReachabilityService.reportSuccess(sshHost);+ return proxy;+ } catch (error) {+ hostReachabilityService.reportFailure(sshHost, error);+ throw error;+ }}/*** Force a full transport rebuild for a host's pooled connection — the manual- * recovery path behind the UI's refresh. Registers the resolver first so it- * also works for a host that has not connected yet this app run.+ * recovery path behind the UI's refresh. Deliberately NOT gated on+ * reachability: forcing a reconnect is how an unreachable host gets retried.*/export async function forceSshReconnect(connectionId: string,@@ -33,19 +43,3 @@ export async function forceSshReconnect(registerSshResolver(connectionId, sshHost);return sshConnectionManager.forceReconnect(connectionId);}--function registerSshResolver(connectionId: string, sshHost: string): void {- sshConnectionManager.register(connectionId, () =>- resolveSshConnectConfig({- kind: 'transient',- config: {- name: sshHost,- host: sshHost,- port: 22,- username: '',- sshConfigAlias: sshHost,- authType: 'agent',- },- })- );-}diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.tsnew file mode 100644index 000000000..fd840a3da--- /dev/null+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.ts@@ -0,0 +1,28 @@+import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';+import { resolveSshConnectConfig } from './resolve-ssh-connect-config';++/**+ * Register the per-id config resolver (the host's `~/.ssh/config` alias →+ * ssh2 ConnectConfig). Lives apart from `connect-agent-ssh` so the reachability+ * probe can rebuild a transport without importing the reachability-gated+ * connect helpers that consume it.+ *+ * host/port/username in the transient config are fallbacks: the real values+ * (and identity/agent) are resolved from the alias via `ssh -G`. Auth goes+ * through the SSH agent — switchdash stores no credentials of its own.+ */+export function registerSshResolver(connectionId: string, sshHost: string): void {+ sshConnectionManager.register(connectionId, () =>+ resolveSshConnectConfig({+ kind: 'transient',+ config: {+ name: sshHost,+ host: sshHost,+ port: 22,+ username: '',+ sshConfigAlias: sshHost,+ authType: 'agent',+ },+ })+ );+}diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.tsindex a2540d9ad..434cc52a1 100644--- a/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts@@ -221,7 +221,9 @@ export class SshConnectionManager extends EventEmitter {*/async forceReconnect(id: string): Promise<SshClientProxy> {const record = this.record(id);- this.deps.log.warn('SshConnectionManager: force reconnect requested', { connectionId: id });+ // Routine: this is also the reachability probe's transport rebuild, so it+ // recurs on the backoff schedule for an unreachable host.+ this.deps.log.info('SshConnectionManager: force reconnect requested', { connectionId: id });record.intentional = false;this.cancelReconnect(record);if (record.client) {@@ -517,7 +519,11 @@ export class SshConnectionManager extends EventEmitter {};client.on('error', (error: Error) => {- this.deps.log.error('SshConnectionManager: connection error', {+ // A remote host being down is an expected condition, not a fault in the+ // app: laptops sleep, VPNs drop, SSO tokens expire. The host's+ // reachability state is what escalates a persistent outage — this is+ // one transport attempt failing, so warn rather than error.+ this.deps.log.warn('SshConnectionManager: connection error', {connectionId: id,error: error.message,});@@ -683,7 +689,10 @@ export class SshConnectionManager extends EventEmitter {// Auth failures won't resolve with blind retries — suspend instead of// hammering the server. connect()/forceReconnect()/system resume revive.if (error instanceof SshAuthError) {- this.deps.log.error('SshConnectionManager: reconnect suspended — auth failure', {+ // Expected (expired or rotated credentials), and the host's+ // reachability state surfaces it to the user as `suspended` — the log+ // does not need to shout about it.+ this.deps.log.warn('SshConnectionManager: reconnect suspended — auth failure', {connectionId: id,});record.reconnect = undefined;diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts b/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.tsindex a9bcd5bda..3cb138943 100644--- a/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts+++ b/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts@@ -1,10 +1,23 @@+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';+import { log } from '@main/lib/logger';import { createRPCController } from '@shared/lib/ipc/rpc';import { getRemoteSwitchSetupService } from './remote-switch-setup';import { switchSetupService } from './switch-setup-service';export const switchSetupController = createRPCController({listOnboardable: () => switchSetupService.listOnboardable(),+ /**+ * Agent types installed on a remote host. An unreachable host has no+ * answerable list, so return none rather than throwing: this is a query that+ * paints UI, and its caller renders the host-unreachable state alongside it,+ * so the degraded result is disclosed. Throwing surfaced an ordinary,+ * expected condition as an unhandled handler error with a stack trace.+ */listOnboardableRemote: async (sshHost: string) => {+ if (hostReachabilityService.isBlocked(sshHost)) {+ log.warn('switchSetup.listOnboardableRemote: host unreachable — no agent types', { sshHost });+ return [];+ }const service = await getRemoteSwitchSetupService(sshHost);const statuses = await service.listAgentTypeStatuses();return statuses.filter((s) => s.installed).map((s) => ({ agentId: s.agentId }));diff --git a/dash/apps/switchdash-desktop/src/main/db/schema.ts b/dash/apps/switchdash-desktop/src/main/db/schema.tsindex b984c5b6e..3b9c90b89 100644--- a/dash/apps/switchdash-desktop/src/main/db/schema.ts+++ b/dash/apps/switchdash-desktop/src/main/db/schema.ts@@ -198,6 +198,30 @@ export const remoteHosts = sqliteTable('remote_hosts', {.default(sql`CURRENT_TIMESTAMP`),});+/**+ * Persisted reachability of a remote SSH host (CHOO-1682). Keyed by the same+ * `~/.ssh/config` alias as `remote_hosts` and `locations.ssh_host`, but kept in+ * its own table because a location can name a host that was never onboarded —+ * reachability must be tracked for any alias we actually talk to, not only the+ * ones on the remote-hosts page.+ *+ * Persisting it means a restart does not forget that a host was down: the app+ * boots into the known-bad state and schedules a probe, instead of every+ * host-dependent path racing to rediscover the failure at once.+ */+export const remoteHostReachability = sqliteTable('remote_host_reachability', {+ sshHost: text('ssh_host').primaryKey(),+ /** One of HostReachabilityStatus — 'unknown' | 'reachable' | 'unreachable' | 'suspended'. */+ status: text('status').notNull().default('unknown'),+ lastError: text('last_error'),+ lastCheckedAt: text('last_checked_at'),+ lastReachableAt: text('last_reachable_at'),+ consecutiveFailures: integer('consecutive_failures').notNull().default(0),+ updatedAt: text('updated_at')+ .notNull()+ .default(sql`CURRENT_TIMESTAMP`),+});+export const sessions = sqliteTable('sessions',{@@ -293,3 +317,5 @@ export type SwitchServerRow = typeof switchServers.$inferSelect;export type SwitchServerInsert = typeof switchServers.$inferInsert;export type RemoteHostRow = typeof remoteHosts.$inferSelect;export type RemoteHostInsert = typeof remoteHosts.$inferInsert;+export type RemoteHostReachabilityRow = typeof remoteHostReachability.$inferSelect;+export type RemoteHostReachabilityInsert = typeof remoteHostReachability.$inferInsert;diff --git a/dash/apps/switchdash-desktop/src/main/index.ts b/dash/apps/switchdash-desktop/src/main/index.tsindex 70266a82a..4b9d491c6 100644--- a/dash/apps/switchdash-desktop/src/main/index.ts+++ b/dash/apps/switchdash-desktop/src/main/index.ts@@ -20,6 +20,8 @@ import { locationSettingsService } from './core/locations/settings/location-settimport { localServerService } from './core/managed-switch-server/local-server-service';import { remoteServerService } from './core/managed-switch-server/remote-server-service';import { promptLibraryService } from './core/prompt-library/service';+import { initializeHostReachability } from './core/remote-hosts/host-reachability-startup';+import { hostReachabilityService } from './core/remote-hosts/production-host-reachability';import {reconcileResourceSampler,stopResourceSampler,@@ -155,6 +157,14 @@ void app.whenReady().then(async () => {// then start the auto_session watchers — the watcher's "is a session already// attending this room?" check relies on those connections being present.void Promise.all([agentHookReady, dependenciesReady, migrationReady]).then(async () => {+ // Must precede the remote watchers: they gate on host reachability, and+ // starting them first would let every agent on a known-down host attempt a+ // connect before the persisted state is loaded.+ try {+ await initializeHostReachability();+ } catch (e) {+ log.error('Failed to initialise host reachability at startup:', e);+ }try {await restoreSwitchRoomSessions();} catch (e) {@@ -183,6 +193,7 @@ void app.whenReady().then(async () => {powerMonitor.on('resume', () => {log.info('powerMonitor: system resumed — refreshing SSH connections');sshConnectionManager.handleSystemResume();+ hostReachabilityService.handleSystemResume();});setupAppProtocol(join(app.getAppPath(), 'out', 'renderer'));diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsxindex d7a04e67d..8be0de954 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx@@ -9,6 +9,8 @@ import type {} from '@renderer/features/locations/stores/agent-onboarding-types';import { agentsStore } from '@renderer/features/locations/stores/agents-store';import { getLocationManagerStore } from '@renderer/features/locations/stores/location-selectors';+import { HostReachabilityNotice } from '@renderer/features/remote-hosts/host-reachability-notice';+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';import { policyHasDeadRule } from '@renderer/features/switch-servers/addressing-policy-editor';import type { ServerVerifyState } from '@renderer/features/switch-servers/agent-server-picker';import { switchServersStore } from '@renderer/features/switch-servers/switch-servers-store';@@ -273,12 +275,17 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc// Remote submit gate: a provider, a remote dir, a detected agent, and a// verified server. Name defaults from the remote dir basename (see below), so// it is not separately gated. Local submit keeps the existing pick validity.+ // Never create an agent on a host we know we cannot reach — it would be born+ // into the failing state this ticket exists to surface (CHOO-1676).+ const runHostReachable = !isRemoteRun || !hostReachabilityStore.isBlocked(runHost);+const canSubmitDetected = isRemoteRun? !!pickState.providerId &&trimmedRemoteDir.length > 0 &&!isChecking &&!!switchAgent &&verifyState === 'found' &&+ runHostReachable &&submitState === 'idle': pickState.isValid &&!isChecking &&@@ -293,6 +300,7 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc!!pickState.serverId &&!!pickState.providerId &&remoteRunValid &&+ runHostReachable &&submitState === 'idle';// Remote configure gate: no agent in the remote dir yet — a valid remote@@ -307,6 +315,7 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc!!pickState.serverId &&!!pickState.providerId &&trimmedRemoteDir.length > 0 &&+ runHostReachable &&submitState === 'idle';const reportCreationError = (error: AgentOnboardingError) => {@@ -581,11 +590,6 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc}><DialogContentArea data-autofocus tabIndex={-1} className="max-h-[calc(100dvh-13rem)] gap-4">- <AgentTypePicker- value={pickState.providerId}- onChange={pickState.setProviderId}- sshHost={isRemoteRun ? runHost : undefined}- /><Field><FieldLabel>Run location</FieldLabel><Select value={runHost} onValueChange={(v) => setRunHost(v ?? LOCAL_RUN_LOCATION)}>@@ -608,7 +612,8 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc: 'This server runs on this computer, so its agents run here too.'}</p>)}- {isRemoteRun && (+ {isRemoteRun && <HostReachabilityNotice sshHost={runHost} />}+ {isRemoteRun && runHostReachable && (<div className="flex items-center gap-2"><Inputvalue={remoteRepoDirDraft}@@ -632,7 +637,19 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc</div>)}</Field>- {!isRemoteRun && <PickExistingPanel state={pickState} showName={!isMissingSwitchAgent} />}+ {/* The host is the first gate: with it unreachable we cannot know which+ agent types it has, so offering a type picker (or a directory to scan)+ would be guessing. Everything below waits for a usable host. */}+ {runHostReachable && (+ <AgentTypePicker+ value={pickState.providerId}+ onChange={pickState.setProviderId}+ sshHost={isRemoteRun ? runHost : undefined}+ />+ )}+ {runHostReachable && !isRemoteRun && (+ <PickExistingPanel state={pickState} showName={!isMissingSwitchAgent} />+ )}{isChecking && (<p className="text-sm text-foreground-muted">Scanning directory for agents…</p>)}diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsxindex 04fa57e43..dce991809 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx@@ -48,9 +48,22 @@ export function AgentTypePicker({return (<div className="flex items-start gap-2 rounded-md border border-border bg-background-1 px-2 py-1.5 text-xs text-foreground-muted"><CircleAlert className="mt-0.5 size-3.5 shrink-0 text-amber-500" />+ {/* Name the host: "no agent type is set up" reads as a global problem,+ but availability is per-host — the connector may well be installed+ locally and simply missing on the machine being targeted. */}<span>- No agent type is set up for Switch yet. Install an agent's Switch connector in- Settings → Agents before onboarding it.+ {sshHost ? (+ <>+ No agent type is set up for Switch on <span className="font-medium">{sshHost}</span>.+ Install an agent's Switch connector on that host in Settings → Remote hosts+ before onboarding it there.+ </>+ ) : (+ <>+ No agent type is set up for Switch on this computer. Install an agent's Switch+ connector in Settings → Agents before onboarding it.+ </>+ )}</span></div>);diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsxindex e52b6636e..429cb28ff 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx@@ -1,6 +1,8 @@import { Loader2, RefreshCw, TriangleAlert } from 'lucide-react';import { observer } from 'mobx-react-lite';import { useConfirmDeleteAgent } from '@renderer/features/locations/hooks/use-confirm-delete-agent';+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';+import { HostUnreachablePanel } from '@renderer/features/remote-hosts/host-unreachable-panel';import { useParams } from '@renderer/lib/layout/navigation-provider';import { isUnregisteredLocation } from '../../stores/location';import {@@ -25,6 +27,14 @@ export const LocationMainPanel = observer(function LocationMainPanel() {return <PendingLocationStatus location={store} />;}+ // A blocked host outranks every other failure state: the location cannot+ // bootstrap, mount, or find its path while its host is down, and showing the+ // downstream symptom instead of the cause is what CHOO-1682 is about.+ const sshHost = store?.data?.sshHost ?? null;+ if (sshHost && hostReachabilityStore.isBlocked(sshHost) && kind !== 'ready') {+ return <HostUnreachablePanel reachability={hostReachabilityStore.get(sshHost)} />;+ }+if (kind === 'bootstrapping') {return <LocationBootstrappingPanel />;}diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts b/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.tsindex ec83e8fcf..8a83b1907 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts@@ -1,9 +1,10 @@import { err, ok } from '@switchdash/shared';import { makeObservable, observable, runInAction } from 'mobx';-import { rpc } from '@renderer/lib/ipc';+import { events, rpc } from '@renderer/lib/ipc';import { appState } from '@renderer/lib/stores/app-state';import { viewStateCache } from '@renderer/lib/stores/view-state-cache';import { type Location } from '@shared/core/locations/locations';+import { hostReachabilityEventChannel } from '@shared/core/remote-hosts/reachability';import type { LocationViewSnapshot } from '@shared/view-state';import type {AgentOnboardingCompletion,@@ -29,6 +30,23 @@ export class LocationManagerStore {constructor() {makeObservable(this, { locations: observable, pendingCreationIds: observable });++ // A location that failed to open because its host was down holds an error+ // that is meaningless the moment the host is back. Without this the user+ // has to retry twice — once for the host, then again for each location —+ // and the second failure looks like the fix did not work (CHOO-1682).+ events.on(hostReachabilityEventChannel, (reachability) => {+ if (reachability.status === 'reachable') this.remountFailedOnHost(reachability.sshHost);+ });+ }++ /** Re-open every errored location on a host, e.g. after it becomes reachable. */+ remountFailedOnHost(sshHost: string): void {+ for (const [id, store] of this.locations) {+ if (store.data?.sshHost !== sshHost) continue;+ if (!isUnmountedLocation(store) || store.phase !== 'error') continue;+ void this.mountLocation(id);+ }}load(): Promise<void> {diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsx b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsxnew file mode 100644index 000000000..c61f11c22--- /dev/null+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsx@@ -0,0 +1,75 @@+import { Loader2, PlugZap, RefreshCw } from 'lucide-react';+import { observer } from 'mobx-react-lite';+import { useEffect } from 'react';+import { Button } from '@renderer/lib/ui/button';+import { hostReachabilityStore } from './host-reachability-store';++/**+ * Inline reachability warning for forms that target a remote host — the+ * add-agent modal above all (CHOO-1676). Validating up front means the user+ * learns the host is unreachable while they can still change it, instead of+ * creating an agent that immediately lands in a failing state.+ *+ * Renders nothing when the host is fine, so it can be dropped into a form+ * unconditionally.+ */+export const HostReachabilityNotice = observer(function HostReachabilityNotice({+ sshHost,+}: {+ sshHost: string;+}) {+ // Verify on selection rather than trusting a possibly stale record: the user+ // is about to commit to this host.+ useEffect(() => {+ void hostReachabilityStore.hydrate().then(() => {+ if (hostReachabilityStore.get(sshHost).status !== 'reachable') {+ void hostReachabilityStore.retry(sshHost);+ }+ });+ }, [sshHost]);++ const reachability = hostReachabilityStore.get(sshHost);+ const retrying = hostReachabilityStore.isRetrying(sshHost) || reachability.probing;++ if (retrying) {+ return (+ <div className="flex items-center gap-2 rounded-md border border-border bg-background-1 px-2.5 py-2 text-xs text-foreground-muted">+ <Loader2 className="size-3.5 shrink-0 animate-spin" />+ <span>+ Checking that <span className="font-medium text-foreground">{sshHost}</span> is reachable…+ </span>+ </div>+ );+ }++ if (!hostReachabilityStore.isBlocked(sshHost)) return null;++ return (+ <div className="flex items-start gap-2.5 rounded-md border border-amber-500/30 bg-amber-500/8 px-2.5 py-2">+ <PlugZap className="mt-0.5 size-4 shrink-0 text-amber-500" />+ <div className="flex min-w-0 flex-1 flex-col gap-1">+ <p className="text-xs font-medium text-foreground">+ {reachability.status === 'suspended'+ ? `SSH authentication to ${sshHost} failed`+ : `Can’t reach ${sshHost}`}+ </p>+ {reachability.lastError && (+ <p className="text-xs break-words text-foreground-muted">{reachability.lastError}</p>+ )}+ <p className="text-xs text-foreground-passive">+ Choose a different run location, or bring the host back and retry.+ </p>+ </div>+ <Button+ type="button"+ variant="outline"+ size="sm"+ className="shrink-0"+ onClick={() => void hostReachabilityStore.retry(sshHost)}+ >+ <RefreshCw className="size-3" />+ Retry+ </Button>+ </div>+ );+});diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.ts b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.tsnew file mode 100644index 000000000..4195aff0a--- /dev/null+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.ts@@ -0,0 +1,77 @@+import { makeAutoObservable, runInAction } from 'mobx';+import { events, rpc } from '@renderer/lib/ipc';+import {+ type HostReachability,+ hostReachabilityEventChannel,+ isHostBlocked,+ unknownHostReachability,+} from '@shared/core/remote-hosts/reachability';++/**+ * Renderer mirror of the main process's host reachability model (CHOO-1682).+ *+ * Every surface that needs to say "this host is down" — the sidebar badge, the+ * location panel, the add-agent modal — reads from here, so they cannot+ * disagree about a host's state or each render its own interpretation of a raw+ * ssh2 error. Hydrated once, then kept current by pushed events.+ */+export class HostReachabilityStore {+ private readonly byHost = new Map<string, HostReachability>();+ /** Hosts with a retry in flight, so buttons can show progress. */+ readonly retrying = new Set<string>();+ private hydrated = false;++ constructor() {+ makeAutoObservable(this);+ events.on(hostReachabilityEventChannel, (reachability) => {+ runInAction(() => {+ this.byHost.set(reachability.sshHost, reachability);+ });+ });+ }++ /** Load the current state for every known host. Idempotent. */+ async hydrate(): Promise<void> {+ if (this.hydrated) return;+ this.hydrated = true;+ const all = await rpc.remoteHosts.listReachability();+ runInAction(() => {+ for (const reachability of all) this.byHost.set(reachability.sshHost, reachability);+ });+ }++ /**+ * Reachability for a host. An unseen host reads as `unknown` rather than+ * throwing — the UI treats "not yet probed" as "no reason to warn".+ */+ get(sshHost: string): HostReachability {+ return this.byHost.get(sshHost) ?? unknownHostReachability(sshHost);+ }++ /** Whether a host is in a state that pauses host-dependent work. */+ isBlocked(sshHost: string | null | undefined): boolean {+ if (!sshHost) return false;+ return isHostBlocked(this.get(sshHost));+ }++ isRetrying(sshHost: string): boolean {+ return this.retrying.has(sshHost);+ }++ /**+ * Probe now, bypassing the backoff. On success this un-pauses every paused+ * host-dependent path in the main process, not just the current view.+ */+ async retry(sshHost: string): Promise<HostReachability> {+ runInAction(() => this.retrying.add(sshHost));+ try {+ const reachability = await rpc.remoteHosts.retryHost(sshHost);+ runInAction(() => this.byHost.set(sshHost, reachability));+ return reachability;+ } finally {+ runInAction(() => this.retrying.delete(sshHost));+ }+ }+}++export const hostReachabilityStore = new HostReachabilityStore();diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsx b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsxnew file mode 100644index 000000000..9668efa29--- /dev/null+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsx@@ -0,0 +1,80 @@+import { PlugZap, RefreshCw } from 'lucide-react';+import { observer } from 'mobx-react-lite';+import { Button } from '@renderer/lib/ui/button';+import type { HostReachability } from '@shared/core/remote-hosts/reachability';+import { hostReachabilityStore } from './host-reachability-store';++function relativeToNow(iso: string | null): string | null {+ if (!iso) return null;+ const deltaMs = new Date(iso).getTime() - Date.now();+ const seconds = Math.round(Math.abs(deltaMs) / 1000);+ if (seconds < 60) return `${seconds}s`;+ const minutes = Math.round(seconds / 60);+ if (minutes < 60) return `${minutes}m`;+ return `${Math.round(minutes / 60)}h`;+}++/**+ * The single "this host is down" surface (CHOO-1682). Replaces the raw ssh2+ * message ("Connection lost before handshake") that used to reach the user with+ * the modeled state: what is wrong, that work is paused rather than silently+ * retrying forever, when the next automatic probe lands, and one button to+ * retry now.+ */+export const HostUnreachablePanel = observer(function HostUnreachablePanel({+ reachability,+}: {+ reachability: HostReachability;+}) {+ const { sshHost, status, lastError, nextProbeAt, lastReachableAt } = reachability;+ const retrying = hostReachabilityStore.isRetrying(sshHost) || reachability.probing;+ const nextProbe = relativeToNow(nextProbeAt);+ const lastSeen = relativeToNow(lastReachableAt);+ const suspended = status === 'suspended';++ return (+ <div className="flex h-full w-full items-center justify-center p-8">+ <div className="flex w-full max-w-md flex-col items-center gap-4 rounded-lg border border-amber-500/25 bg-amber-500/5 px-6 py-7 text-center">+ <div className="flex size-10 items-center justify-center rounded-full bg-amber-500/12">+ <PlugZap className="size-5 text-amber-500" />+ </div>++ <div className="flex flex-col gap-1.5">+ <p className="text-sm font-medium text-foreground">+ {suspended ? 'SSH authentication failed' : 'Host unreachable'}+ </p>+ <p className="font-mono text-xs text-foreground-muted">{sshHost}</p>+ </div>++ {lastError && (+ <p className="max-w-sm text-xs break-words text-foreground-muted">{lastError}</p>+ )}++ <p className="max-w-sm text-xs text-foreground-passive">+ {suspended+ ? 'Automatic retries are paused — a rejected credential will not fix itself. Restore SSH access to this host, then retry.'+ : 'Work on this host is paused so it is not retried continuously. It resumes automatically once the host is back.'}+ </p>++ <Button+ type="button"+ variant="outline"+ size="sm"+ disabled={retrying}+ onClick={() => void hostReachabilityStore.retry(sshHost)}+ >+ <RefreshCw className={retrying ? 'size-3.5 animate-spin' : 'size-3.5'} />+ {retrying ? 'Checking…' : 'Retry connection'}+ </Button>++ {(nextProbe || lastSeen) && (+ <div className="flex items-center gap-2 text-[11px] text-foreground-passive">+ {!suspended && nextProbe && <span>Next check in {nextProbe}</span>}+ {!suspended && nextProbe && lastSeen && <span aria-hidden>·</span>}+ {lastSeen && <span>Last reachable {lastSeen} ago</span>}+ </div>+ )}+ </div>+ </div>+ );+});diff --git a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsxindex fee67e138..ced907eae 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx+++ b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx@@ -3,6 +3,7 @@ import {Bot,ChevronRight,ExternalLink,+ PlugZap,Plus,RotateCcw,Server,@@ -15,6 +16,7 @@ import {getLocationStore,locationViewKind,} from '@renderer/features/locations/stores/location-selectors';+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';import { hasSessionError } from '@renderer/features/sessions/stores/session-selectors';import { switchRoomsStore } from '@renderer/features/switch-servers/switch-rooms-store';import { AgentIcon } from '@renderer/lib/components/agent-icon';@@ -100,6 +102,10 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({if (!location) return null;+ const sshHost = location.data?.sshHost ?? null;+ const hostReachability = sshHost ? hostReachabilityStore.get(sshHost) : null;+ const hostUnreachable = hostReachabilityStore.isBlocked(sshHost);+const iconClass ='absolute h-4 w-4 opacity-100 transition-opacity duration-150 group-hover/row:opacity-0';const gatewayUrl =@@ -151,7 +157,12 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({{location.data?.sshHost != null && (<Tooltip><TooltipTrigger>- <Server className="h-3.5 w-3.5 shrink-0 text-foreground-muted" />+ <Server+ className={cn(+ 'h-3.5 w-3.5 shrink-0',+ hostUnreachable ? 'text-foreground-warning' : 'text-foreground-muted'+ )}+ /></TooltipTrigger><TooltipContent>Runs remotely on {location.data.sshHost}@@ -159,6 +170,22 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({</TooltipContent></Tooltip>)}+ {/* The host being down is why this agent is idle, so say so on+ the row itself — previously you had to select the agent to+ discover its host was failing to connect (CHOO-1682). */}+ {hostUnreachable && hostReachability && (+ <Tooltip>+ <TooltipTrigger>+ <PlugZap className="h-3.5 w-3.5 shrink-0 text-foreground-warning" />+ </TooltipTrigger>+ <TooltipContent>+ {hostReachability.status === 'suspended'+ ? `SSH authentication to ${hostReachability.sshHost} failed — work is paused until you retry`+ : `Host ${hostReachability.sshHost} is unreachable — work is paused`}+ {hostReachability.lastError ? ` · ${hostReachability.lastError}` : ''}+ </TooltipContent>+ </Tooltip>+ )}{locationViewKind(location) === 'ready' && hasSessionError(agent.locationId) && (<Tooltip><TooltipTrigger>diff --git a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsxindex f22aa87a3..4ba6923dc 100644--- a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx+++ b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx@@ -2,6 +2,7 @@ import { observer } from 'mobx-react-lite';import { Fragment, useEffect } from 'react';import { agentsStore } from '@renderer/features/locations/stores/agents-store';import type { LocationStore } from '@renderer/features/locations/stores/location';+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';import type { SessionStore } from '@renderer/features/sessions/stores/session-store';import { switchRoomsStore as roomConnectionsStore } from '@renderer/features/switch-rooms/switch-rooms-store';import { switchRoomsStore } from '@renderer/features/switch-servers/switch-rooms-store';@@ -59,6 +60,7 @@ export const SidebarGroupedList = observer(function SidebarGroupedList() {useEffect(() => {roomConnectionsStore.ensureLoaded();void agentsStore.load();+ void hostReachabilityStore.hydrate();void switchServersStore.init().then(() => switchRoomsStore.loadRoomNames());const onFocus = () => {void agentsStore.load();diff --git a/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.ts b/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.tsnew file mode 100644index 000000000..f6bf5edeb--- /dev/null+++ b/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.ts@@ -0,0 +1,87 @@+/**+ * First-class model of a remote SSH host's reachability (CHOO-1682).+ *+ * Before this, "can we reach host X?" was answered independently by every+ * consumer: the pooled SSH connection's in-memory state, an on-demand+ * `testConnection`, or simply attempting the work and interpreting the ssh2+ * error. That produced two failure modes — unbounded retry loops hammering a+ * dead host (the reconciler reconnecting every 2s per agent), and ad-hoc raw+ * transport errors surfacing in the UI with no shared notion of "this host is+ * down". Reachability is now one per-host state machine that every+ * host-dependent path consults up front.+ */++import { defineEvent } from '@shared/lib/ipc/events';++/**+ * Where a host sits in the reachability state machine.+ *+ * - `unknown` — never probed this app run (and nothing persisted). Work is+ * allowed through: the first attempt is what establishes reachability.+ * - `reachable` — a probe or live connection succeeded. Work proceeds.+ * - `unreachable` — probes are failing. Host-dependent background work pauses;+ * a bounded backoff probe keeps checking so recovery is automatic.+ * - `suspended` — authentication failed. Retrying a rejected key never+ * self-heals, so no automatic probing happens; the user must fix auth and+ * retry explicitly.+ */+export type HostReachabilityStatus = 'unknown' | 'reachable' | 'unreachable' | 'suspended';++/** The full reachability record for one host alias. */+export type HostReachability = {+ /** The `~/.ssh/config` Host alias this record describes. */+ sshHost: string;+ status: HostReachabilityStatus;+ /** Message from the most recent failed probe; null while reachable. */+ lastError: string | null;+ /** ISO timestamp of the last probe (successful or not). */+ lastCheckedAt: string | null;+ /** ISO timestamp of the last time the host was confirmed reachable. */+ lastReachableAt: string | null;+ /** Consecutive failed probes — drives the backoff step. */+ consecutiveFailures: number;+ /** ISO timestamp of the next scheduled probe; null when none is scheduled. */+ nextProbeAt: string | null;+ /** True while a probe is in flight (for spinners; not a persisted state). */+ probing: boolean;+};++/**+ * Whether host-dependent work should be held back. `unknown` is deliberately+ * NOT blocked — a host we have never probed gets one real attempt rather than+ * being gated behind a probe that would duplicate it.+ */+export function isHostBlocked(reachability: HostReachability): boolean {+ return reachability.status === 'unreachable' || reachability.status === 'suspended';+}++/** Human-readable one-liner for the blocked state, used in errors and the UI. */+export function hostBlockedReason(reachability: HostReachability): string {+ const since = reachability.lastCheckedAt ? ` since ${reachability.lastCheckedAt}` : '';+ if (reachability.status === 'suspended') {+ return `SSH authentication to ${reachability.sshHost} failed${since}: ${+ reachability.lastError ?? 'unknown error'+ }. Fix the host's SSH credentials, then retry.`;+ }+ return `Host ${reachability.sshHost} is unreachable${since}: ${+ reachability.lastError ?? 'unknown error'+ }`;+}++export function unknownHostReachability(sshHost: string): HostReachability {+ return {+ sshHost,+ status: 'unknown',+ lastError: null,+ lastCheckedAt: null,+ lastReachableAt: null,+ consecutiveFailures: 0,+ nextProbeAt: null,+ probing: false,+ };+}++/** Pushed to the renderer whenever a host's reachability record changes. */+export const hostReachabilityEventChannel = defineEvent<HostReachability>(+ 'remote-hosts:reachability-changed'+);