Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(switchdash): model remote-host reachability as first-class state (CHOO-1682) (#82)

Louis Amaudruz496b5b9360a4
diff --git a/dash/AGENTS.md b/dash/AGENTS.md
index a5952d706..a80a046f3 100644
--- a/dash/AGENTS.md
+++ b/dash/AGENTS.md
@@ -248,7 +248,18 @@ pnpm run test
- Migration tests run in the `migrations` project via `pnpm run test:migrations`.
- Fixture generation runs in the `fixtures` project via `pnpm run db:fixtures`.
- Renderer browser tests run in the `browser` project using Playwright-backed
- `@vitest/browser-playwright`.
+ `@vitest/browser-playwright`. **Agents: skip these — do not run or try to fix
+ them.** They need a real Chromium and its system libraries (e.g. `libnspr4`),
+ which dev VMs generally lack, so they fail for environment reasons unrelated
+ to your change. CI is the gate for them. Run the node projects instead:
+
+```bash
+pnpm vitest run --project node --project main-db
+```
+
+ If `pnpm run test` reports failures only from the `browser` project (a
+ Playwright launch error rather than an assertion), treat the run as green and
+ say so — do not install browsers or chase the failure.
- Main-process tests are colocated in `src/main/core/**/*.test.ts`.
- Renderer unit tests live under `src/renderer/tests/`.
- Renderer browser tests live under `src/renderer/tests/browser/`.
diff --git a/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sql b/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sql
new file mode 100644
index 000000000..f411245a3
--- /dev/null
+++ b/dash/apps/switchdash-desktop/drizzle/0043_furry_maggott.sql
@@ -0,0 +1,9 @@
+CREATE TABLE `remote_host_reachability` (
+ `ssh_host` text PRIMARY KEY NOT NULL,
+ `status` text DEFAULT 'unknown' NOT NULL,
+ `last_error` text,
+ `last_checked_at` text,
+ `last_reachable_at` text,
+ `consecutive_failures` integer DEFAULT 0 NOT NULL,
+ `updated_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
diff --git a/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.json b/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.json
new file mode 100644
index 000000000..55c5735ea
--- /dev/null
+++ b/dash/apps/switchdash-desktop/drizzle/meta/0043_snapshot.json
@@ -0,0 +1,773 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "24ca41c8-3ae9-4b92-976a-08a6a6d4ab4c",
+ "prevId": "4744931c-4640-42c4-9ac3-6b1b0946a0b4",
+ "tables": {
+ "agents": {
+ "name": "agents",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "location_id": {
+ "name": "location_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "switch_agent_id": {
+ "name": "switch_agent_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "api_endpoint": {
+ "name": "api_endpoint",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "server_id": {
+ "name": "server_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "auto_approve": {
+ "name": "auto_approve",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_agents_location_id": {
+ "name": "idx_agents_location_id",
+ "columns": ["location_id"],
+ "isUnique": false
+ },
+ "idx_agents_server_id": {
+ "name": "idx_agents_server_id",
+ "columns": ["server_id"],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "agents_location_id_locations_id_fk": {
+ "name": "agents_location_id_locations_id_fk",
+ "tableFrom": "agents",
+ "tableTo": "locations",
+ "columnsFrom": ["location_id"],
+ "columnsTo": ["id"],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "agents_server_id_switch_servers_id_fk": {
+ "name": "agents_server_id_switch_servers_id_fk",
+ "tableFrom": "agents",
+ "tableTo": "switch_servers",
+ "columnsFrom": ["server_id"],
+ "columnsTo": ["id"],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "app_secrets": {
+ "name": "app_secrets",
+ "columns": {
+ "key": {
+ "name": "key",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "secret": {
+ "name": "secret",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "idx_app_secrets_key": {
+ "name": "idx_app_secrets_key",
+ "columns": ["key"],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "app_settings": {
+ "name": "app_settings",
+ "columns": {
+ "key": {
+ "name": "key",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "value": {
+ "name": "value",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_app_settings_key": {
+ "name": "idx_app_settings_key",
+ "columns": ["key"],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "kv": {
+ "name": "kv",
+ "columns": {
+ "key": {
+ "name": "key",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "value": {
+ "name": "value",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_kv_key": {
+ "name": "idx_kv_key",
+ "columns": ["key"],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "location_settings": {
+ "name": "location_settings",
+ "columns": {
+ "location_id": {
+ "name": "location_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "base_settings_json": {
+ "name": "base_settings_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'{}'"
+ },
+ "shareable_settings_json": {
+ "name": "shareable_settings_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'{}'"
+ },
+ "legacy_config_migrated_at": {
+ "name": "legacy_config_migrated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "location_settings_location_id_locations_id_fk": {
+ "name": "location_settings_location_id_locations_id_fk",
+ "tableFrom": "location_settings",
+ "tableTo": "locations",
+ "columnsFrom": ["location_id"],
+ "columnsTo": ["id"],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "locations": {
+ "name": "locations",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "ssh_host": {
+ "name": "ssh_host",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "''"
+ },
+ "dir": {
+ "name": "dir",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_locations_host_dir": {
+ "name": "idx_locations_host_dir",
+ "columns": ["ssh_host", "dir"],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "messages": {
+ "name": "messages",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "session_id": {
+ "name": "session_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "content": {
+ "name": "content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "sender": {
+ "name": "sender",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "idx_messages_session_id": {
+ "name": "idx_messages_session_id",
+ "columns": ["session_id"],
+ "isUnique": false
+ },
+ "idx_messages_timestamp": {
+ "name": "idx_messages_timestamp",
+ "columns": ["timestamp"],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "messages_session_id_sessions_id_fk": {
+ "name": "messages_session_id_sessions_id_fk",
+ "tableFrom": "messages",
+ "tableTo": "sessions",
+ "columnsFrom": ["session_id"],
+ "columnsTo": ["id"],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "remote_host_reachability": {
+ "name": "remote_host_reachability",
+ "columns": {
+ "ssh_host": {
+ "name": "ssh_host",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'unknown'"
+ },
+ "last_error": {
+ "name": "last_error",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_checked_at": {
+ "name": "last_checked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_reachable_at": {
+ "name": "last_reachable_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "consecutive_failures": {
+ "name": "consecutive_failures",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "remote_hosts": {
+ "name": "remote_hosts",
+ "columns": {
+ "ssh_host": {
+ "name": "ssh_host",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "sessions": {
+ "name": "sessions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "agent_id": {
+ "name": "agent_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "config": {
+ "name": "config",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "shell_id": {
+ "name": "shell_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'system'"
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "agent_session_id": {
+ "name": "agent_session_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "agent_status": {
+ "name": "agent_status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "agent_status_seen": {
+ "name": "agent_status_seen",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": 1
+ },
+ "is_initial_session": {
+ "name": "is_initial_session",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_pinned": {
+ "name": "is_pinned",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_interacted_at": {
+ "name": "last_interacted_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status_changed_at": {
+ "name": "status_changed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_sessions_agent_id": {
+ "name": "idx_sessions_agent_id",
+ "columns": ["agent_id"],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "sessions_agent_id_agents_id_fk": {
+ "name": "sessions_agent_id_agents_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "agents",
+ "columnsFrom": ["agent_id"],
+ "columnsTo": ["id"],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "switch_servers": {
+ "name": "switch_servers",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "gateway_url": {
+ "name": "gateway_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "api_url": {
+ "name": "api_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "managed": {
+ "name": "managed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "management_kind": {
+ "name": "management_kind",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "ssh_host": {
+ "name": "ssh_host",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "CURRENT_TIMESTAMP"
+ }
+ },
+ "indexes": {
+ "idx_switch_servers_gateway_url": {
+ "name": "idx_switch_servers_gateway_url",
+ "columns": ["gateway_url"],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
diff --git a/dash/apps/switchdash-desktop/drizzle/meta/_journal.json b/dash/apps/switchdash-desktop/drizzle/meta/_journal.json
index 6ae857a92..bf8cac704 100644
--- a/dash/apps/switchdash-desktop/drizzle/meta/_journal.json
+++ b/dash/apps/switchdash-desktop/drizzle/meta/_journal.json
@@ -302,6 +302,13 @@
"when": 1785164441913,
"tag": "0042_drop_agent_definition_name",
"breakpoints": true
+ },
+ {
+ "idx": 43,
+ "version": "6",
+ "when": 1785224611278,
+ "tag": "0043_furry_maggott",
+ "breakpoints": true
}
]
}
diff --git a/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts b/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts
index 2a582bdca..ce3925e5d 100644
--- a/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/agents/remote-session-reconciler.ts
@@ -4,6 +4,9 @@ import { probeAgentSidecar } from '@main/core/agent-runtime/impl/ensure-agent-si
import type { SidecarEndpoint } from '@main/core/agent-runtime/impl/remote-sidecar-launcher';
import { httpGetJsonOverChannel } from '@main/core/agent-runtime/impl/sidecar-http';
import { sshConnectionIdForHost } from '@main/core/locations/location-transport';
+import type { HostReachabilityChange } from '@main/core/remote-hosts/host-reachability-service';
+import { HostUnreachableError } from '@main/core/remote-hosts/host-reachability-service';
+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';
import { sessionHooks } from '@main/core/sessions/session-hooks';
import { sessionService } from '@main/core/sessions/session-service';
import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';
@@ -108,6 +111,23 @@ class RemoteSessionReconciler {
sessionHooks.on('session:remote-terminated', ({ terminatedSessionId }) => {
void this.handleRemoteTerminated(terminatedSessionId);
});
+
+ // A host coming back is the signal to resume the agents that were paused on
+ // it — without this they would idle until their next 2s tick, which is
+ // harmless, but ticking immediately makes recovery feel instant and keeps
+ // the paused/resumed transition symmetric.
+ hostReachabilityService.on('change', ({ current }: HostReachabilityChange) => {
+ if (current.status === 'reachable') void this.onHostReachable(current.sshHost);
+ });
+ }
+
+ /** Tick every agent whose location sits on a host that just became reachable. */
+ private async onHostReachable(sshHost: string): Promise<void> {
+ for (const agentId of [...this.timers.keys()]) {
+ const agent = await getAgentById(agentId);
+ const location = agent ? await getRemoteAgentLocation(agent) : null;
+ if (location?.sshHost === sshHost) void this.tick(agentId);
+ }
}
private async handleRemoteTerminated(sessionId: string): Promise<void> {
@@ -227,7 +247,17 @@ class RemoteSessionReconciler {
}),
]);
} catch (error) {
- log.warn('RemoteSessionReconciler: reconcile failed', { agentId, error: String(error) });
+ // A host that went unreachable mid-pass is already reported once, loudly,
+ // by the reachability service. Repeating it here every 2s per agent is the
+ // log flood CHOO-1682 is about.
+ if (error instanceof HostUnreachableError) {
+ log.debug('RemoteSessionReconciler: host unreachable — pass skipped', {
+ agentId,
+ sshHost: error.reachability.sshHost,
+ });
+ } else {
+ log.warn('RemoteSessionReconciler: reconcile failed', { agentId, error: String(error) });
+ }
} finally {
if (deadline) clearTimeout(deadline);
this.inFlight.delete(agentId);
@@ -250,6 +280,12 @@ class RemoteSessionReconciler {
const sidecarKey = `${location.id}::${agent.name ?? agent.id}`;
if (!this.claimSidecar(agentId, sidecarKey)) return;
+ // The host is known to be down: skip the pass entirely rather than
+ // attempting a connect every 2s per agent. Recovery is driven by the
+ // reachability service's single backoff probe, and `onHostReachable` ticks
+ // this agent the moment the host comes back, so nothing is lost by waiting.
+ if (hostReachabilityService.isBlocked(location.sshHost)) return;
+
// The manager is mid-backoff rebuilding the transport: let it finish
// rather than racing it with connect attempts every tick. Disconnected and
// suspended states DO proceed — connectRemoteAgent's connect() is what
diff --git a/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts b/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts
index d0114487d..a64fb7ad0 100644
--- a/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/locations/location-manager.ts
@@ -1,5 +1,6 @@
import { err, ok, type Result } from '@switchdash/shared';
import type { IDisposable } from '@switchdash/shared';
+import { HostUnreachableError } from '@main/core/remote-hosts/host-reachability-service';
import { HookCore, type Hookable } from '@main/lib/hookable';
import { LifecycleMap } from '@main/lib/lifecycle-map';
import { log } from '@main/lib/logger';
@@ -51,10 +52,13 @@ class LocationManager implements Hookable<LocationManagerHooks>, IDisposable {
return ok(provider);
} catch (e) {
const initError = toLifecycleError(e);
- log.error('LocationManager: error during location initialization', {
- locationId: location.id,
- ...initError,
- });
+ // A location on a host that is down cannot open, and that is an
+ // expected condition the host's reachability state already reports —
+ // logging it as an app error made every outage look like a defect.
+ const message = 'LocationManager: error during location initialization';
+ const metadata = { locationId: location.id, ...initError };
+ if (e instanceof HostUnreachableError) log.warn(message, metadata);
+ else log.error(message, metadata);
return err(initError);
}
});
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts
index ed937db95..85e4f16c5 100644
--- a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/controller.ts
@@ -23,10 +23,12 @@ import {
getRemoteSwitchSetupService,
type RemoteSwitchSetupService,
} from '@main/core/switch-setup/remote-switch-setup';
+import { hostBlockedReason, type HostReachability } from '@shared/core/remote-hosts/reachability';
import type { ConnectionState, SshHealthState } from '@shared/core/ssh/ssh';
import { createRPCController } from '@shared/lib/ipc/rpc';
import type { SwitchAgentConfig } from '@shared/switch-agents';
import { listSshConfigHosts } from './list-ssh-config-hosts';
+import { hostReachabilityService } from './production-host-reachability';
import { listRemoteHosts, removeRemoteHost, upsertRemoteHost, type RemoteHost } from './store';
/** A single dependency's status on a remote host, enriched for the UI. */
@@ -89,15 +91,16 @@ async function probeGhAuth(sshHost: string): Promise<GhAuthStatus> {
}
}
+/**
+ * Ad-hoc reachability check. Routed through the reachability service rather
+ * than connecting directly, so a manual "Test connection" and the background
+ * probe share one code path — and a successful test immediately un-pauses the
+ * host-dependent work that was gated on the old state.
+ */
async function testConnection(sshHost: string): Promise<TestConnectionResult> {
- try {
- const proxy = await ensureSshConnected(sshConnectionIdForHost(sshHost), sshHost);
- const ctx = new SshExecutionContext(proxy);
- await ctx.exec('uname', ['-s']);
- return { ok: true };
- } catch (error) {
- return { ok: false, message: error instanceof Error ? error.message : String(error) };
- }
+ const reachability = await hostReachabilityService.checkNow(sshHost);
+ if (reachability.status === 'reachable') return { ok: true };
+ return { ok: false, message: reachability.lastError ?? hostBlockedReason(reachability) };
}
async function probeDeps(sshHost: string): Promise<RemoteDependencyView[]> {
@@ -241,13 +244,36 @@ export const remoteHostsController = createRPCController({
getConnectionStatus: (sshHost: string): Promise<HostConnectionStatus> =>
Promise.resolve(hostConnectionStatus(sshHost)),
+ /** Modeled reachability for one host — the state the UI gates its display on. */
+ getReachability: (sshHost: string): Promise<HostReachability> =>
+ Promise.resolve(hostReachabilityService.get(sshHost)),
+
+ /** Every host the reachability model knows about, for the initial UI hydrate. */
+ listReachability: (): Promise<HostReachability[]> =>
+ Promise.resolve(hostReachabilityService.getAll()),
+
+ /**
+ * Probe now, bypassing the backoff — the "Retry connection" button. On
+ * success this clears the blocked state, which resumes every paused
+ * host-dependent path (reconciler included) rather than only fixing the
+ * screen the user is looking at.
+ */
+ retryHost: (sshHost: string): Promise<HostReachability> =>
+ hostReachabilityService.checkNow(sshHost),
+
/**
* Force a full transport rebuild for the host's pooled connection — the
* manual recovery path for a wedged or given-up connection. Returns the
* post-rebuild status.
*/
reconnectHost: async (sshHost: string): Promise<HostConnectionStatus> => {
- await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);
+ try {
+ await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);
+ } catch (error) {
+ hostReachabilityService.reportFailure(sshHost, error);
+ throw error;
+ }
+ hostReachabilityService.reportSuccess(sshHost);
return hostConnectionStatus(sshHost);
},
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.ts
new file mode 100644
index 000000000..63384d7f4
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.test.ts
@@ -0,0 +1,196 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { HostReachability } from '@shared/core/remote-hosts/reachability';
+import {
+ HostReachabilityService,
+ HostUnreachableError,
+ probeDelayFor,
+} from './host-reachability-service';
+
+vi.mock('./reachability-store', () => ({
+ listPersistedReachability: vi.fn(async () => []),
+ savePersistedReachability: vi.fn(async () => {}),
+ hydrate: (record: unknown) => ({
+ ...(record as object),
+ nextProbeAt: null,
+ probing: false,
+ }),
+}));
+
+class AuthError extends Error {}
+
+function makeService(probe: (sshHost: string) => Promise<void>) {
+ const published: HostReachability[] = [];
+ const timers: Array<{ fn: () => void; ms: number }> = [];
+ const service = new HostReachabilityService({
+ probe,
+ isAuthError: (error) => error instanceof AuthError,
+ publish: (reachability) => published.push(reachability),
+ log: { info: () => {}, warn: () => {} },
+ setTimer: (fn, ms) => {
+ timers.push({ fn, ms });
+ return timers.length as unknown as NodeJS.Timeout;
+ },
+ clearTimer: () => {},
+ now: () => 1_700_000_000_000,
+ });
+ return { service, published, timers };
+}
+
+/** Flush the microtask queue so fire-and-forget transitions settle. */
+const settle = () => new Promise((resolve) => setImmediate(resolve));
+
+describe('HostReachabilityService', () => {
+ let service: HostReachabilityService;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it('treats a never-probed host as unknown and lets work through', () => {
+ ({ service } = makeService(async () => {}));
+ expect(service.get('vm').status).toBe('unknown');
+ expect(service.isBlocked('vm')).toBe(false);
+ expect(() => service.requireReachable('vm')).not.toThrow();
+ });
+
+ it('blocks work and throws the modeled reason once a host is unreachable', async () => {
+ ({ service } = makeService(async () => {
+ throw new Error('Connection lost before handshake');
+ }));
+
+ await service.checkNow('vm');
+
+ expect(service.get('vm').status).toBe('unreachable');
+ expect(service.isBlocked('vm')).toBe(true);
+ expect(() => service.requireReachable('vm')).toThrow(HostUnreachableError);
+ expect(() => service.requireReachable('vm')).toThrow(/Connection lost before handshake/);
+ });
+
+ it('does not auto-probe an auth failure — a rejected key never self-heals', async () => {
+ const { service: svc, timers } = makeService(async () => {
+ throw new AuthError('All configured authentication methods failed');
+ });
+
+ await svc.checkNow('vm');
+
+ expect(svc.get('vm').status).toBe('suspended');
+ expect(timers).toHaveLength(0);
+ });
+
+ it('backs off between probes instead of retrying at a fixed interval', async () => {
+ const { service: svc, timers } = makeService(async () => {
+ throw new Error('unreachable');
+ });
+
+ await svc.checkNow('vm');
+ expect(timers.at(-1)?.ms).toBe(1_000);
+
+ await svc.checkNow('vm');
+ expect(timers.at(-1)?.ms).toBe(5_000);
+
+ await svc.checkNow('vm');
+ expect(timers.at(-1)?.ms).toBe(15_000);
+ });
+
+ it('probes tightly at first, then caps at five minutes', () => {
+ expect(probeDelayFor(1)).toBe(1_000);
+ expect(probeDelayFor(4)).toBe(30_000);
+ expect(probeDelayFor(6)).toBe(300_000);
+ expect(probeDelayFor(50)).toBe(300_000);
+ });
+
+ it('clears the blocked state and emits a change when the host recovers', async () => {
+ let reachable = false;
+ const { service: svc } = makeService(async () => {
+ if (!reachable) throw new Error('down');
+ });
+ const changes: string[] = [];
+ svc.on('change', ({ current }) => changes.push(current.status));
+
+ await svc.checkNow('vm');
+ expect(svc.isBlocked('vm')).toBe(true);
+
+ reachable = true;
+ await svc.checkNow('vm');
+
+ expect(svc.get('vm').status).toBe('reachable');
+ expect(svc.isBlocked('vm')).toBe(false);
+ expect(svc.get('vm').consecutiveFailures).toBe(0);
+ expect(svc.get('vm').lastError).toBeNull();
+ expect(changes).toEqual(['unreachable', 'reachable']);
+ });
+
+ it('coalesces concurrent probes onto one round trip', async () => {
+ const probe = vi.fn(async () => {});
+ const { service: svc } = makeService(probe);
+
+ await Promise.all([svc.checkNow('vm'), svc.checkNow('vm'), svc.checkNow('vm')]);
+
+ expect(probe).toHaveBeenCalledTimes(1);
+ });
+
+ it('probes each host independently', async () => {
+ const { service: svc } = makeService(async (sshHost) => {
+ if (sshHost === 'bad') throw new Error('down');
+ });
+
+ await svc.checkNow('good');
+ await svc.checkNow('bad');
+
+ expect(svc.isBlocked('good')).toBe(false);
+ expect(svc.isBlocked('bad')).toBe(true);
+ });
+
+ it('lets ordinary traffic keep the record fresh without a probe', async () => {
+ const probe = vi.fn(async () => {
+ throw new Error('down');
+ });
+ const { service: svc } = makeService(probe);
+
+ await svc.checkNow('vm');
+ expect(svc.isBlocked('vm')).toBe(true);
+
+ svc.reportSuccess('vm');
+ await settle();
+
+ expect(svc.get('vm').status).toBe('reachable');
+ expect(probe).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports a transport failure from real work as unreachable', async () => {
+ const { service: svc } = makeService(async () => {});
+
+ svc.reportFailure('vm', new Error('Connection lost before handshake'));
+ await settle();
+
+ expect(svc.get('vm').status).toBe('unreachable');
+ expect(svc.get('vm').lastError).toBe('Connection lost before handshake');
+ });
+
+ it('folds a live SSH connection into the host model', async () => {
+ const { service: svc } = makeService(async () => {
+ throw new Error('down');
+ });
+
+ await svc.checkNow('vm');
+ svc.handleSshConnectionEvent('vm', { type: 'connected', connectionId: 'agent-ssh:vm' });
+ await settle();
+
+ expect(svc.get('vm').status).toBe('reachable');
+ });
+
+ it('re-probes blocked hosts after a system resume', async () => {
+ const probe = vi.fn(async () => {
+ throw new Error('down');
+ });
+ const { service: svc } = makeService(probe);
+
+ await svc.checkNow('vm');
+ probe.mockClear();
+
+ svc.handleSystemResume();
+ await settle();
+
+ expect(probe).toHaveBeenCalledWith('vm');
+ });
+});
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.ts
new file mode 100644
index 000000000..1689a48c9
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-service.ts
@@ -0,0 +1,365 @@
+import { EventEmitter } from 'node:events';
+import {
+ hostBlockedReason,
+ type HostReachability,
+ type HostReachabilityStatus,
+ isHostBlocked,
+ unknownHostReachability,
+} from '@shared/core/remote-hosts/reachability';
+import type { SshConnectionEvent } from '@shared/core/ssh/sshEvents';
+import {
+ hydrate,
+ listPersistedReachability,
+ savePersistedReachability,
+ type PersistedHostReachability,
+} from './reachability-store';
+
+/**
+ * Backoff schedule between probes of an unreachable host, capped at five
+ * minutes and repeated.
+ *
+ * The early steps stay tight because the common cause is a credential the user
+ * is actively re-establishing (VPN, `aws sso login`) and they expect the app to
+ * notice quickly; the long tail keeps a host that is genuinely gone from
+ * costing anything. Manual retry short-circuits the whole schedule, so the cap
+ * only governs the unattended case.
+ */
+const PROBE_BACKOFF_MS = [1_000, 5_000, 15_000, 30_000, 60_000, 300_000];
+
+export function probeDelayFor(consecutiveFailures: number): number {
+ const step = Math.min(Math.max(consecutiveFailures, 1), PROBE_BACKOFF_MS.length);
+ return PROBE_BACKOFF_MS[step - 1]!;
+}
+
+/** Probes a host's transport. Resolves on success, throws on failure. */
+export type HostProbe = (sshHost: string) => Promise<void>;
+
+/** Distinguishes a rejected credential (no point retrying) from a dead network. */
+export type AuthErrorPredicate = (error: unknown) => boolean;
+
+export type HostReachabilityLog = {
+ info: (message: string, metadata?: Record<string, unknown>) => void;
+ warn: (message: string, metadata?: Record<string, unknown>) => void;
+};
+
+export type HostReachabilityServiceDeps = {
+ probe: HostProbe;
+ isAuthError: AuthErrorPredicate;
+ publish: (reachability: HostReachability) => void;
+ log: HostReachabilityLog;
+ /** Injected so tests can drive the backoff without real timers. */
+ setTimer?: (fn: () => void, ms: number) => NodeJS.Timeout;
+ clearTimer?: (timer: NodeJS.Timeout) => void;
+ now?: () => number;
+};
+
+type HostEntry = {
+ reachability: HostReachability;
+ timer?: NodeJS.Timeout;
+ /** In-flight probe, so concurrent callers coalesce onto one round trip. */
+ inFlight?: Promise<HostReachability>;
+};
+
+/**
+ * Raised when host-dependent work is attempted against a host known to be
+ * unreachable. Carries the reachability record so callers (and the UI) can
+ * render the modeled state rather than re-deriving it from a transport error.
+ */
+export class HostUnreachableError extends Error {
+ readonly reachability: HostReachability;
+
+ constructor(reachability: HostReachability) {
+ super(hostBlockedReason(reachability));
+ this.name = 'HostUnreachableError';
+ this.reachability = reachability;
+ }
+}
+
+export type HostReachabilityChange = {
+ previous: HostReachabilityStatus;
+ current: HostReachability;
+};
+
+/**
+ * The single source of truth for whether a remote SSH host can be reached.
+ *
+ * One probe loop per host, shared by every consumer: ten agents on a dead host
+ * produce one probe per backoff step, not ten reconnect storms. Consumers ask
+ * `isBlocked`/`requireReachable` before doing host-dependent work and subscribe
+ * to `change` to resume when a host comes back.
+ */
+export class HostReachabilityService extends EventEmitter {
+ private readonly deps: Required<HostReachabilityServiceDeps>;
+ private readonly hosts = new Map<string, HostEntry>();
+ private loaded = false;
+
+ constructor(deps: HostReachabilityServiceDeps) {
+ super();
+ this.deps = {
+ setTimer: (fn, ms) => setTimeout(fn, ms),
+ clearTimer: (timer) => clearTimeout(timer),
+ now: () => Date.now(),
+ ...deps,
+ };
+ }
+
+ /**
+ * Load persisted state so the app boots knowing which hosts were down. Hosts
+ * that were unreachable at shutdown get a probe scheduled rather than being
+ * hammered by every consumer at once on startup.
+ */
+ async initialize(): Promise<void> {
+ if (this.loaded) return;
+ this.loaded = true;
+ const persisted = await listPersistedReachability();
+ for (const record of persisted) {
+ const reachability = hydrate(record);
+ this.hosts.set(record.sshHost, { reachability });
+ if (isHostBlocked(reachability) && reachability.status !== 'suspended') {
+ this.scheduleProbe(record.sshHost);
+ }
+ }
+ this.deps.log.info('HostReachabilityService: restored persisted host reachability', {
+ hosts: persisted.length,
+ });
+ }
+
+ get(sshHost: string): HostReachability {
+ return this.entry(sshHost).reachability;
+ }
+
+ getAll(): HostReachability[] {
+ return [...this.hosts.values()].map((entry) => entry.reachability);
+ }
+
+ /** True when host-dependent work should be held back for this host. */
+ isBlocked(sshHost: string): boolean {
+ return isHostBlocked(this.get(sshHost));
+ }
+
+ /**
+ * Gate for host-dependent work. Throws `HostUnreachableError` immediately —
+ * with the modeled reason — instead of letting the caller discover the
+ * failure through a 20-second SSH timeout.
+ */
+ requireReachable(sshHost: string): void {
+ const reachability = this.get(sshHost);
+ if (isHostBlocked(reachability)) throw new HostUnreachableError(reachability);
+ }
+
+ /**
+ * Record that a host was just reached successfully by real work (not a
+ * probe). Lets ordinary traffic keep the record fresh so a busy host is
+ * never probed redundantly.
+ */
+ reportSuccess(sshHost: string): void {
+ const entry = this.entry(sshHost);
+ if (entry.reachability.status === 'reachable') return;
+ this.cancelProbe(entry);
+ void this.transition(sshHost, {
+ status: 'reachable',
+ lastError: null,
+ consecutiveFailures: 0,
+ });
+ }
+
+ /**
+ * Record that host-dependent work failed at the transport level. Moves the
+ * host into `unreachable` (or `suspended` for auth) and starts the shared
+ * backoff probe, so the failing caller does not have to retry on its own.
+ */
+ reportFailure(sshHost: string, error: unknown): void {
+ const message = error instanceof Error ? error.message : String(error);
+ const entry = this.entry(sshHost);
+ const status: HostReachabilityStatus = this.deps.isAuthError(error)
+ ? 'suspended'
+ : 'unreachable';
+ void this.transition(sshHost, {
+ status,
+ lastError: message,
+ consecutiveFailures: entry.reachability.consecutiveFailures + 1,
+ }).then(() => {
+ if (status === 'unreachable') this.scheduleProbe(sshHost);
+ });
+ }
+
+ /**
+ * Probe now, bypassing the backoff — the "Retry connection" path. Also used
+ * to validate a host before it is adopted (e.g. the add-agent modal), so
+ * validation and background recovery share one code path.
+ */
+ async checkNow(sshHost: string): Promise<HostReachability> {
+ const entry = this.entry(sshHost);
+ if (entry.inFlight) return entry.inFlight;
+ this.cancelProbe(entry);
+
+ const run = (async (): Promise<HostReachability> => {
+ this.setProbing(sshHost, true);
+ try {
+ await this.deps.probe(sshHost);
+ return await this.transition(sshHost, {
+ status: 'reachable',
+ lastError: null,
+ consecutiveFailures: 0,
+ });
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ const status: HostReachabilityStatus = this.deps.isAuthError(error)
+ ? 'suspended'
+ : 'unreachable';
+ const next = await this.transition(sshHost, {
+ status,
+ lastError: message,
+ consecutiveFailures: this.entry(sshHost).reachability.consecutiveFailures + 1,
+ });
+ if (status === 'unreachable') this.scheduleProbe(sshHost);
+ return next;
+ } finally {
+ this.entry(sshHost).inFlight = undefined;
+ this.setProbing(sshHost, false);
+ }
+ })();
+
+ entry.inFlight = run;
+ return run;
+ }
+
+ /**
+ * Fold the pooled connection's own lifecycle into the host model, so a host
+ * that recovers (or dies) through ordinary session traffic updates the shared
+ * state without waiting for the next probe.
+ */
+ handleSshConnectionEvent(sshHost: string, event: SshConnectionEvent): void {
+ if (event.type === 'connected' || event.type === 'reconnected') {
+ this.reportSuccess(sshHost);
+ return;
+ }
+ if (event.type === 'reconnect-failed') {
+ void this.transition(sshHost, {
+ status: 'suspended',
+ lastError: this.get(sshHost).lastError ?? 'SSH authentication failed',
+ consecutiveFailures: this.get(sshHost).consecutiveFailures + 1,
+ });
+ return;
+ }
+ if (event.type === 'error') {
+ this.reportFailure(sshHost, new Error(event.errorMessage));
+ }
+ }
+
+ /** After a system resume every host is suspect — re-probe the blocked ones. */
+ handleSystemResume(): void {
+ for (const [sshHost, entry] of this.hosts) {
+ if (isHostBlocked(entry.reachability)) void this.checkNow(sshHost);
+ }
+ }
+
+ /** Stop all timers (app shutdown / tests). */
+ dispose(): void {
+ for (const entry of this.hosts.values()) this.cancelProbe(entry);
+ }
+
+ private entry(sshHost: string): HostEntry {
+ let entry = this.hosts.get(sshHost);
+ if (!entry) {
+ entry = { reachability: unknownHostReachability(sshHost) };
+ this.hosts.set(sshHost, entry);
+ }
+ return entry;
+ }
+
+ private setProbing(sshHost: string, probing: boolean): void {
+ const entry = this.entry(sshHost);
+ if (entry.reachability.probing === probing) return;
+ entry.reachability = { ...entry.reachability, probing };
+ this.deps.publish(entry.reachability);
+ }
+
+ private async transition(
+ sshHost: string,
+ patch: Pick<HostReachability, 'status' | 'lastError' | 'consecutiveFailures'>
+ ): Promise<HostReachability> {
+ const entry = this.entry(sshHost);
+ const previous = entry.reachability.status;
+ const checkedAt = new Date(this.deps.now()).toISOString();
+
+ const next: HostReachability = {
+ ...entry.reachability,
+ ...patch,
+ sshHost,
+ lastCheckedAt: checkedAt,
+ lastReachableAt:
+ patch.status === 'reachable' ? checkedAt : entry.reachability.lastReachableAt,
+ nextProbeAt: patch.status === 'reachable' ? null : entry.reachability.nextProbeAt,
+ };
+ entry.reachability = next;
+
+ // Log only on a status change: an unreachable host must be loud once, not
+ // once per retry — the log spam was half of what CHOO-1682 reports.
+ if (previous !== next.status) {
+ const metadata = { sshHost, previous, status: next.status, error: next.lastError };
+ if (next.status === 'reachable') {
+ this.deps.log.info('HostReachabilityService: host reachable', metadata);
+ } else {
+ this.deps.log.warn('HostReachabilityService: host state changed', metadata);
+ }
+ }
+
+ this.deps.publish(next);
+ if (previous !== next.status) {
+ this.emit('change', { previous, current: next } satisfies HostReachabilityChange);
+ }
+
+ await this.persist(next);
+ return next;
+ }
+
+ private async persist(reachability: HostReachability): Promise<void> {
+ const record: PersistedHostReachability = {
+ sshHost: reachability.sshHost,
+ status: reachability.status,
+ lastError: reachability.lastError,
+ lastCheckedAt: reachability.lastCheckedAt,
+ lastReachableAt: reachability.lastReachableAt,
+ consecutiveFailures: reachability.consecutiveFailures,
+ };
+ try {
+ await savePersistedReachability(record);
+ } catch (error) {
+ // Persistence is a convenience (surviving restart); losing it must not
+ // take down the in-memory model every consumer depends on.
+ this.deps.log.warn('HostReachabilityService: failed to persist reachability', {
+ sshHost: reachability.sshHost,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ }
+
+ private scheduleProbe(sshHost: string): void {
+ const entry = this.entry(sshHost);
+ this.cancelProbe(entry);
+ if (entry.reachability.status === 'suspended') return;
+
+ const delayMs = probeDelayFor(entry.reachability.consecutiveFailures);
+ entry.reachability = {
+ ...entry.reachability,
+ nextProbeAt: new Date(this.deps.now() + delayMs).toISOString(),
+ };
+ this.deps.publish(entry.reachability);
+
+ entry.timer = this.deps.setTimer(() => {
+ entry.timer = undefined;
+ void this.checkNow(sshHost);
+ }, delayMs);
+ }
+
+ private cancelProbe(entry: HostEntry): void {
+ if (entry.timer) {
+ this.deps.clearTimer(entry.timer);
+ entry.timer = undefined;
+ }
+ if (entry.reachability.nextProbeAt !== null) {
+ entry.reachability = { ...entry.reachability, nextProbeAt: null };
+ }
+ }
+}
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.ts
new file mode 100644
index 000000000..5d0f446bc
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/host-reachability-startup.ts
@@ -0,0 +1,44 @@
+import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';
+import type { SshConnectionManagerEvent } from '@main/core/ssh/lifecycle/ssh-connection-manager';
+import { hostReachabilityService } from './production-host-reachability';
+
+/** Reverse of `sshConnectionIdForHost`; null for ids the pool didn't mint for a host. */
+function hostForConnectionId(connectionId: string): string | null {
+ const prefix = 'agent-ssh:';
+ return connectionId.startsWith(prefix) ? connectionId.slice(prefix.length) : null;
+}
+
+/**
+ * Load persisted host reachability and bind the pooled SSH connection's
+ * lifecycle into it, so state stays fresh from ordinary traffic rather than
+ * only from probes.
+ */
+export async function initializeHostReachability(): Promise<void> {
+ await hostReachabilityService.initialize();
+
+ sshConnectionManager.on('connection-event', (event: SshConnectionManagerEvent) => {
+ const sshHost = hostForConnectionId(event.connectionId);
+ if (!sshHost) return;
+ if (event.type === 'connected' || event.type === 'reconnected') {
+ hostReachabilityService.handleSshConnectionEvent(sshHost, {
+ type: event.type,
+ connectionId: event.connectionId,
+ });
+ return;
+ }
+ if (event.type === 'reconnect-failed') {
+ hostReachabilityService.handleSshConnectionEvent(sshHost, {
+ type: 'reconnect-failed',
+ connectionId: event.connectionId,
+ });
+ return;
+ }
+ if (event.type === 'error') {
+ hostReachabilityService.handleSshConnectionEvent(sshHost, {
+ type: 'error',
+ connectionId: event.connectionId,
+ errorMessage: event.error.message,
+ });
+ }
+ });
+}
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.ts
new file mode 100644
index 000000000..d0a91e6d5
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/production-host-reachability.ts
@@ -0,0 +1,26 @@
+import { SshExecutionContext } from '@main/core/execution-context/ssh-execution-context';
+import { sshConnectionIdForHost } from '@main/core/locations/location-transport';
+import { forceSshReconnect } from '@main/core/ssh/connect/connect-agent-ssh';
+import { SshAuthError } from '@main/core/ssh/lifecycle/ssh-connection-manager';
+import { events } from '@main/lib/events';
+import { log } from '@main/lib/logger';
+import { hostReachabilityEventChannel } from '@shared/core/remote-hosts/reachability';
+import { HostReachabilityService } from './host-reachability-service';
+
+/**
+ * The reachability probe: rebuild the host's transport and run a trivial
+ * command. `forceSshReconnect` rather than `ensureSshConnected` because a probe
+ * must answer "can I reach this host *now*", not "is there a pooled connection
+ * object" — a wedged connection would otherwise report success.
+ */
+async function probeHost(sshHost: string): Promise<void> {
+ const proxy = await forceSshReconnect(sshConnectionIdForHost(sshHost), sshHost);
+ await new SshExecutionContext(proxy).exec('uname', ['-s']);
+}
+
+export const hostReachabilityService = new HostReachabilityService({
+ probe: probeHost,
+ isAuthError: (error) => error instanceof SshAuthError,
+ publish: (reachability) => events.emit(hostReachabilityEventChannel, reachability),
+ log,
+});
diff --git a/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.ts b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.ts
new file mode 100644
index 000000000..ec036736c
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/remote-hosts/reachability-store.ts
@@ -0,0 +1,77 @@
+import { eq } from 'drizzle-orm';
+import { db } from '@main/db/client';
+import { remoteHostReachability, type RemoteHostReachabilityRow } from '@main/db/schema';
+import {
+ type HostReachability,
+ type HostReachabilityStatus,
+ unknownHostReachability,
+} from '@shared/core/remote-hosts/reachability';
+
+/** Persisted shape of a reachability record — everything except live probe state. */
+export type PersistedHostReachability = Omit<HostReachability, 'nextProbeAt' | 'probing'>;
+
+const STATUSES: HostReachabilityStatus[] = ['unknown', 'reachable', 'unreachable', 'suspended'];
+
+function toStatus(raw: string, sshHost: string): HostReachabilityStatus {
+ const status = STATUSES.find((s) => s === raw);
+ if (!status) {
+ throw new Error(`Unknown reachability status '${raw}' persisted for host ${sshHost}`);
+ }
+ return status;
+}
+
+function toRecord(row: RemoteHostReachabilityRow): PersistedHostReachability {
+ return {
+ sshHost: row.sshHost,
+ status: toStatus(row.status, row.sshHost),
+ lastError: row.lastError,
+ lastCheckedAt: row.lastCheckedAt,
+ lastReachableAt: row.lastReachableAt,
+ consecutiveFailures: row.consecutiveFailures,
+ };
+}
+
+export async function listPersistedReachability(): Promise<PersistedHostReachability[]> {
+ const rows = await db.select().from(remoteHostReachability);
+ return rows.map(toRecord);
+}
+
+export async function getPersistedReachability(
+ sshHost: string
+): Promise<PersistedHostReachability | null> {
+ const [row] = await db
+ .select()
+ .from(remoteHostReachability)
+ .where(eq(remoteHostReachability.sshHost, sshHost))
+ .limit(1);
+ return row ? toRecord(row) : null;
+}
+
+export async function savePersistedReachability(record: PersistedHostReachability): Promise<void> {
+ const values = {
+ sshHost: record.sshHost,
+ status: record.status,
+ lastError: record.lastError,
+ lastCheckedAt: record.lastCheckedAt,
+ lastReachableAt: record.lastReachableAt,
+ consecutiveFailures: record.consecutiveFailures,
+ updatedAt: new Date().toISOString(),
+ };
+ await db
+ .insert(remoteHostReachability)
+ .values(values)
+ .onConflictDoUpdate({ target: remoteHostReachability.sshHost, set: values });
+}
+
+export async function deletePersistedReachability(sshHost: string): Promise<void> {
+ await db.delete(remoteHostReachability).where(eq(remoteHostReachability.sshHost, sshHost));
+}
+
+/** A persisted record widened back to the full live shape (no probe scheduled yet). */
+export function hydrate(record: PersistedHostReachability): HostReachability {
+ return { ...record, nextProbeAt: null, probing: false };
+}
+
+export function emptyFor(sshHost: string): HostReachability {
+ return unknownHostReachability(sshHost);
+}
diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts
index ccd10ea25..574c60d80 100644
--- a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/connect-agent-ssh.ts
@@ -1,30 +1,40 @@
+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';
import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';
import type { SshClientProxy } from '@main/core/ssh/lifecycle/ssh-client-proxy';
-import { resolveSshConnectConfig } from './resolve-ssh-connect-config';
+import { registerSshResolver } from './ssh-resolver';
/**
- * Register the per-id config resolver (the host's `~/.ssh/config` alias →
- * ssh2 ConnectConfig) and establish the pooled connection. The resolver is
- * reused for auto-reconnect; `connect` coalesces concurrent calls and reuses a
- * live connection, so calling this repeatedly for the same host is safe.
+ * Establish (or reuse) the pooled connection to a host, gated on the host's
+ * modeled reachability.
*
- * host/port/username in the transient config are fallbacks: the real values
- * (and identity/agent) are resolved from the alias via `ssh -G`. Auth goes
- * through the SSH agent, matching switchdash's remote stack — switchdash stores no
- * credentials of its own.
+ * This is the single funnel every remote capability goes through — exec
+ * contexts, SFTP, terminals, PTYs, sidecar deploys, dependency probes, the
+ * managed server — so gating here is what centralizes reachability for all of
+ * them (CHOO-1682). A host known to be unreachable fails fast with the modeled
+ * reason instead of each caller rediscovering it through a 20s SSH timeout, and
+ * every real connect outcome feeds the shared host state so ordinary traffic
+ * keeps it fresh without extra probes.
*/
export async function ensureSshConnected(
connectionId: string,
sshHost: string
): Promise<SshClientProxy> {
+ hostReachabilityService.requireReachable(sshHost);
registerSshResolver(connectionId, sshHost);
- return sshConnectionManager.connect(connectionId);
+ try {
+ const proxy = await sshConnectionManager.connect(connectionId);
+ hostReachabilityService.reportSuccess(sshHost);
+ return proxy;
+ } catch (error) {
+ hostReachabilityService.reportFailure(sshHost, error);
+ throw error;
+ }
}
/**
* Force a full transport rebuild for a host's pooled connection — the manual
- * recovery path behind the UI's refresh. Registers the resolver first so it
- * also works for a host that has not connected yet this app run.
+ * recovery path behind the UI's refresh. Deliberately NOT gated on
+ * reachability: forcing a reconnect is how an unreachable host gets retried.
*/
export async function forceSshReconnect(
connectionId: string,
@@ -33,19 +43,3 @@ export async function forceSshReconnect(
registerSshResolver(connectionId, sshHost);
return sshConnectionManager.forceReconnect(connectionId);
}
-
-function registerSshResolver(connectionId: string, sshHost: string): void {
- sshConnectionManager.register(connectionId, () =>
- resolveSshConnectConfig({
- kind: 'transient',
- config: {
- name: sshHost,
- host: sshHost,
- port: 22,
- username: '',
- sshConfigAlias: sshHost,
- authType: 'agent',
- },
- })
- );
-}
diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.ts
new file mode 100644
index 000000000..fd840a3da
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/connect/ssh-resolver.ts
@@ -0,0 +1,28 @@
+import { sshConnectionManager } from '@main/core/ssh/lifecycle/production-ssh-connection-manager';
+import { resolveSshConnectConfig } from './resolve-ssh-connect-config';
+
+/**
+ * Register the per-id config resolver (the host's `~/.ssh/config` alias →
+ * ssh2 ConnectConfig). Lives apart from `connect-agent-ssh` so the reachability
+ * probe can rebuild a transport without importing the reachability-gated
+ * connect helpers that consume it.
+ *
+ * host/port/username in the transient config are fallbacks: the real values
+ * (and identity/agent) are resolved from the alias via `ssh -G`. Auth goes
+ * through the SSH agent — switchdash stores no credentials of its own.
+ */
+export function registerSshResolver(connectionId: string, sshHost: string): void {
+ sshConnectionManager.register(connectionId, () =>
+ resolveSshConnectConfig({
+ kind: 'transient',
+ config: {
+ name: sshHost,
+ host: sshHost,
+ port: 22,
+ username: '',
+ sshConfigAlias: sshHost,
+ authType: 'agent',
+ },
+ })
+ );
+}
diff --git a/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts b/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
index a2540d9ad..434cc52a1 100644
--- a/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/ssh/lifecycle/ssh-connection-manager.ts
@@ -221,7 +221,9 @@ export class SshConnectionManager extends EventEmitter {
*/
async forceReconnect(id: string): Promise<SshClientProxy> {
const record = this.record(id);
- this.deps.log.warn('SshConnectionManager: force reconnect requested', { connectionId: id });
+ // Routine: this is also the reachability probe's transport rebuild, so it
+ // recurs on the backoff schedule for an unreachable host.
+ this.deps.log.info('SshConnectionManager: force reconnect requested', { connectionId: id });
record.intentional = false;
this.cancelReconnect(record);
if (record.client) {
@@ -517,7 +519,11 @@ export class SshConnectionManager extends EventEmitter {
};
client.on('error', (error: Error) => {
- this.deps.log.error('SshConnectionManager: connection error', {
+ // A remote host being down is an expected condition, not a fault in the
+ // app: laptops sleep, VPNs drop, SSO tokens expire. The host's
+ // reachability state is what escalates a persistent outage — this is
+ // one transport attempt failing, so warn rather than error.
+ this.deps.log.warn('SshConnectionManager: connection error', {
connectionId: id,
error: error.message,
});
@@ -683,7 +689,10 @@ export class SshConnectionManager extends EventEmitter {
// Auth failures won't resolve with blind retries — suspend instead of
// hammering the server. connect()/forceReconnect()/system resume revive.
if (error instanceof SshAuthError) {
- this.deps.log.error('SshConnectionManager: reconnect suspended — auth failure', {
+ // Expected (expired or rotated credentials), and the host's
+ // reachability state surfaces it to the user as `suspended` — the log
+ // does not need to shout about it.
+ this.deps.log.warn('SshConnectionManager: reconnect suspended — auth failure', {
connectionId: id,
});
record.reconnect = undefined;
diff --git a/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts b/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts
index a9bcd5bda..3cb138943 100644
--- a/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts
+++ b/dash/apps/switchdash-desktop/src/main/core/switch-setup/controller.ts
@@ -1,10 +1,23 @@
+import { hostReachabilityService } from '@main/core/remote-hosts/production-host-reachability';
+import { log } from '@main/lib/logger';
import { createRPCController } from '@shared/lib/ipc/rpc';
import { getRemoteSwitchSetupService } from './remote-switch-setup';
import { switchSetupService } from './switch-setup-service';
export const switchSetupController = createRPCController({
listOnboardable: () => switchSetupService.listOnboardable(),
+ /**
+ * Agent types installed on a remote host. An unreachable host has no
+ * answerable list, so return none rather than throwing: this is a query that
+ * paints UI, and its caller renders the host-unreachable state alongside it,
+ * so the degraded result is disclosed. Throwing surfaced an ordinary,
+ * expected condition as an unhandled handler error with a stack trace.
+ */
listOnboardableRemote: async (sshHost: string) => {
+ if (hostReachabilityService.isBlocked(sshHost)) {
+ log.warn('switchSetup.listOnboardableRemote: host unreachable — no agent types', { sshHost });
+ return [];
+ }
const service = await getRemoteSwitchSetupService(sshHost);
const statuses = await service.listAgentTypeStatuses();
return statuses.filter((s) => s.installed).map((s) => ({ agentId: s.agentId }));
diff --git a/dash/apps/switchdash-desktop/src/main/db/schema.ts b/dash/apps/switchdash-desktop/src/main/db/schema.ts
index b984c5b6e..3b9c90b89 100644
--- a/dash/apps/switchdash-desktop/src/main/db/schema.ts
+++ b/dash/apps/switchdash-desktop/src/main/db/schema.ts
@@ -198,6 +198,30 @@ export const remoteHosts = sqliteTable('remote_hosts', {
.default(sql`CURRENT_TIMESTAMP`),
});
+/**
+ * Persisted reachability of a remote SSH host (CHOO-1682). Keyed by the same
+ * `~/.ssh/config` alias as `remote_hosts` and `locations.ssh_host`, but kept in
+ * its own table because a location can name a host that was never onboarded —
+ * reachability must be tracked for any alias we actually talk to, not only the
+ * ones on the remote-hosts page.
+ *
+ * Persisting it means a restart does not forget that a host was down: the app
+ * boots into the known-bad state and schedules a probe, instead of every
+ * host-dependent path racing to rediscover the failure at once.
+ */
+export const remoteHostReachability = sqliteTable('remote_host_reachability', {
+ sshHost: text('ssh_host').primaryKey(),
+ /** One of HostReachabilityStatus — 'unknown' | 'reachable' | 'unreachable' | 'suspended'. */
+ status: text('status').notNull().default('unknown'),
+ lastError: text('last_error'),
+ lastCheckedAt: text('last_checked_at'),
+ lastReachableAt: text('last_reachable_at'),
+ consecutiveFailures: integer('consecutive_failures').notNull().default(0),
+ updatedAt: text('updated_at')
+ .notNull()
+ .default(sql`CURRENT_TIMESTAMP`),
+});
+
export const sessions = sqliteTable(
'sessions',
{
@@ -293,3 +317,5 @@ export type SwitchServerRow = typeof switchServers.$inferSelect;
export type SwitchServerInsert = typeof switchServers.$inferInsert;
export type RemoteHostRow = typeof remoteHosts.$inferSelect;
export type RemoteHostInsert = typeof remoteHosts.$inferInsert;
+export type RemoteHostReachabilityRow = typeof remoteHostReachability.$inferSelect;
+export type RemoteHostReachabilityInsert = typeof remoteHostReachability.$inferInsert;
diff --git a/dash/apps/switchdash-desktop/src/main/index.ts b/dash/apps/switchdash-desktop/src/main/index.ts
index 70266a82a..4b9d491c6 100644
--- a/dash/apps/switchdash-desktop/src/main/index.ts
+++ b/dash/apps/switchdash-desktop/src/main/index.ts
@@ -20,6 +20,8 @@ import { locationSettingsService } from './core/locations/settings/location-sett
import { localServerService } from './core/managed-switch-server/local-server-service';
import { remoteServerService } from './core/managed-switch-server/remote-server-service';
import { promptLibraryService } from './core/prompt-library/service';
+import { initializeHostReachability } from './core/remote-hosts/host-reachability-startup';
+import { hostReachabilityService } from './core/remote-hosts/production-host-reachability';
import {
reconcileResourceSampler,
stopResourceSampler,
@@ -155,6 +157,14 @@ void app.whenReady().then(async () => {
// then start the auto_session watchers — the watcher's "is a session already
// attending this room?" check relies on those connections being present.
void Promise.all([agentHookReady, dependenciesReady, migrationReady]).then(async () => {
+ // Must precede the remote watchers: they gate on host reachability, and
+ // starting them first would let every agent on a known-down host attempt a
+ // connect before the persisted state is loaded.
+ try {
+ await initializeHostReachability();
+ } catch (e) {
+ log.error('Failed to initialise host reachability at startup:', e);
+ }
try {
await restoreSwitchRoomSessions();
} catch (e) {
@@ -183,6 +193,7 @@ void app.whenReady().then(async () => {
powerMonitor.on('resume', () => {
log.info('powerMonitor: system resumed — refreshing SSH connections');
sshConnectionManager.handleSystemResume();
+ hostReachabilityService.handleSystemResume();
});
setupAppProtocol(join(app.getAppPath(), 'out', 'renderer'));
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
index d7a04e67d..8be0de954 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/add-agent-modal.tsx
@@ -9,6 +9,8 @@ import type {
} from '@renderer/features/locations/stores/agent-onboarding-types';
import { agentsStore } from '@renderer/features/locations/stores/agents-store';
import { getLocationManagerStore } from '@renderer/features/locations/stores/location-selectors';
+import { HostReachabilityNotice } from '@renderer/features/remote-hosts/host-reachability-notice';
+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';
import { policyHasDeadRule } from '@renderer/features/switch-servers/addressing-policy-editor';
import type { ServerVerifyState } from '@renderer/features/switch-servers/agent-server-picker';
import { switchServersStore } from '@renderer/features/switch-servers/switch-servers-store';
@@ -273,12 +275,17 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
// Remote submit gate: a provider, a remote dir, a detected agent, and a
// verified server. Name defaults from the remote dir basename (see below), so
// it is not separately gated. Local submit keeps the existing pick validity.
+ // Never create an agent on a host we know we cannot reach — it would be born
+ // into the failing state this ticket exists to surface (CHOO-1676).
+ const runHostReachable = !isRemoteRun || !hostReachabilityStore.isBlocked(runHost);
+
const canSubmitDetected = isRemoteRun
? !!pickState.providerId &&
trimmedRemoteDir.length > 0 &&
!isChecking &&
!!switchAgent &&
verifyState === 'found' &&
+ runHostReachable &&
submitState === 'idle'
: pickState.isValid &&
!isChecking &&
@@ -293,6 +300,7 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
!!pickState.serverId &&
!!pickState.providerId &&
remoteRunValid &&
+ runHostReachable &&
submitState === 'idle';
// Remote configure gate: no agent in the remote dir yet — a valid remote
@@ -307,6 +315,7 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
!!pickState.serverId &&
!!pickState.providerId &&
trimmedRemoteDir.length > 0 &&
+ runHostReachable &&
submitState === 'idle';
const reportCreationError = (error: AgentOnboardingError) => {
@@ -581,11 +590,6 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
}
>
<DialogContentArea data-autofocus tabIndex={-1} className="max-h-[calc(100dvh-13rem)] gap-4">
- <AgentTypePicker
- value={pickState.providerId}
- onChange={pickState.setProviderId}
- sshHost={isRemoteRun ? runHost : undefined}
- />
<Field>
<FieldLabel>Run location</FieldLabel>
<Select value={runHost} onValueChange={(v) => setRunHost(v ?? LOCAL_RUN_LOCATION)}>
@@ -608,7 +612,8 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
: 'This server runs on this computer, so its agents run here too.'}
</p>
)}
- {isRemoteRun && (
+ {isRemoteRun && <HostReachabilityNotice sshHost={runHost} />}
+ {isRemoteRun && runHostReachable && (
<div className="flex items-center gap-2">
<Input
value={remoteRepoDirDraft}
@@ -632,7 +637,19 @@ export const AddAgentModal = observer(function AddAgentModal({ onClose }: AddLoc
</div>
)}
</Field>
- {!isRemoteRun && <PickExistingPanel state={pickState} showName={!isMissingSwitchAgent} />}
+ {/* The host is the first gate: with it unreachable we cannot know which
+ agent types it has, so offering a type picker (or a directory to scan)
+ would be guessing. Everything below waits for a usable host. */}
+ {runHostReachable && (
+ <AgentTypePicker
+ value={pickState.providerId}
+ onChange={pickState.setProviderId}
+ sshHost={isRemoteRun ? runHost : undefined}
+ />
+ )}
+ {runHostReachable && !isRemoteRun && (
+ <PickExistingPanel state={pickState} showName={!isMissingSwitchAgent} />
+ )}
{isChecking && (
<p className="text-sm text-foreground-muted">Scanning directory for agents…</p>
)}
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx
index 04fa57e43..dce991809 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/add-agent-modal/agent-type-picker.tsx
@@ -48,9 +48,22 @@ export function AgentTypePicker({
return (
<div className="flex items-start gap-2 rounded-md border border-border bg-background-1 px-2 py-1.5 text-xs text-foreground-muted">
<CircleAlert className="mt-0.5 size-3.5 shrink-0 text-amber-500" />
+ {/* Name the host: "no agent type is set up" reads as a global problem,
+ but availability is per-host — the connector may well be installed
+ locally and simply missing on the machine being targeted. */}
<span>
- No agent type is set up for Switch yet. Install an agent&apos;s Switch connector in
- Settings &rarr; Agents before onboarding it.
+ {sshHost ? (
+ <>
+ No agent type is set up for Switch on <span className="font-medium">{sshHost}</span>.
+ Install an agent&apos;s Switch connector on that host in Settings &rarr; Remote hosts
+ before onboarding it there.
+ </>
+ ) : (
+ <>
+ No agent type is set up for Switch on this computer. Install an agent&apos;s Switch
+ connector in Settings &rarr; Agents before onboarding it.
+ </>
+ )}
</span>
</div>
);
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx
index e52b6636e..429cb28ff 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/components/main-panel/main-panel.tsx
@@ -1,6 +1,8 @@
import { Loader2, RefreshCw, TriangleAlert } from 'lucide-react';
import { observer } from 'mobx-react-lite';
import { useConfirmDeleteAgent } from '@renderer/features/locations/hooks/use-confirm-delete-agent';
+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';
+import { HostUnreachablePanel } from '@renderer/features/remote-hosts/host-unreachable-panel';
import { useParams } from '@renderer/lib/layout/navigation-provider';
import { isUnregisteredLocation } from '../../stores/location';
import {
@@ -25,6 +27,14 @@ export const LocationMainPanel = observer(function LocationMainPanel() {
return <PendingLocationStatus location={store} />;
}
+ // A blocked host outranks every other failure state: the location cannot
+ // bootstrap, mount, or find its path while its host is down, and showing the
+ // downstream symptom instead of the cause is what CHOO-1682 is about.
+ const sshHost = store?.data?.sshHost ?? null;
+ if (sshHost && hostReachabilityStore.isBlocked(sshHost) && kind !== 'ready') {
+ return <HostUnreachablePanel reachability={hostReachabilityStore.get(sshHost)} />;
+ }
+
if (kind === 'bootstrapping') {
return <LocationBootstrappingPanel />;
}
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts b/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts
index ec83e8fcf..8a83b1907 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts
+++ b/dash/apps/switchdash-desktop/src/renderer/features/locations/stores/location-manager.ts
@@ -1,9 +1,10 @@
import { err, ok } from '@switchdash/shared';
import { makeObservable, observable, runInAction } from 'mobx';
-import { rpc } from '@renderer/lib/ipc';
+import { events, rpc } from '@renderer/lib/ipc';
import { appState } from '@renderer/lib/stores/app-state';
import { viewStateCache } from '@renderer/lib/stores/view-state-cache';
import { type Location } from '@shared/core/locations/locations';
+import { hostReachabilityEventChannel } from '@shared/core/remote-hosts/reachability';
import type { LocationViewSnapshot } from '@shared/view-state';
import type {
AgentOnboardingCompletion,
@@ -29,6 +30,23 @@ export class LocationManagerStore {
constructor() {
makeObservable(this, { locations: observable, pendingCreationIds: observable });
+
+ // A location that failed to open because its host was down holds an error
+ // that is meaningless the moment the host is back. Without this the user
+ // has to retry twice — once for the host, then again for each location —
+ // and the second failure looks like the fix did not work (CHOO-1682).
+ events.on(hostReachabilityEventChannel, (reachability) => {
+ if (reachability.status === 'reachable') this.remountFailedOnHost(reachability.sshHost);
+ });
+ }
+
+ /** Re-open every errored location on a host, e.g. after it becomes reachable. */
+ remountFailedOnHost(sshHost: string): void {
+ for (const [id, store] of this.locations) {
+ if (store.data?.sshHost !== sshHost) continue;
+ if (!isUnmountedLocation(store) || store.phase !== 'error') continue;
+ void this.mountLocation(id);
+ }
}
load(): Promise<void> {
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsx b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsx
new file mode 100644
index 000000000..c61f11c22
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-notice.tsx
@@ -0,0 +1,75 @@
+import { Loader2, PlugZap, RefreshCw } from 'lucide-react';
+import { observer } from 'mobx-react-lite';
+import { useEffect } from 'react';
+import { Button } from '@renderer/lib/ui/button';
+import { hostReachabilityStore } from './host-reachability-store';
+
+/**
+ * Inline reachability warning for forms that target a remote host — the
+ * add-agent modal above all (CHOO-1676). Validating up front means the user
+ * learns the host is unreachable while they can still change it, instead of
+ * creating an agent that immediately lands in a failing state.
+ *
+ * Renders nothing when the host is fine, so it can be dropped into a form
+ * unconditionally.
+ */
+export const HostReachabilityNotice = observer(function HostReachabilityNotice({
+ sshHost,
+}: {
+ sshHost: string;
+}) {
+ // Verify on selection rather than trusting a possibly stale record: the user
+ // is about to commit to this host.
+ useEffect(() => {
+ void hostReachabilityStore.hydrate().then(() => {
+ if (hostReachabilityStore.get(sshHost).status !== 'reachable') {
+ void hostReachabilityStore.retry(sshHost);
+ }
+ });
+ }, [sshHost]);
+
+ const reachability = hostReachabilityStore.get(sshHost);
+ const retrying = hostReachabilityStore.isRetrying(sshHost) || reachability.probing;
+
+ if (retrying) {
+ return (
+ <div className="flex items-center gap-2 rounded-md border border-border bg-background-1 px-2.5 py-2 text-xs text-foreground-muted">
+ <Loader2 className="size-3.5 shrink-0 animate-spin" />
+ <span>
+ Checking that <span className="font-medium text-foreground">{sshHost}</span> is reachable…
+ </span>
+ </div>
+ );
+ }
+
+ if (!hostReachabilityStore.isBlocked(sshHost)) return null;
+
+ return (
+ <div className="flex items-start gap-2.5 rounded-md border border-amber-500/30 bg-amber-500/8 px-2.5 py-2">
+ <PlugZap className="mt-0.5 size-4 shrink-0 text-amber-500" />
+ <div className="flex min-w-0 flex-1 flex-col gap-1">
+ <p className="text-xs font-medium text-foreground">
+ {reachability.status === 'suspended'
+ ? `SSH authentication to ${sshHost} failed`
+ : `Can’t reach ${sshHost}`}
+ </p>
+ {reachability.lastError && (
+ <p className="text-xs break-words text-foreground-muted">{reachability.lastError}</p>
+ )}
+ <p className="text-xs text-foreground-passive">
+ Choose a different run location, or bring the host back and retry.
+ </p>
+ </div>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="shrink-0"
+ onClick={() => void hostReachabilityStore.retry(sshHost)}
+ >
+ <RefreshCw className="size-3" />
+ Retry
+ </Button>
+ </div>
+ );
+});
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.ts b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.ts
new file mode 100644
index 000000000..4195aff0a
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-reachability-store.ts
@@ -0,0 +1,77 @@
+import { makeAutoObservable, runInAction } from 'mobx';
+import { events, rpc } from '@renderer/lib/ipc';
+import {
+ type HostReachability,
+ hostReachabilityEventChannel,
+ isHostBlocked,
+ unknownHostReachability,
+} from '@shared/core/remote-hosts/reachability';
+
+/**
+ * Renderer mirror of the main process's host reachability model (CHOO-1682).
+ *
+ * Every surface that needs to say "this host is down" — the sidebar badge, the
+ * location panel, the add-agent modal — reads from here, so they cannot
+ * disagree about a host's state or each render its own interpretation of a raw
+ * ssh2 error. Hydrated once, then kept current by pushed events.
+ */
+export class HostReachabilityStore {
+ private readonly byHost = new Map<string, HostReachability>();
+ /** Hosts with a retry in flight, so buttons can show progress. */
+ readonly retrying = new Set<string>();
+ private hydrated = false;
+
+ constructor() {
+ makeAutoObservable(this);
+ events.on(hostReachabilityEventChannel, (reachability) => {
+ runInAction(() => {
+ this.byHost.set(reachability.sshHost, reachability);
+ });
+ });
+ }
+
+ /** Load the current state for every known host. Idempotent. */
+ async hydrate(): Promise<void> {
+ if (this.hydrated) return;
+ this.hydrated = true;
+ const all = await rpc.remoteHosts.listReachability();
+ runInAction(() => {
+ for (const reachability of all) this.byHost.set(reachability.sshHost, reachability);
+ });
+ }
+
+ /**
+ * Reachability for a host. An unseen host reads as `unknown` rather than
+ * throwing — the UI treats "not yet probed" as "no reason to warn".
+ */
+ get(sshHost: string): HostReachability {
+ return this.byHost.get(sshHost) ?? unknownHostReachability(sshHost);
+ }
+
+ /** Whether a host is in a state that pauses host-dependent work. */
+ isBlocked(sshHost: string | null | undefined): boolean {
+ if (!sshHost) return false;
+ return isHostBlocked(this.get(sshHost));
+ }
+
+ isRetrying(sshHost: string): boolean {
+ return this.retrying.has(sshHost);
+ }
+
+ /**
+ * Probe now, bypassing the backoff. On success this un-pauses every paused
+ * host-dependent path in the main process, not just the current view.
+ */
+ async retry(sshHost: string): Promise<HostReachability> {
+ runInAction(() => this.retrying.add(sshHost));
+ try {
+ const reachability = await rpc.remoteHosts.retryHost(sshHost);
+ runInAction(() => this.byHost.set(sshHost, reachability));
+ return reachability;
+ } finally {
+ runInAction(() => this.retrying.delete(sshHost));
+ }
+ }
+}
+
+export const hostReachabilityStore = new HostReachabilityStore();
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsx b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsx
new file mode 100644
index 000000000..9668efa29
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/renderer/features/remote-hosts/host-unreachable-panel.tsx
@@ -0,0 +1,80 @@
+import { PlugZap, RefreshCw } from 'lucide-react';
+import { observer } from 'mobx-react-lite';
+import { Button } from '@renderer/lib/ui/button';
+import type { HostReachability } from '@shared/core/remote-hosts/reachability';
+import { hostReachabilityStore } from './host-reachability-store';
+
+function relativeToNow(iso: string | null): string | null {
+ if (!iso) return null;
+ const deltaMs = new Date(iso).getTime() - Date.now();
+ const seconds = Math.round(Math.abs(deltaMs) / 1000);
+ if (seconds < 60) return `${seconds}s`;
+ const minutes = Math.round(seconds / 60);
+ if (minutes < 60) return `${minutes}m`;
+ return `${Math.round(minutes / 60)}h`;
+}
+
+/**
+ * The single "this host is down" surface (CHOO-1682). Replaces the raw ssh2
+ * message ("Connection lost before handshake") that used to reach the user with
+ * the modeled state: what is wrong, that work is paused rather than silently
+ * retrying forever, when the next automatic probe lands, and one button to
+ * retry now.
+ */
+export const HostUnreachablePanel = observer(function HostUnreachablePanel({
+ reachability,
+}: {
+ reachability: HostReachability;
+}) {
+ const { sshHost, status, lastError, nextProbeAt, lastReachableAt } = reachability;
+ const retrying = hostReachabilityStore.isRetrying(sshHost) || reachability.probing;
+ const nextProbe = relativeToNow(nextProbeAt);
+ const lastSeen = relativeToNow(lastReachableAt);
+ const suspended = status === 'suspended';
+
+ return (
+ <div className="flex h-full w-full items-center justify-center p-8">
+ <div className="flex w-full max-w-md flex-col items-center gap-4 rounded-lg border border-amber-500/25 bg-amber-500/5 px-6 py-7 text-center">
+ <div className="flex size-10 items-center justify-center rounded-full bg-amber-500/12">
+ <PlugZap className="size-5 text-amber-500" />
+ </div>
+
+ <div className="flex flex-col gap-1.5">
+ <p className="text-sm font-medium text-foreground">
+ {suspended ? 'SSH authentication failed' : 'Host unreachable'}
+ </p>
+ <p className="font-mono text-xs text-foreground-muted">{sshHost}</p>
+ </div>
+
+ {lastError && (
+ <p className="max-w-sm text-xs break-words text-foreground-muted">{lastError}</p>
+ )}
+
+ <p className="max-w-sm text-xs text-foreground-passive">
+ {suspended
+ ? 'Automatic retries are paused — a rejected credential will not fix itself. Restore SSH access to this host, then retry.'
+ : 'Work on this host is paused so it is not retried continuously. It resumes automatically once the host is back.'}
+ </p>
+
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ disabled={retrying}
+ onClick={() => void hostReachabilityStore.retry(sshHost)}
+ >
+ <RefreshCw className={retrying ? 'size-3.5 animate-spin' : 'size-3.5'} />
+ {retrying ? 'Checking…' : 'Retry connection'}
+ </Button>
+
+ {(nextProbe || lastSeen) && (
+ <div className="flex items-center gap-2 text-[11px] text-foreground-passive">
+ {!suspended && nextProbe && <span>Next check in {nextProbe}</span>}
+ {!suspended && nextProbe && lastSeen && <span aria-hidden>·</span>}
+ {lastSeen && <span>Last reachable {lastSeen} ago</span>}
+ </div>
+ )}
+ </div>
+ </div>
+ );
+});
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx
index fee67e138..ced907eae 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/agent-item.tsx
@@ -3,6 +3,7 @@ import {
Bot,
ChevronRight,
ExternalLink,
+ PlugZap,
Plus,
RotateCcw,
Server,
@@ -15,6 +16,7 @@ import {
getLocationStore,
locationViewKind,
} from '@renderer/features/locations/stores/location-selectors';
+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';
import { hasSessionError } from '@renderer/features/sessions/stores/session-selectors';
import { switchRoomsStore } from '@renderer/features/switch-servers/switch-rooms-store';
import { AgentIcon } from '@renderer/lib/components/agent-icon';
@@ -100,6 +102,10 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({
if (!location) return null;
+ const sshHost = location.data?.sshHost ?? null;
+ const hostReachability = sshHost ? hostReachabilityStore.get(sshHost) : null;
+ const hostUnreachable = hostReachabilityStore.isBlocked(sshHost);
+
const iconClass =
'absolute h-4 w-4 opacity-100 transition-opacity duration-150 group-hover/row:opacity-0';
const gatewayUrl =
@@ -151,7 +157,12 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({
{location.data?.sshHost != null && (
<Tooltip>
<TooltipTrigger>
- <Server className="h-3.5 w-3.5 shrink-0 text-foreground-muted" />
+ <Server
+ className={cn(
+ 'h-3.5 w-3.5 shrink-0',
+ hostUnreachable ? 'text-foreground-warning' : 'text-foreground-muted'
+ )}
+ />
</TooltipTrigger>
<TooltipContent>
Runs remotely on {location.data.sshHost}
@@ -159,6 +170,22 @@ export const SidebarAgentItem = observer(function SidebarAgentItem({
</TooltipContent>
</Tooltip>
)}
+ {/* The host being down is why this agent is idle, so say so on
+ the row itself — previously you had to select the agent to
+ discover its host was failing to connect (CHOO-1682). */}
+ {hostUnreachable && hostReachability && (
+ <Tooltip>
+ <TooltipTrigger>
+ <PlugZap className="h-3.5 w-3.5 shrink-0 text-foreground-warning" />
+ </TooltipTrigger>
+ <TooltipContent>
+ {hostReachability.status === 'suspended'
+ ? `SSH authentication to ${hostReachability.sshHost} failed — work is paused until you retry`
+ : `Host ${hostReachability.sshHost} is unreachable — work is paused`}
+ {hostReachability.lastError ? ` · ${hostReachability.lastError}` : ''}
+ </TooltipContent>
+ </Tooltip>
+ )}
{locationViewKind(location) === 'ready' && hasSessionError(agent.locationId) && (
<Tooltip>
<TooltipTrigger>
diff --git a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx
index f22aa87a3..4ba6923dc 100644
--- a/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx
+++ b/dash/apps/switchdash-desktop/src/renderer/features/sidebar/sidebar-grouped-list.tsx
@@ -2,6 +2,7 @@ import { observer } from 'mobx-react-lite';
import { Fragment, useEffect } from 'react';
import { agentsStore } from '@renderer/features/locations/stores/agents-store';
import type { LocationStore } from '@renderer/features/locations/stores/location';
+import { hostReachabilityStore } from '@renderer/features/remote-hosts/host-reachability-store';
import type { SessionStore } from '@renderer/features/sessions/stores/session-store';
import { switchRoomsStore as roomConnectionsStore } from '@renderer/features/switch-rooms/switch-rooms-store';
import { switchRoomsStore } from '@renderer/features/switch-servers/switch-rooms-store';
@@ -59,6 +60,7 @@ export const SidebarGroupedList = observer(function SidebarGroupedList() {
useEffect(() => {
roomConnectionsStore.ensureLoaded();
void agentsStore.load();
+ void hostReachabilityStore.hydrate();
void switchServersStore.init().then(() => switchRoomsStore.loadRoomNames());
const onFocus = () => {
void agentsStore.load();
diff --git a/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.ts b/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.ts
new file mode 100644
index 000000000..f6bf5edeb
--- /dev/null
+++ b/dash/apps/switchdash-desktop/src/shared/core/remote-hosts/reachability.ts
@@ -0,0 +1,87 @@
+/**
+ * First-class model of a remote SSH host's reachability (CHOO-1682).
+ *
+ * Before this, "can we reach host X?" was answered independently by every
+ * consumer: the pooled SSH connection's in-memory state, an on-demand
+ * `testConnection`, or simply attempting the work and interpreting the ssh2
+ * error. That produced two failure modes — unbounded retry loops hammering a
+ * dead host (the reconciler reconnecting every 2s per agent), and ad-hoc raw
+ * transport errors surfacing in the UI with no shared notion of "this host is
+ * down". Reachability is now one per-host state machine that every
+ * host-dependent path consults up front.
+ */
+
+import { defineEvent } from '@shared/lib/ipc/events';
+
+/**
+ * Where a host sits in the reachability state machine.
+ *
+ * - `unknown` — never probed this app run (and nothing persisted). Work is
+ * allowed through: the first attempt is what establishes reachability.
+ * - `reachable` — a probe or live connection succeeded. Work proceeds.
+ * - `unreachable` — probes are failing. Host-dependent background work pauses;
+ * a bounded backoff probe keeps checking so recovery is automatic.
+ * - `suspended` — authentication failed. Retrying a rejected key never
+ * self-heals, so no automatic probing happens; the user must fix auth and
+ * retry explicitly.
+ */
+export type HostReachabilityStatus = 'unknown' | 'reachable' | 'unreachable' | 'suspended';
+
+/** The full reachability record for one host alias. */
+export type HostReachability = {
+ /** The `~/.ssh/config` Host alias this record describes. */
+ sshHost: string;
+ status: HostReachabilityStatus;
+ /** Message from the most recent failed probe; null while reachable. */
+ lastError: string | null;
+ /** ISO timestamp of the last probe (successful or not). */
+ lastCheckedAt: string | null;
+ /** ISO timestamp of the last time the host was confirmed reachable. */
+ lastReachableAt: string | null;
+ /** Consecutive failed probes — drives the backoff step. */
+ consecutiveFailures: number;
+ /** ISO timestamp of the next scheduled probe; null when none is scheduled. */
+ nextProbeAt: string | null;
+ /** True while a probe is in flight (for spinners; not a persisted state). */
+ probing: boolean;
+};
+
+/**
+ * Whether host-dependent work should be held back. `unknown` is deliberately
+ * NOT blocked — a host we have never probed gets one real attempt rather than
+ * being gated behind a probe that would duplicate it.
+ */
+export function isHostBlocked(reachability: HostReachability): boolean {
+ return reachability.status === 'unreachable' || reachability.status === 'suspended';
+}
+
+/** Human-readable one-liner for the blocked state, used in errors and the UI. */
+export function hostBlockedReason(reachability: HostReachability): string {
+ const since = reachability.lastCheckedAt ? ` since ${reachability.lastCheckedAt}` : '';
+ if (reachability.status === 'suspended') {
+ return `SSH authentication to ${reachability.sshHost} failed${since}: ${
+ reachability.lastError ?? 'unknown error'
+ }. Fix the host's SSH credentials, then retry.`;
+ }
+ return `Host ${reachability.sshHost} is unreachable${since}: ${
+ reachability.lastError ?? 'unknown error'
+ }`;
+}
+
+export function unknownHostReachability(sshHost: string): HostReachability {
+ return {
+ sshHost,
+ status: 'unknown',
+ lastError: null,
+ lastCheckedAt: null,
+ lastReachableAt: null,
+ consecutiveFailures: 0,
+ nextProbeAt: null,
+ probing: false,
+ };
+}
+
+/** Pushed to the renderer whenever a host's reachability record changes. */
+export const hostReachabilityEventChannel = defineEvent<HostReachability>(
+ 'remote-hosts:reachability-changed'
+);