Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.
TypeScript adhdagent-orchestrationai-agentscanvasclaude-codecodexcoding-agentsdeveloper-toolselectrongeminilinuxmacosmultiplexeropencodeparallel-agentsterminalterminal-emulatorterminal-multiplexertmuxworkspace-manager
fix(ssh): don't flash "lost verified connection" on one slow tunnel probe
review6033366ec783
diff --git a/src/main/remote-ssh/remote-hooks.ts b/src/main/remote-ssh/remote-hooks.tsindex cb5f0d335..95c858907 100644--- a/src/main/remote-ssh/remote-hooks.ts+++ b/src/main/remote-ssh/remote-hooks.ts@@ -54,6 +54,7 @@ import {mergeInstructionsBlock} from '../../core/context-link-core'import { posixQuote, type SshConnection } from '../../shared/ssh'+import { describeTunnelProbe } from './tunnel-repair'/** POSIX dirname of an absolute remote path. `path.dirname` would apply the LOCAL separator* rules, which is wrong the moment the desktop is Windows and the host is Linux. */@@ -128,10 +129,16 @@ export class RemoteHooks {* run, which is a tunnel that cannot deliver, not an unknown. The caller repairs by re-running* `setup()`, which is idempotent and re-verifies end-to-end.*/- async tunnelAlive(projectId: string, conn: SshConnection, controlPath: string, token: string): Promise<boolean> {+ async tunnelAlive(+ projectId: string,+ conn: SshConnection,+ controlPath: string,+ token: string+ ): Promise<{ alive: boolean; detail: string }> {const spec = this.specs.get(projectId)- if (!spec || !token) return false- return this.verifyTunnel(conn, controlPath, spec.sock, token)+ if (!spec) return { alive: false, detail: 'no forward registered for this project in this app run' }+ if (!token) return { alive: false, detail: 'no hook token' }+ return this.probeTunnel(conn, controlPath, spec.sock, token)}async setup(@@ -928,6 +935,16 @@ export class RemoteHooks {sock: string,token: string): Promise<boolean> {+ return (await this.probeTunnel(conn, controlPath, sock, token)).alive+ }++ /** `verifyTunnel` with the reason kept: the watchdog logs WHY a live tunnel stopped answering. */+ private async probeTunnel(+ conn: SshConnection,+ controlPath: string,+ sock: string,+ token: string+ ): Promise<{ alive: boolean; detail: string }> {try {const cmd =`curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST --unix-socket ${posixQuote(sock)} ` +@@ -946,9 +963,10 @@ export class RemoteHooks {childArgs(conn, controlPath, cmd),curlHeaderConfigLine('x-nodeterm-hook-token', token))- return r.code === 0 && r.stdout.trim() === '204'- } catch {- return false+ const alive = r.code === 0 && r.stdout.trim() === '204'+ return { alive, detail: alive ? 'ok' : describeTunnelProbe(r.code, r.stdout) }+ } catch (e) {+ return { alive: false, detail: `probe threw: ${e instanceof Error ? e.message : String(e)}` }}}diff --git a/src/main/remote-ssh/ssh-project.test.ts b/src/main/remote-ssh/ssh-project.test.tsindex 41cdc2070..d78f25b83 100644--- a/src/main/remote-ssh/ssh-project.test.ts+++ b/src/main/remote-ssh/ssh-project.test.ts@@ -1482,6 +1482,41 @@ describe('SshProjectManager', () => {expect(onStatus).toHaveBeenCalledWith({ projectId: 'p1', status: 'connected', hookTunnelVerified: true })})+ it('ONE failed probe never raises the banner — neither when the repair heals it nor when the next probe does', async () => {+ // Field report (Linux desktop → Mac on the same desk, no sleep): "Agent status and canvas+ // control lost their verified connection" kept appearing. The probe is one `curl -m 5` over+ // the shared master, and a single slow round trip used to put the banner up by itself.+ let dead = 0+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const mgr = makeVerifiedMgr(vi.fn(), () => (dead-- > 0 ? '000' : '204'))+ const onStatus = (mgr as unknown as { r: { onStatus: ReturnType<typeof vi.fn> } }).r.onStatus+ await mgr.connect('p1', conn, '/remote/cwd')+ // (a) probe fails once, the repair's own verify succeeds → healed silently.+ dead = 1+ await mgr.connect('p1', conn, '/remote/cwd')+ // (b) probe AND repair fail once, the next tick's probe answers → still silent.+ dead = 3 // the probe + both of setup()'s verify attempts+ await mgr.connect('p1', conn, '/remote/cwd')+ await mgr.connect('p1', conn, '/remote/cwd')+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified !== undefined)).toEqual([])+ // …but every failure is logged with its cause, the only field evidence there is.+ expect(warn.mock.calls.map((c) => String(c[0])).filter((l) => l.includes('hook tunnel probe failed'))).toEqual([+ '[ssh-project] hook tunnel probe failed for p1 (#1): unexpected HTTP answer (exit 0, http 000)',+ '[ssh-project] hook tunnel probe failed for p1 (#1): unexpected HTTP answer (exit 0, http 000)'+ ])+ })++ it('TWO consecutive failed probes raise the banner, even while the repair is backing off', async () => {+ const mgr = makeVerifiedMgr(vi.fn(), () => '000')+ vi.spyOn(console, 'warn').mockImplementation(() => {})+ const onStatus = (mgr as unknown as { r: { onStatus: ReturnType<typeof vi.fn> } }).r.onStatus+ await mgr.connect('p1', conn, '/remote/cwd') // establish fails verification → no spec+ await mgr.connect('p1', conn, '/remote/cwd') // failure #1 → repair attempted, fails+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified === false)).toHaveLength(0)+ await mgr.connect('p1', conn, '/remote/cwd') // failure #2 → inside backoff, still reported+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified === false)).toHaveLength(1)+ })+it('rebinds the forward on repair — the endpoint is re-advertised, not merely re-probed', async () => {// The whole failure is a master with no `-R`, so a repair that did not call `-O forward`// would leave every hook POST dying exactly as before while reporting success.diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.tsindex 86662005c..be7035180 100644--- a/src/main/remote-ssh/ssh-project.ts+++ b/src/main/remote-ssh/ssh-project.ts@@ -56,6 +56,7 @@ import { RemoteHooks } from './remote-hooks'import {recordTunnelRepair,shouldAttemptTunnelRepair,+ shouldReportTunnelLost,type TunnelRepairState} from './tunnel-repair'@@ -454,6 +455,9 @@ export class SshProjectManager {* re-installed on every watchdog tick. See `tunnel-repair.ts` for why the first one is free. */private tunnelRepair = new Map<string, TunnelRepairState>()private lostHookTunnels = new Set<string>()+ /** Consecutive failed liveness probes per project — the banner waits for a second one+ * (`shouldReportTunnelLost`); a single slow probe is not a lost tunnel. */+ private tunnelProbeFailures = new Map<string, number>()private hookTunnelHealth(projectId: string, verified: boolean): void {if (verified) {@@ -491,25 +495,39 @@ export class SshProjectManager {const hook = this.r.getHook()// No hook server yet ⇒ nothing to point at, and `setup()` would refuse anyway.if (!hook?.port || !hook.token) return- const alive = await this.remoteHooks.tunnelAlive(projectId, existing.conn, existing.controlPath, hook.token)+ const probe = await this.remoteHooks.tunnelAlive(projectId, existing.conn, existing.controlPath, hook.token)if (this.conns.get(projectId) !== existing) return- if (alive) {+ if (probe.alive) {this.tunnelRepair.delete(projectId)+ this.tunnelProbeFailures.delete(projectId)this.hookTunnelHealth(projectId, true)return}- this.hookTunnelHealth(projectId, false)+ const failures = (this.tunnelProbeFailures.get(projectId) ?? 0) + 1+ this.tunnelProbeFailures.set(projectId, failures)+ // Logged on every failure, not only the reported ones: this line is the only field evidence+ // of WHY a tunnel stopped answering (dead listener vs slow master vs refused ssh).+ console.warn(`[ssh-project] hook tunnel probe failed for ${projectId} (#${failures}): ${probe.detail}`)const now = Date.now()- if (!shouldAttemptTunnelRepair(this.tunnelRepair.get(projectId), now)) return+ if (!shouldAttemptTunnelRepair(this.tunnelRepair.get(projectId), now)) {+ if (shouldReportTunnelLost(failures)) this.hookTunnelHealth(projectId, false)+ return+ }const res = await this.remoteHooks.setup(projectId, existing.conn, existing.controlPath, hook)this.tunnelRepair.set(projectId, recordTunnelRepair(this.tunnelRepair.get(projectId), !!res, now))- if (!res) return+ if (!res) {+ if (this.conns.get(projectId) === existing && shouldReportTunnelLost(failures)) {+ this.hookTunnelHealth(projectId, false)+ }+ return+ }// Ownership re-check: `setup()` is several round-trips, and a disconnect + reconnect inside// that window means this entry is no longer the live one — the same rule the establish path// applies before rebuilding a master. Writing the endpoint onto a superseded entry would// point sessions at a socket belonging to a connection nobody holds.if (this.conns.get(projectId) !== existing) returnexisting.hookEndpointPath = res.endpointPath+ this.tunnelProbeFailures.delete(projectId)this.hookTunnelHealth(projectId, true)// Same contract as the establish path: hook events lost while the tunnel was down are gone// for good, so the working agents need a resync. Fire-and-forget behind a catch — a repair@@ -897,6 +915,9 @@ export class SshProjectManager {continue}const hookEndpointPath = res?.endpointPath+ // A fresh establish starts a fresh probe streak: a failure before this reconnect and one+ // after it are not two opinions about the same tunnel.+ if (res) this.tunnelProbeFailures.delete(projectId)// Resolve the remote $HOME once and retain it (the hook setup above also learns it but// doesn't surface it). Phase 2b uses it to jail remote transcript reads. Fail-open: an// unresolved home just disables the remote context meter / subagent transcript / search.@@ -2433,6 +2454,7 @@ export class SshProjectManager {}// Cancel the reverse hook tunnel (over the still-live master) BEFORE tearing the master down.await this.remoteHooks.teardown(projectId, c.conn, c.controlPath)+ this.tunnelProbeFailures.delete(projectId)void this.r.run(exitMasterArgs(c.conn, c.controlPath))c.master.kill()this.conns.delete(projectId)diff --git a/src/main/remote-ssh/tunnel-repair.test.ts b/src/main/remote-ssh/tunnel-repair.test.tsindex c47db1bb6..4cdc6cf4f 100644--- a/src/main/remote-ssh/tunnel-repair.test.ts+++ b/src/main/remote-ssh/tunnel-repair.test.ts@@ -1,6 +1,8 @@import { describe, expect, it } from 'vitest'import {TUNNEL_REPAIR_DELAYS_MS,+ describeTunnelProbe,+ shouldReportTunnelLost,recordTunnelRepair,shouldAttemptTunnelRepair,tunnelRepairDelayMs@@ -76,3 +78,23 @@ describe('recordTunnelRepair', () => {expect(recordTunnelRepair(undefined, false, 4_242).lastAttemptAt).toBe(4_242)})})++describe('shouldReportTunnelLost', () => {+ it('waits for a SECOND consecutive failure — one slow probe is not a lost tunnel', () => {+ expect(shouldReportTunnelLost(0)).toBe(false)+ expect(shouldReportTunnelLost(1)).toBe(false)+ expect(shouldReportTunnelLost(2)).toBe(true)+ expect(shouldReportTunnelLost(5)).toBe(true)+ })+})++describe('describeTunnelProbe', () => {+ it('names the cause the exit code carries (ssh 255, curl 7 / 28) and keeps the raw values', () => {+ expect(describeTunnelProbe(255, '')).toBe('ssh failed (exit 255, http none)')+ expect(describeTunnelProbe(7, '000')).toBe('nothing listening on the socket (exit 7, http 000)')+ expect(describeTunnelProbe(28, '000\n')).toBe('curl timed out (exit 28, http 000)')+ expect(describeTunnelProbe(0, '421')).toBe('another hook server answered (bearer mismatch) (exit 0, http 421)')+ expect(describeTunnelProbe(0, '500')).toBe('unexpected HTTP answer (exit 0, http 500)')+ expect(describeTunnelProbe(1, '')).toBe('probe failed (exit 1, http none)')+ })+})diff --git a/src/main/remote-ssh/tunnel-repair.ts b/src/main/remote-ssh/tunnel-repair.tsindex c26b207fb..04e4beba1 100644--- a/src/main/remote-ssh/tunnel-repair.ts+++ b/src/main/remote-ssh/tunnel-repair.ts@@ -59,3 +59,44 @@ export function recordTunnelRepair(): TunnelRepairState {return { failures: ok ? 0 : (state?.failures ?? 0) + 1, lastAttemptAt: now }}++/**+ * How many CONSECUTIVE failed liveness probes it takes before the user is told the tunnel is lost.+ *+ * One was the old rule, and it made the banner a flapping alarm: the probe is a single `curl -m 5`+ * over the shared master, so one slow round trip (a busy mux, a queued child gate, a curl timeout)+ * put "lost their verified connection" on screen for a tunnel that was fine — field report from a+ * Linux desktop driving a Mac on the same desk, no sleep involved. The REPAIR still runs on the+ * first failure (see `shouldAttemptTunnelRepair`); only the warning waits for a second opinion,+ * one watchdog tick later. A repair that succeeds resets the streak, so a real-but-healed loss+ * never shows a banner at all.+ */+export const TUNNEL_LOST_STRIKES = 2++export function shouldReportTunnelLost(consecutiveFailures: number): boolean {+ return consecutiveFailures >= TUNNEL_LOST_STRIKES+}++/**+ * One line naming WHY a tunnel probe failed, for the log. The probe used to fold every failure into+ * `false`, so a field report could not tell a dead listener from a slow master from a refused ssh.+ * `code` is the ssh child's exit status: ssh's own failures are 255, otherwise it is the remote+ * `curl`'s (7 = connection refused → nobody listening on the socket, 28 = the 5 s timeout).+ * `stdout` is the `%{http_code}` curl printed (`000` = no HTTP answer; 421 = a different owner).+ */+export function describeTunnelProbe(code: number, stdout: string): string {+ const http = stdout.trim() || 'none'+ const hint =+ code === 255+ ? 'ssh failed'+ : code === 7+ ? 'nothing listening on the socket'+ : code === 28+ ? 'curl timed out'+ : code === 0 && http === '421'+ ? 'another hook server answered (bearer mismatch)'+ : code === 0+ ? 'unexpected HTTP answer'+ : 'probe failed'+ return `${hint} (exit ${code}, http ${http})`+}