Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(ssh): don't flash "lost verified connection" on one slow tunnel probe

review6033366ec783
diff --git a/src/main/remote-ssh/remote-hooks.ts b/src/main/remote-ssh/remote-hooks.ts
index cb5f0d335..95c858907 100644
--- a/src/main/remote-ssh/remote-hooks.ts
+++ b/src/main/remote-ssh/remote-hooks.ts
@@ -54,6 +54,7 @@ import {
mergeInstructionsBlock
} from '../../core/context-link-core'
import { posixQuote, type SshConnection } from '../../shared/ssh'
+import { describeTunnelProbe } from './tunnel-repair'
/** POSIX dirname of an absolute remote path. `path.dirname` would apply the LOCAL separator
* rules, which is wrong the moment the desktop is Windows and the host is Linux. */
@@ -128,10 +129,16 @@ export class RemoteHooks {
* run, which is a tunnel that cannot deliver, not an unknown. The caller repairs by re-running
* `setup()`, which is idempotent and re-verifies end-to-end.
*/
- async tunnelAlive(projectId: string, conn: SshConnection, controlPath: string, token: string): Promise<boolean> {
+ async tunnelAlive(
+ projectId: string,
+ conn: SshConnection,
+ controlPath: string,
+ token: string
+ ): Promise<{ alive: boolean; detail: string }> {
const spec = this.specs.get(projectId)
- if (!spec || !token) return false
- return this.verifyTunnel(conn, controlPath, spec.sock, token)
+ if (!spec) return { alive: false, detail: 'no forward registered for this project in this app run' }
+ if (!token) return { alive: false, detail: 'no hook token' }
+ return this.probeTunnel(conn, controlPath, spec.sock, token)
}
async setup(
@@ -928,6 +935,16 @@ export class RemoteHooks {
sock: string,
token: string
): Promise<boolean> {
+ return (await this.probeTunnel(conn, controlPath, sock, token)).alive
+ }
+
+ /** `verifyTunnel` with the reason kept: the watchdog logs WHY a live tunnel stopped answering. */
+ private async probeTunnel(
+ conn: SshConnection,
+ controlPath: string,
+ sock: string,
+ token: string
+ ): Promise<{ alive: boolean; detail: string }> {
try {
const cmd =
`curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST --unix-socket ${posixQuote(sock)} ` +
@@ -946,9 +963,10 @@ export class RemoteHooks {
childArgs(conn, controlPath, cmd),
curlHeaderConfigLine('x-nodeterm-hook-token', token)
)
- return r.code === 0 && r.stdout.trim() === '204'
- } catch {
- return false
+ const alive = r.code === 0 && r.stdout.trim() === '204'
+ return { alive, detail: alive ? 'ok' : describeTunnelProbe(r.code, r.stdout) }
+ } catch (e) {
+ return { alive: false, detail: `probe threw: ${e instanceof Error ? e.message : String(e)}` }
}
}
diff --git a/src/main/remote-ssh/ssh-project.test.ts b/src/main/remote-ssh/ssh-project.test.ts
index 41cdc2070..d78f25b83 100644
--- a/src/main/remote-ssh/ssh-project.test.ts
+++ b/src/main/remote-ssh/ssh-project.test.ts
@@ -1482,6 +1482,41 @@ describe('SshProjectManager', () => {
expect(onStatus).toHaveBeenCalledWith({ projectId: 'p1', status: 'connected', hookTunnelVerified: true })
})
+ it('ONE failed probe never raises the banner — neither when the repair heals it nor when the next probe does', async () => {
+ // Field report (Linux desktop → Mac on the same desk, no sleep): "Agent status and canvas
+ // control lost their verified connection" kept appearing. The probe is one `curl -m 5` over
+ // the shared master, and a single slow round trip used to put the banner up by itself.
+ let dead = 0
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const mgr = makeVerifiedMgr(vi.fn(), () => (dead-- > 0 ? '000' : '204'))
+ const onStatus = (mgr as unknown as { r: { onStatus: ReturnType<typeof vi.fn> } }).r.onStatus
+ await mgr.connect('p1', conn, '/remote/cwd')
+ // (a) probe fails once, the repair's own verify succeeds → healed silently.
+ dead = 1
+ await mgr.connect('p1', conn, '/remote/cwd')
+ // (b) probe AND repair fail once, the next tick's probe answers → still silent.
+ dead = 3 // the probe + both of setup()'s verify attempts
+ await mgr.connect('p1', conn, '/remote/cwd')
+ await mgr.connect('p1', conn, '/remote/cwd')
+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified !== undefined)).toEqual([])
+ // …but every failure is logged with its cause, the only field evidence there is.
+ expect(warn.mock.calls.map((c) => String(c[0])).filter((l) => l.includes('hook tunnel probe failed'))).toEqual([
+ '[ssh-project] hook tunnel probe failed for p1 (#1): unexpected HTTP answer (exit 0, http 000)',
+ '[ssh-project] hook tunnel probe failed for p1 (#1): unexpected HTTP answer (exit 0, http 000)'
+ ])
+ })
+
+ it('TWO consecutive failed probes raise the banner, even while the repair is backing off', async () => {
+ const mgr = makeVerifiedMgr(vi.fn(), () => '000')
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const onStatus = (mgr as unknown as { r: { onStatus: ReturnType<typeof vi.fn> } }).r.onStatus
+ await mgr.connect('p1', conn, '/remote/cwd') // establish fails verification → no spec
+ await mgr.connect('p1', conn, '/remote/cwd') // failure #1 → repair attempted, fails
+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified === false)).toHaveLength(0)
+ await mgr.connect('p1', conn, '/remote/cwd') // failure #2 → inside backoff, still reported
+ expect(onStatus.mock.calls.filter(([e]) => e.hookTunnelVerified === false)).toHaveLength(1)
+ })
+
it('rebinds the forward on repair — the endpoint is re-advertised, not merely re-probed', async () => {
// The whole failure is a master with no `-R`, so a repair that did not call `-O forward`
// would leave every hook POST dying exactly as before while reporting success.
diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.ts
index 86662005c..be7035180 100644
--- a/src/main/remote-ssh/ssh-project.ts
+++ b/src/main/remote-ssh/ssh-project.ts
@@ -56,6 +56,7 @@ import { RemoteHooks } from './remote-hooks'
import {
recordTunnelRepair,
shouldAttemptTunnelRepair,
+ shouldReportTunnelLost,
type TunnelRepairState
} from './tunnel-repair'
@@ -454,6 +455,9 @@ export class SshProjectManager {
* re-installed on every watchdog tick. See `tunnel-repair.ts` for why the first one is free. */
private tunnelRepair = new Map<string, TunnelRepairState>()
private lostHookTunnels = new Set<string>()
+ /** Consecutive failed liveness probes per project — the banner waits for a second one
+ * (`shouldReportTunnelLost`); a single slow probe is not a lost tunnel. */
+ private tunnelProbeFailures = new Map<string, number>()
private hookTunnelHealth(projectId: string, verified: boolean): void {
if (verified) {
@@ -491,25 +495,39 @@ export class SshProjectManager {
const hook = this.r.getHook()
// No hook server yet ⇒ nothing to point at, and `setup()` would refuse anyway.
if (!hook?.port || !hook.token) return
- const alive = await this.remoteHooks.tunnelAlive(projectId, existing.conn, existing.controlPath, hook.token)
+ const probe = await this.remoteHooks.tunnelAlive(projectId, existing.conn, existing.controlPath, hook.token)
if (this.conns.get(projectId) !== existing) return
- if (alive) {
+ if (probe.alive) {
this.tunnelRepair.delete(projectId)
+ this.tunnelProbeFailures.delete(projectId)
this.hookTunnelHealth(projectId, true)
return
}
- this.hookTunnelHealth(projectId, false)
+ const failures = (this.tunnelProbeFailures.get(projectId) ?? 0) + 1
+ this.tunnelProbeFailures.set(projectId, failures)
+ // Logged on every failure, not only the reported ones: this line is the only field evidence
+ // of WHY a tunnel stopped answering (dead listener vs slow master vs refused ssh).
+ console.warn(`[ssh-project] hook tunnel probe failed for ${projectId} (#${failures}): ${probe.detail}`)
const now = Date.now()
- if (!shouldAttemptTunnelRepair(this.tunnelRepair.get(projectId), now)) return
+ if (!shouldAttemptTunnelRepair(this.tunnelRepair.get(projectId), now)) {
+ if (shouldReportTunnelLost(failures)) this.hookTunnelHealth(projectId, false)
+ return
+ }
const res = await this.remoteHooks.setup(projectId, existing.conn, existing.controlPath, hook)
this.tunnelRepair.set(projectId, recordTunnelRepair(this.tunnelRepair.get(projectId), !!res, now))
- if (!res) return
+ if (!res) {
+ if (this.conns.get(projectId) === existing && shouldReportTunnelLost(failures)) {
+ this.hookTunnelHealth(projectId, false)
+ }
+ return
+ }
// Ownership re-check: `setup()` is several round-trips, and a disconnect + reconnect inside
// that window means this entry is no longer the live one — the same rule the establish path
// applies before rebuilding a master. Writing the endpoint onto a superseded entry would
// point sessions at a socket belonging to a connection nobody holds.
if (this.conns.get(projectId) !== existing) return
existing.hookEndpointPath = res.endpointPath
+ this.tunnelProbeFailures.delete(projectId)
this.hookTunnelHealth(projectId, true)
// Same contract as the establish path: hook events lost while the tunnel was down are gone
// for good, so the working agents need a resync. Fire-and-forget behind a catch — a repair
@@ -897,6 +915,9 @@ export class SshProjectManager {
continue
}
const hookEndpointPath = res?.endpointPath
+ // A fresh establish starts a fresh probe streak: a failure before this reconnect and one
+ // after it are not two opinions about the same tunnel.
+ if (res) this.tunnelProbeFailures.delete(projectId)
// Resolve the remote $HOME once and retain it (the hook setup above also learns it but
// doesn't surface it). Phase 2b uses it to jail remote transcript reads. Fail-open: an
// unresolved home just disables the remote context meter / subagent transcript / search.
@@ -2433,6 +2454,7 @@ export class SshProjectManager {
}
// Cancel the reverse hook tunnel (over the still-live master) BEFORE tearing the master down.
await this.remoteHooks.teardown(projectId, c.conn, c.controlPath)
+ this.tunnelProbeFailures.delete(projectId)
void this.r.run(exitMasterArgs(c.conn, c.controlPath))
c.master.kill()
this.conns.delete(projectId)
diff --git a/src/main/remote-ssh/tunnel-repair.test.ts b/src/main/remote-ssh/tunnel-repair.test.ts
index c47db1bb6..4cdc6cf4f 100644
--- a/src/main/remote-ssh/tunnel-repair.test.ts
+++ b/src/main/remote-ssh/tunnel-repair.test.ts
@@ -1,6 +1,8 @@
import { describe, expect, it } from 'vitest'
import {
TUNNEL_REPAIR_DELAYS_MS,
+ describeTunnelProbe,
+ shouldReportTunnelLost,
recordTunnelRepair,
shouldAttemptTunnelRepair,
tunnelRepairDelayMs
@@ -76,3 +78,23 @@ describe('recordTunnelRepair', () => {
expect(recordTunnelRepair(undefined, false, 4_242).lastAttemptAt).toBe(4_242)
})
})
+
+describe('shouldReportTunnelLost', () => {
+ it('waits for a SECOND consecutive failure — one slow probe is not a lost tunnel', () => {
+ expect(shouldReportTunnelLost(0)).toBe(false)
+ expect(shouldReportTunnelLost(1)).toBe(false)
+ expect(shouldReportTunnelLost(2)).toBe(true)
+ expect(shouldReportTunnelLost(5)).toBe(true)
+ })
+})
+
+describe('describeTunnelProbe', () => {
+ it('names the cause the exit code carries (ssh 255, curl 7 / 28) and keeps the raw values', () => {
+ expect(describeTunnelProbe(255, '')).toBe('ssh failed (exit 255, http none)')
+ expect(describeTunnelProbe(7, '000')).toBe('nothing listening on the socket (exit 7, http 000)')
+ expect(describeTunnelProbe(28, '000\n')).toBe('curl timed out (exit 28, http 000)')
+ expect(describeTunnelProbe(0, '421')).toBe('another hook server answered (bearer mismatch) (exit 0, http 421)')
+ expect(describeTunnelProbe(0, '500')).toBe('unexpected HTTP answer (exit 0, http 500)')
+ expect(describeTunnelProbe(1, '')).toBe('probe failed (exit 1, http none)')
+ })
+})
diff --git a/src/main/remote-ssh/tunnel-repair.ts b/src/main/remote-ssh/tunnel-repair.ts
index c26b207fb..04e4beba1 100644
--- a/src/main/remote-ssh/tunnel-repair.ts
+++ b/src/main/remote-ssh/tunnel-repair.ts
@@ -59,3 +59,44 @@ export function recordTunnelRepair(
): TunnelRepairState {
return { failures: ok ? 0 : (state?.failures ?? 0) + 1, lastAttemptAt: now }
}
+
+/**
+ * How many CONSECUTIVE failed liveness probes it takes before the user is told the tunnel is lost.
+ *
+ * One was the old rule, and it made the banner a flapping alarm: the probe is a single `curl -m 5`
+ * over the shared master, so one slow round trip (a busy mux, a queued child gate, a curl timeout)
+ * put "lost their verified connection" on screen for a tunnel that was fine — field report from a
+ * Linux desktop driving a Mac on the same desk, no sleep involved. The REPAIR still runs on the
+ * first failure (see `shouldAttemptTunnelRepair`); only the warning waits for a second opinion,
+ * one watchdog tick later. A repair that succeeds resets the streak, so a real-but-healed loss
+ * never shows a banner at all.
+ */
+export const TUNNEL_LOST_STRIKES = 2
+
+export function shouldReportTunnelLost(consecutiveFailures: number): boolean {
+ return consecutiveFailures >= TUNNEL_LOST_STRIKES
+}
+
+/**
+ * One line naming WHY a tunnel probe failed, for the log. The probe used to fold every failure into
+ * `false`, so a field report could not tell a dead listener from a slow master from a refused ssh.
+ * `code` is the ssh child's exit status: ssh's own failures are 255, otherwise it is the remote
+ * `curl`'s (7 = connection refused → nobody listening on the socket, 28 = the 5 s timeout).
+ * `stdout` is the `%{http_code}` curl printed (`000` = no HTTP answer; 421 = a different owner).
+ */
+export function describeTunnelProbe(code: number, stdout: string): string {
+ const http = stdout.trim() || 'none'
+ const hint =
+ code === 255
+ ? 'ssh failed'
+ : code === 7
+ ? 'nothing listening on the socket'
+ : code === 28
+ ? 'curl timed out'
+ : code === 0 && http === '421'
+ ? 'another hook server answered (bearer mismatch)'
+ : code === 0
+ ? 'unexpected HTTP answer'
+ : 'probe failed'
+ return `${hint} (exit ${code}, http ${http})`
+}