Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(ssh): wake-from-sleep drops a half-dead ControlMaster instead of trusting -O check

reviewfd9bbe6480d3
diff --git a/src/core/remote-ssh/control-master.ts b/src/core/remote-ssh/control-master.ts
index 1a2f39895..becbf4a05 100644
--- a/src/core/remote-ssh/control-master.ts
+++ b/src/core/remote-ssh/control-master.ts
@@ -223,6 +223,32 @@ export function childArgs(conn: SshConnection, controlPath: string, remote?: str
export function checkMasterArgs(conn: SshConnection, controlPath: string): string[] {
return ['-O', 'check', '-o', `ControlPath=${controlPath}`, ...portArgs(conn), target(conn)]
}
+/**
+ * A real round trip over the master: open one session channel and run `true`. This is the
+ * liveness question `-O check` cannot answer. `-O check` is served by the master PROCESS over the
+ * local control socket, so it says "Master running" for a master whose TCP died under a sleep:
+ * measured against a black-holed connection, `-O check` exited 0 while a mux'd command hung until
+ * ServerAlive gave up ~74 s later. A channel open has to reach sshd and come back, so a
+ * half-dead master cannot answer it. The caller bounds it with a timeout; a TIMEOUT is the only
+ * verdict that means dead (see `SshProjectManager.revalidateAll`).
+ *
+ * `ControlMaster=no`: if the socket is gone this must not become a fresh master (which would
+ * answer on the new connection and say nothing about the old one). `BatchMode=yes`: if ssh falls
+ * back to a direct connection it must never park on a prompt with no one to answer it.
+ */
+export function masterRoundTripArgs(conn: SshConnection, controlPath: string): string[] {
+ return [
+ '-o',
+ 'ControlMaster=no',
+ '-o',
+ `ControlPath=${controlPath}`,
+ '-o',
+ 'BatchMode=yes',
+ ...portArgs(conn),
+ target(conn),
+ 'true'
+ ]
+}
export function exitMasterArgs(conn: SshConnection, controlPath: string): string[] {
return ['-O', 'exit', '-o', `ControlPath=${controlPath}`, ...portArgs(conn), target(conn)]
}
diff --git a/src/main/index.ts b/src/main/index.ts
index 6d503ed9c..91ff20c85 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -4215,11 +4215,12 @@ app.whenReady().then(async () => {
}, PREWARM_START_MS)
}
// Wake-from-sleep: re-validate every SSH master NOW instead of letting ServerAlive discover the
- // dead TCP ~60s later — until it does, every remote terminal looks alive and is dead (no echo,
- // no scroll). The small delay lets the network interface come back up first; connect() is
- // idempotent so a master that survived the nap is a cheap `-O check` no-op.
+ // dead TCP ~60-75s later — until it does, every remote terminal looks alive and is dead (no
+ // echo, no scroll). The small delay lets the network interface come back up first. `roundTrip`
+ // is the load-bearing part: `-O check` alone answers "Master running" for a master whose TCP
+ // died in the sleep (measured), so without a real round trip this pass was a no-op.
powerMonitor.on('resume', () => {
- setTimeout(() => void sshProjectManager?.revalidateAll(), 2000)
+ setTimeout(() => void sshProjectManager?.revalidateAll({ roundTrip: true }), 2000)
})
// While connected, poll each SSH project's server file: the mobile companion appends the
// sessions it starts to <remoteCwd>/.nodeterm/project.json, and this is how those nodes reach
diff --git a/src/main/remote-ssh/ssh-project.test.ts b/src/main/remote-ssh/ssh-project.test.ts
index 41cdc2070..3be02ea19 100644
--- a/src/main/remote-ssh/ssh-project.test.ts
+++ b/src/main/remote-ssh/ssh-project.test.ts
@@ -2635,6 +2635,95 @@ describe('master watchdog', () => {
})
})
+describe('wake-from-sleep round trip (revalidateAll({ roundTrip: true }))', () => {
+ afterEach(() => vi.restoreAllMocks())
+
+ // A master whose TCP died in a sleep: the PROCESS still serves `-O check` (exit 0) until it is
+ // told `-O exit`; a real round trip hangs. Measured against a black-holed sshd connection.
+ function makeHalfDeadMgr(probeVerdict: 'answered' | 'timeout' | 'throws' | 'absent') {
+ vi.spyOn(fs, 'mkdir').mockResolvedValue(undefined as never)
+ vi.spyOn(fs, 'stat').mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }))
+ vi.spyOn(fs, 'rm').mockResolvedValue(undefined)
+ const statuses: string[] = []
+ let exited = false
+ const spawnMaster = vi.fn(() => {
+ exited = false
+ return { kill: vi.fn(), on: vi.fn() }
+ })
+ const run = vi.fn(async (args: string[]) => {
+ if (args.includes('-O') && args.includes('exit')) {
+ exited = true
+ return { code: 0, stdout: '' }
+ }
+ if (args.includes('-O') && args.includes('check')) return { code: exited ? 255 : 0, stdout: '' }
+ return { code: 0, stdout: '' }
+ })
+ const probe = vi.fn(async (_args: string[], _timeoutMs: number) => {
+ if (probeVerdict === 'throws') throw new Error('spawn failed')
+ return probeVerdict as 'answered' | 'timeout'
+ })
+ const mgr = new SshProjectManager({
+ userDataDir: '/ud',
+ spawnMaster,
+ run,
+ ...(probeVerdict === 'absent' ? {} : { probe }),
+ runScp: vi.fn(async () => ({ code: 0 })),
+ getHook: () => ({ port: 1, token: 't', version: '1' }),
+ onStatus: (e) => statuses.push(e.status)
+ })
+ const exitCalls = () => run.mock.calls.filter(([a]) => a.includes('-O') && a.includes('exit')).length
+ return { mgr, statuses, spawnMaster, probe, exitCalls }
+ }
+
+ it('a master that times out the round trip is ended and re-established', async () => {
+ const { mgr, statuses, spawnMaster, probe, exitCalls } = makeHalfDeadMgr('timeout')
+ await mgr.connect('p1', conn)
+ await mgr.revalidateAll({ roundTrip: true })
+ const [args, timeoutMs] = probe.mock.calls[0]
+ expect(args).toEqual(expect.arrayContaining(['ControlMaster=no', 'BatchMode=yes', 'true']))
+ expect(args).toContain(`ControlPath=${controlPathFor('p1')}`)
+ expect(timeoutMs).toBeGreaterThan(0)
+ // Ours, then connect()'s own dead-branch `-O exit` (its no-op-if-already-gone belt).
+ expect(exitCalls()).toBeGreaterThanOrEqual(1)
+ expect(spawnMaster).toHaveBeenCalledTimes(2)
+ expect(statuses).toContain('reconnecting')
+ expect(statuses.at(-1)).toBe('connected')
+ })
+
+ it('a master that answers is left alone', async () => {
+ const { mgr, statuses, spawnMaster, exitCalls } = makeHalfDeadMgr('answered')
+ await mgr.connect('p1', conn)
+ await mgr.revalidateAll({ roundTrip: true })
+ expect(exitCalls()).toBe(0)
+ expect(spawnMaster).toHaveBeenCalledTimes(1)
+ expect(statuses).not.toContain('reconnecting')
+ })
+
+ it('a probe that cannot run convicts nothing', async () => {
+ const { mgr, spawnMaster, exitCalls } = makeHalfDeadMgr('throws')
+ await mgr.connect('p1', conn)
+ await mgr.revalidateAll({ roundTrip: true })
+ expect(exitCalls()).toBe(0)
+ expect(spawnMaster).toHaveBeenCalledTimes(1)
+ })
+
+ it('the watchdog pass (no roundTrip) never probes — its cost stays one `-O check`', async () => {
+ const { mgr, probe, exitCalls } = makeHalfDeadMgr('timeout')
+ await mgr.connect('p1', conn)
+ await mgr.revalidateAll()
+ expect(probe).not.toHaveBeenCalled()
+ expect(exitCalls()).toBe(0)
+ })
+
+ it('without a probe runner the wake pass is the old `-O check` revalidate', async () => {
+ const { mgr, spawnMaster, exitCalls } = makeHalfDeadMgr('absent')
+ await mgr.connect('p1', conn)
+ await mgr.revalidateAll({ roundTrip: true })
+ expect(exitCalls()).toBe(0)
+ expect(spawnMaster).toHaveBeenCalledTimes(1)
+ })
+})
+
describe('lastSshErrorLine', () => {
it('picks the actionable last line, skipping debug noise', () => {
const stderr = [
diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.ts
index 86662005c..f8a943ad2 100644
--- a/src/main/remote-ssh/ssh-project.ts
+++ b/src/main/remote-ssh/ssh-project.ts
@@ -43,6 +43,7 @@ import {
mkDirArgs,
exitMasterArgs,
checkMasterArgs,
+ masterRoundTripArgs,
remoteTmuxKillArgs,
remoteTmuxKillEverySocketArgs,
childArgs,
@@ -103,6 +104,12 @@ interface Runners {
}
/** Run a one-shot ssh, resolving its stdout + exit code; optional stdin written to the child. */
run: (args: string[], stdin?: string) => Promise<{ code: number; stdout: string }>
+ /** Run a one-shot ssh with a hard timeout, OUTSIDE the per-master child gate, and say only
+ * whether it finished in time. Used by the wake-from-sleep liveness probe: after a sleep the
+ * gate is often full of children hung on the dead master, and a probe queued behind them would
+ * measure the queue, not the master. Optional: without it the wake pass falls back to the
+ * `-O check`-only revalidate it did before. */
+ probe?: (args: string[], timeoutMs: number) => Promise<'answered' | 'timeout'>
/** Run a one-shot scp (file upload over the master); resolves its exit code. */
runScp: (args: string[]) => Promise<{ code: number }>
/** Live loopback hook-server coordinates (injected so the manager stays testable). */
@@ -218,6 +225,12 @@ const MASTER_STDERR_CAP = 8 * 1024
* long exec polls can churn direct-fallback connections after an unnoticed master death. */
const MASTER_WATCHDOG_MS = 45_000
+/** Budget for the wake-from-sleep round trip (`dropMasterIfHalfDead`). A healthy master answers a
+ * mux'd `true` in a couple of RTTs plus the remote shell's startup; a half-dead one never
+ * answers. Generous on purpose: a false conviction costs every terminal of the project a
+ * reconnect, a late one only a few more seconds of frozen screen — against ~74 s without it. */
+const WAKE_ROUND_TRIP_TIMEOUT_MS = 10_000
+
/** How long connect() waits for `-O check` to answer when nothing is blocking on a human.
*
* These are WALL-CLOCK budgets, not attempt counts, and that distinction is load bearing: each
@@ -1658,7 +1671,7 @@ export class SshProjectManager {
* 'connected' flush respawns the dead nodes. Failures just leave the normal status-event
* error path in charge (connect reports it before throwing).
*/
- async revalidateAll(): Promise<void> {
+ async revalidateAll(opts?: { roundTrip?: boolean }): Promise<void> {
// Per project CONCURRENTLY, not serially: a re-establish can park on the askpass passphrase
// prompt for up to PROMPT_WAIT_MS (5 min), and a serial pass wedged EVERY other project's
// check behind it for that whole window - the direct-fallback churn the watchdog exists to
@@ -1673,6 +1686,7 @@ export class SshProjectManager {
// established for the NEW endpoint, silently reverting the project to the old host.
const e = this.conns.get(projectId)
if (!e) return // disconnected while the pass was being set up
+ if (opts?.roundTrip) await this.dropMasterIfHalfDead(projectId, e)
try {
await this.connect(projectId, e.conn, e.remoteCwd)
} catch {
@@ -1682,6 +1696,42 @@ export class SshProjectManager {
)
}
+ /**
+ * Wake-from-sleep only: end a master that answers `-O check` but can no longer reach sshd.
+ *
+ * The reuse branch of `connect()` trusts `-O check`, and after a sleep that is exactly the wrong
+ * witness — the master PROCESS is alive locally while its TCP is gone (measured: `-O check` exit
+ * 0 on a black-holed master, a mux'd command hanging until ServerAlive gave up ~74 s later). So
+ * the wake pass was a no-op, and for that minute+ every remote terminal kept its last screen
+ * while keys and the wheel went nowhere. Here a real round trip is timed instead; on a TIMEOUT
+ * the master is sent `-O exit`, which the process serves locally and at once (measured: 5 ms,
+ * every mux'd terminal client exiting 255 in the same instant). Those 255s feed the renderer's
+ * SshReconnector, and the `connect()` that follows in `revalidateAll` finds the check failing
+ * and re-establishes, so the terminals respawn onto a fresh master seconds after wake.
+ *
+ * Only a timeout convicts. A probe that finished — with any exit code — reached a verdict
+ * without hanging, so the master is not the half-dead kind this exists for; a fast failure
+ * (socket already gone, master already dead) is `connect()`'s normal job. Skipped while a
+ * connect is in flight for the project: that attempt owns the master and may be mid-handshake.
+ */
+ private async dropMasterIfHalfDead(projectId: string, e: Conn): Promise<void> {
+ if (!this.r.probe || this.inFlight.has(projectId)) return
+ let verdict: 'answered' | 'timeout'
+ try {
+ verdict = await this.r.probe(masterRoundTripArgs(e.conn, e.controlPath), WAKE_ROUND_TRIP_TIMEOUT_MS)
+ } catch {
+ return // a probe that could not run proves nothing either way
+ }
+ if (verdict !== 'timeout') return
+ // Re-check ownership: a disconnect or endpoint change during the probe means this entry's
+ // master is no longer ours to end.
+ if (this.conns.get(projectId) !== e || this.inFlight.has(projectId)) return
+ console.warn(
+ `[ssh-project] master for ${sshHostKey(e.conn)} did not answer a round trip after wake; ending it so terminals reconnect`
+ )
+ await this.r.run(exitMasterArgs(e.conn, e.controlPath)).catch(() => {})
+ }
+
/** The connection's cached remote `--permission-mode auto` capability (undefined = not
* probed / not connected). Feeds the agent-status settings block the phone reads. */
remoteAutoPermFor(projectId: string): boolean | undefined {
@@ -2706,6 +2756,17 @@ export function initSshProject(
}
})
),
+ // Deliberately NOT through `sshChildGate`: after a sleep the gate is typically full of children
+ // hung on the very master this probes, and queueing behind them would time the queue.
+ probe: (args, timeoutMs) =>
+ new Promise((resolve) => {
+ execFile(
+ ssh,
+ args,
+ { timeout: timeoutMs, env: { ...process.env, ...appSshAgent.env() } },
+ (err) => resolve(err && (err as { killed?: boolean }).killed ? 'timeout' : 'answered')
+ )
+ }),
runScp: (args) =>
new Promise((resolve) => {
// Same reason as `run`: scp re-authenticates when the master socket is gone.