Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.
TypeScript adhdagent-orchestrationai-agentscanvasclaude-codecodexcoding-agentsdeveloper-toolselectrongeminilinuxmacosmultiplexeropencodeparallel-agentsterminalterminal-emulatorterminal-multiplexertmuxworkspace-manager
fix(ssh): wake-from-sleep drops a half-dead ControlMaster instead of trusting -O check
reviewfd9bbe6480d3
diff --git a/src/core/remote-ssh/control-master.ts b/src/core/remote-ssh/control-master.tsindex 1a2f39895..becbf4a05 100644--- a/src/core/remote-ssh/control-master.ts+++ b/src/core/remote-ssh/control-master.ts@@ -223,6 +223,32 @@ export function childArgs(conn: SshConnection, controlPath: string, remote?: strexport function checkMasterArgs(conn: SshConnection, controlPath: string): string[] {return ['-O', 'check', '-o', `ControlPath=${controlPath}`, ...portArgs(conn), target(conn)]}+/**+ * A real round trip over the master: open one session channel and run `true`. This is the+ * liveness question `-O check` cannot answer. `-O check` is served by the master PROCESS over the+ * local control socket, so it says "Master running" for a master whose TCP died under a sleep:+ * measured against a black-holed connection, `-O check` exited 0 while a mux'd command hung until+ * ServerAlive gave up ~74 s later. A channel open has to reach sshd and come back, so a+ * half-dead master cannot answer it. The caller bounds it with a timeout; a TIMEOUT is the only+ * verdict that means dead (see `SshProjectManager.revalidateAll`).+ *+ * `ControlMaster=no`: if the socket is gone this must not become a fresh master (which would+ * answer on the new connection and say nothing about the old one). `BatchMode=yes`: if ssh falls+ * back to a direct connection it must never park on a prompt with no one to answer it.+ */+export function masterRoundTripArgs(conn: SshConnection, controlPath: string): string[] {+ return [+ '-o',+ 'ControlMaster=no',+ '-o',+ `ControlPath=${controlPath}`,+ '-o',+ 'BatchMode=yes',+ ...portArgs(conn),+ target(conn),+ 'true'+ ]+}export function exitMasterArgs(conn: SshConnection, controlPath: string): string[] {return ['-O', 'exit', '-o', `ControlPath=${controlPath}`, ...portArgs(conn), target(conn)]}diff --git a/src/main/index.ts b/src/main/index.tsindex 6d503ed9c..91ff20c85 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -4215,11 +4215,12 @@ app.whenReady().then(async () => {}, PREWARM_START_MS)}// Wake-from-sleep: re-validate every SSH master NOW instead of letting ServerAlive discover the- // dead TCP ~60s later — until it does, every remote terminal looks alive and is dead (no echo,- // no scroll). The small delay lets the network interface come back up first; connect() is- // idempotent so a master that survived the nap is a cheap `-O check` no-op.+ // dead TCP ~60-75s later — until it does, every remote terminal looks alive and is dead (no+ // echo, no scroll). The small delay lets the network interface come back up first. `roundTrip`+ // is the load-bearing part: `-O check` alone answers "Master running" for a master whose TCP+ // died in the sleep (measured), so without a real round trip this pass was a no-op.powerMonitor.on('resume', () => {- setTimeout(() => void sshProjectManager?.revalidateAll(), 2000)+ setTimeout(() => void sshProjectManager?.revalidateAll({ roundTrip: true }), 2000)})// While connected, poll each SSH project's server file: the mobile companion appends the// sessions it starts to <remoteCwd>/.nodeterm/project.json, and this is how those nodes reachdiff --git a/src/main/remote-ssh/ssh-project.test.ts b/src/main/remote-ssh/ssh-project.test.tsindex 41cdc2070..3be02ea19 100644--- a/src/main/remote-ssh/ssh-project.test.ts+++ b/src/main/remote-ssh/ssh-project.test.ts@@ -2635,6 +2635,95 @@ describe('master watchdog', () => {})})+describe('wake-from-sleep round trip (revalidateAll({ roundTrip: true }))', () => {+ afterEach(() => vi.restoreAllMocks())++ // A master whose TCP died in a sleep: the PROCESS still serves `-O check` (exit 0) until it is+ // told `-O exit`; a real round trip hangs. Measured against a black-holed sshd connection.+ function makeHalfDeadMgr(probeVerdict: 'answered' | 'timeout' | 'throws' | 'absent') {+ vi.spyOn(fs, 'mkdir').mockResolvedValue(undefined as never)+ vi.spyOn(fs, 'stat').mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }))+ vi.spyOn(fs, 'rm').mockResolvedValue(undefined)+ const statuses: string[] = []+ let exited = false+ const spawnMaster = vi.fn(() => {+ exited = false+ return { kill: vi.fn(), on: vi.fn() }+ })+ const run = vi.fn(async (args: string[]) => {+ if (args.includes('-O') && args.includes('exit')) {+ exited = true+ return { code: 0, stdout: '' }+ }+ if (args.includes('-O') && args.includes('check')) return { code: exited ? 255 : 0, stdout: '' }+ return { code: 0, stdout: '' }+ })+ const probe = vi.fn(async (_args: string[], _timeoutMs: number) => {+ if (probeVerdict === 'throws') throw new Error('spawn failed')+ return probeVerdict as 'answered' | 'timeout'+ })+ const mgr = new SshProjectManager({+ userDataDir: '/ud',+ spawnMaster,+ run,+ ...(probeVerdict === 'absent' ? {} : { probe }),+ runScp: vi.fn(async () => ({ code: 0 })),+ getHook: () => ({ port: 1, token: 't', version: '1' }),+ onStatus: (e) => statuses.push(e.status)+ })+ const exitCalls = () => run.mock.calls.filter(([a]) => a.includes('-O') && a.includes('exit')).length+ return { mgr, statuses, spawnMaster, probe, exitCalls }+ }++ it('a master that times out the round trip is ended and re-established', async () => {+ const { mgr, statuses, spawnMaster, probe, exitCalls } = makeHalfDeadMgr('timeout')+ await mgr.connect('p1', conn)+ await mgr.revalidateAll({ roundTrip: true })+ const [args, timeoutMs] = probe.mock.calls[0]+ expect(args).toEqual(expect.arrayContaining(['ControlMaster=no', 'BatchMode=yes', 'true']))+ expect(args).toContain(`ControlPath=${controlPathFor('p1')}`)+ expect(timeoutMs).toBeGreaterThan(0)+ // Ours, then connect()'s own dead-branch `-O exit` (its no-op-if-already-gone belt).+ expect(exitCalls()).toBeGreaterThanOrEqual(1)+ expect(spawnMaster).toHaveBeenCalledTimes(2)+ expect(statuses).toContain('reconnecting')+ expect(statuses.at(-1)).toBe('connected')+ })++ it('a master that answers is left alone', async () => {+ const { mgr, statuses, spawnMaster, exitCalls } = makeHalfDeadMgr('answered')+ await mgr.connect('p1', conn)+ await mgr.revalidateAll({ roundTrip: true })+ expect(exitCalls()).toBe(0)+ expect(spawnMaster).toHaveBeenCalledTimes(1)+ expect(statuses).not.toContain('reconnecting')+ })++ it('a probe that cannot run convicts nothing', async () => {+ const { mgr, spawnMaster, exitCalls } = makeHalfDeadMgr('throws')+ await mgr.connect('p1', conn)+ await mgr.revalidateAll({ roundTrip: true })+ expect(exitCalls()).toBe(0)+ expect(spawnMaster).toHaveBeenCalledTimes(1)+ })++ it('the watchdog pass (no roundTrip) never probes — its cost stays one `-O check`', async () => {+ const { mgr, probe, exitCalls } = makeHalfDeadMgr('timeout')+ await mgr.connect('p1', conn)+ await mgr.revalidateAll()+ expect(probe).not.toHaveBeenCalled()+ expect(exitCalls()).toBe(0)+ })++ it('without a probe runner the wake pass is the old `-O check` revalidate', async () => {+ const { mgr, spawnMaster, exitCalls } = makeHalfDeadMgr('absent')+ await mgr.connect('p1', conn)+ await mgr.revalidateAll({ roundTrip: true })+ expect(exitCalls()).toBe(0)+ expect(spawnMaster).toHaveBeenCalledTimes(1)+ })+})+describe('lastSshErrorLine', () => {it('picks the actionable last line, skipping debug noise', () => {const stderr = [diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.tsindex 86662005c..f8a943ad2 100644--- a/src/main/remote-ssh/ssh-project.ts+++ b/src/main/remote-ssh/ssh-project.ts@@ -43,6 +43,7 @@ import {mkDirArgs,exitMasterArgs,checkMasterArgs,+ masterRoundTripArgs,remoteTmuxKillArgs,remoteTmuxKillEverySocketArgs,childArgs,@@ -103,6 +104,12 @@ interface Runners {}/** Run a one-shot ssh, resolving its stdout + exit code; optional stdin written to the child. */run: (args: string[], stdin?: string) => Promise<{ code: number; stdout: string }>+ /** Run a one-shot ssh with a hard timeout, OUTSIDE the per-master child gate, and say only+ * whether it finished in time. Used by the wake-from-sleep liveness probe: after a sleep the+ * gate is often full of children hung on the dead master, and a probe queued behind them would+ * measure the queue, not the master. Optional: without it the wake pass falls back to the+ * `-O check`-only revalidate it did before. */+ probe?: (args: string[], timeoutMs: number) => Promise<'answered' | 'timeout'>/** Run a one-shot scp (file upload over the master); resolves its exit code. */runScp: (args: string[]) => Promise<{ code: number }>/** Live loopback hook-server coordinates (injected so the manager stays testable). */@@ -218,6 +225,12 @@ const MASTER_STDERR_CAP = 8 * 1024* long exec polls can churn direct-fallback connections after an unnoticed master death. */const MASTER_WATCHDOG_MS = 45_000+/** Budget for the wake-from-sleep round trip (`dropMasterIfHalfDead`). A healthy master answers a+ * mux'd `true` in a couple of RTTs plus the remote shell's startup; a half-dead one never+ * answers. Generous on purpose: a false conviction costs every terminal of the project a+ * reconnect, a late one only a few more seconds of frozen screen — against ~74 s without it. */+const WAKE_ROUND_TRIP_TIMEOUT_MS = 10_000+/** How long connect() waits for `-O check` to answer when nothing is blocking on a human.** These are WALL-CLOCK budgets, not attempt counts, and that distinction is load bearing: each@@ -1658,7 +1671,7 @@ export class SshProjectManager {* 'connected' flush respawns the dead nodes. Failures just leave the normal status-event* error path in charge (connect reports it before throwing).*/- async revalidateAll(): Promise<void> {+ async revalidateAll(opts?: { roundTrip?: boolean }): Promise<void> {// Per project CONCURRENTLY, not serially: a re-establish can park on the askpass passphrase// prompt for up to PROMPT_WAIT_MS (5 min), and a serial pass wedged EVERY other project's// check behind it for that whole window - the direct-fallback churn the watchdog exists to@@ -1673,6 +1686,7 @@ export class SshProjectManager {// established for the NEW endpoint, silently reverting the project to the old host.const e = this.conns.get(projectId)if (!e) return // disconnected while the pass was being set up+ if (opts?.roundTrip) await this.dropMasterIfHalfDead(projectId, e)try {await this.connect(projectId, e.conn, e.remoteCwd)} catch {@@ -1682,6 +1696,42 @@ export class SshProjectManager {)}+ /**+ * Wake-from-sleep only: end a master that answers `-O check` but can no longer reach sshd.+ *+ * The reuse branch of `connect()` trusts `-O check`, and after a sleep that is exactly the wrong+ * witness — the master PROCESS is alive locally while its TCP is gone (measured: `-O check` exit+ * 0 on a black-holed master, a mux'd command hanging until ServerAlive gave up ~74 s later). So+ * the wake pass was a no-op, and for that minute+ every remote terminal kept its last screen+ * while keys and the wheel went nowhere. Here a real round trip is timed instead; on a TIMEOUT+ * the master is sent `-O exit`, which the process serves locally and at once (measured: 5 ms,+ * every mux'd terminal client exiting 255 in the same instant). Those 255s feed the renderer's+ * SshReconnector, and the `connect()` that follows in `revalidateAll` finds the check failing+ * and re-establishes, so the terminals respawn onto a fresh master seconds after wake.+ *+ * Only a timeout convicts. A probe that finished — with any exit code — reached a verdict+ * without hanging, so the master is not the half-dead kind this exists for; a fast failure+ * (socket already gone, master already dead) is `connect()`'s normal job. Skipped while a+ * connect is in flight for the project: that attempt owns the master and may be mid-handshake.+ */+ private async dropMasterIfHalfDead(projectId: string, e: Conn): Promise<void> {+ if (!this.r.probe || this.inFlight.has(projectId)) return+ let verdict: 'answered' | 'timeout'+ try {+ verdict = await this.r.probe(masterRoundTripArgs(e.conn, e.controlPath), WAKE_ROUND_TRIP_TIMEOUT_MS)+ } catch {+ return // a probe that could not run proves nothing either way+ }+ if (verdict !== 'timeout') return+ // Re-check ownership: a disconnect or endpoint change during the probe means this entry's+ // master is no longer ours to end.+ if (this.conns.get(projectId) !== e || this.inFlight.has(projectId)) return+ console.warn(+ `[ssh-project] master for ${sshHostKey(e.conn)} did not answer a round trip after wake; ending it so terminals reconnect`+ )+ await this.r.run(exitMasterArgs(e.conn, e.controlPath)).catch(() => {})+ }+/** The connection's cached remote `--permission-mode auto` capability (undefined = not* probed / not connected). Feeds the agent-status settings block the phone reads. */remoteAutoPermFor(projectId: string): boolean | undefined {@@ -2706,6 +2756,17 @@ export function initSshProject(}})),+ // Deliberately NOT through `sshChildGate`: after a sleep the gate is typically full of children+ // hung on the very master this probes, and queueing behind them would time the queue.+ probe: (args, timeoutMs) =>+ new Promise((resolve) => {+ execFile(+ ssh,+ args,+ { timeout: timeoutMs, env: { ...process.env, ...appSshAgent.env() } },+ (err) => resolve(err && (err as { killed?: boolean }).killed ? 'timeout' : 'answered')+ )+ }),runScp: (args) =>new Promise((resolve) => {// Same reason as `run`: scp re-authenticates when the master socket is gone.