Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

perf(ssh): cap concurrent ssh exec children per ControlMaster

enes9e9907b4f584
diff --git a/src/core/remote-ssh/ssh-child-gate.test.ts b/src/core/remote-ssh/ssh-child-gate.test.ts
new file mode 100644
index 000000000..44084954d
--- /dev/null
+++ b/src/core/remote-ssh/ssh-child-gate.test.ts
@@ -0,0 +1,162 @@
+import { describe, it, expect } from 'vitest'
+import {
+ SshChildGate,
+ controlPathOf,
+ isMuxControlCommand,
+ SSH_CHILD_CONCURRENCY
+} from './ssh-child-gate'
+import { childArgs, checkMasterArgs, exitMasterArgs, hookForwardArgs } from './control-master'
+import type { SshConnection } from '../../shared/ssh'
+
+const conn: SshConnection = { user: 'deploy', host: 'h.example.com' }
+
+function deferred() {
+ let resolve!: () => void
+ const promise = new Promise<void>((r) => {
+ resolve = r
+ })
+ return { promise, resolve }
+}
+
+describe('controlPathOf', () => {
+ it('reads the path out of the argv our own builders emit', () => {
+ expect(controlPathOf(childArgs(conn, '/cm/p1.sock', 'true'))).toBe('/cm/p1.sock')
+ expect(controlPathOf(checkMasterArgs(conn, '/cm/p1.sock'))).toBe('/cm/p1.sock')
+ })
+
+ it('accepts the single-token spelling ssh also accepts', () => {
+ expect(controlPathOf(['-oControlPath=/cm/x.sock', 'h'])).toBe('/cm/x.sock')
+ })
+
+ it('is undefined when no control path is named', () => {
+ expect(controlPathOf(['-G', '[email protected]'])).toBeUndefined()
+ })
+
+ it('does not mistake another ControlPath-suffixed option for the path', () => {
+ expect(controlPathOf(['-o', 'ControlPersist=300', '-o', 'ControlMaster=auto'])).toBeUndefined()
+ })
+})
+
+describe('isMuxControlCommand', () => {
+ it('recognises every -O command we build', () => {
+ expect(isMuxControlCommand(checkMasterArgs(conn, '/cm/p.sock'))).toBe(true)
+ expect(isMuxControlCommand(exitMasterArgs(conn, '/cm/p.sock'))).toBe(true)
+ expect(isMuxControlCommand(hookForwardArgs(conn, '/cm/p.sock', '/tmp/s.sock', 1234))).toBe(true)
+ })
+
+ it('an ordinary exec child is not one', () => {
+ expect(isMuxControlCommand(childArgs(conn, '/cm/p.sock', 'printf ok'))).toBe(false)
+ })
+})
+
+describe('SshChildGate', () => {
+ it('never runs more than the limit at once for one control path', async () => {
+ const gate = new SshChildGate(3)
+ const args = childArgs(conn, '/cm/p1.sock', 'true')
+ let live = 0
+ let peak = 0
+ const gates = Array.from({ length: 8 }, () => deferred())
+
+ const runs = gates.map((g) =>
+ gate.run(args, async () => {
+ live++
+ peak = Math.max(peak, live)
+ await g.promise
+ live--
+ })
+ )
+
+ await Promise.resolve()
+ expect(peak).toBe(3)
+ // Release them one at a time; the queue must refill but never overshoot.
+ for (const g of gates) {
+ g.resolve()
+ await Promise.resolve()
+ await Promise.resolve()
+ }
+ await Promise.all(runs)
+ expect(peak).toBe(3)
+ expect(gate.inFlight('/cm/p1.sock')).toBe(0)
+ expect(gate.queued('/cm/p1.sock')).toBe(0)
+ })
+
+ it('budgets are per control path — one busy host never throttles another', async () => {
+ const gate = new SshChildGate(1)
+ const a = deferred()
+ const b = deferred()
+ let startedB = false
+
+ void gate.run(childArgs(conn, '/cm/a.sock', 'true'), () => a.promise)
+ const runB = gate.run(childArgs(conn, '/cm/b.sock', 'true'), async () => {
+ startedB = true
+ await b.promise
+ })
+
+ await Promise.resolve()
+ expect(startedB).toBe(true)
+ b.resolve()
+ a.resolve()
+ await runB
+ })
+
+ it('mux control commands bypass the queue entirely', async () => {
+ const gate = new SshChildGate(1)
+ const held = deferred()
+ void gate.run(childArgs(conn, '/cm/p1.sock', 'true'), () => held.promise)
+ await Promise.resolve()
+
+ // The watchdog's health probe must not wait behind a 5 MB transcript read.
+ let checked = false
+ await gate.run(checkMasterArgs(conn, '/cm/p1.sock'), async () => {
+ checked = true
+ })
+ expect(checked).toBe(true)
+ held.resolve()
+ })
+
+ it('releases the slot when the work throws', async () => {
+ const gate = new SshChildGate(1)
+ const args = childArgs(conn, '/cm/p1.sock', 'true')
+ await expect(
+ gate.run(args, async () => {
+ throw new Error('spawn failed')
+ })
+ ).rejects.toThrow('spawn failed')
+ expect(gate.inFlight('/cm/p1.sock')).toBe(0)
+
+ let ran = false
+ await gate.run(args, async () => {
+ ran = true
+ })
+ expect(ran).toBe(true)
+ })
+
+ it('runs FIFO, so a queued read cannot be starved by later arrivals', async () => {
+ const gate = new SshChildGate(1)
+ const args = childArgs(conn, '/cm/p1.sock', 'true')
+ const order: number[] = []
+ const first = deferred()
+
+ const runs = [
+ gate.run(args, async () => {
+ order.push(0)
+ await first.promise
+ }),
+ gate.run(args, async () => {
+ order.push(1)
+ }),
+ gate.run(args, async () => {
+ order.push(2)
+ })
+ ]
+ await Promise.resolve()
+ first.resolve()
+ await Promise.all(runs)
+ expect(order).toEqual([0, 1, 2])
+ })
+
+ it('ships a ceiling below a stock sshd MaxSessions', () => {
+ expect(SSH_CHILD_CONCURRENCY).toBeLessThan(10)
+ expect(SSH_CHILD_CONCURRENCY).toBeGreaterThan(1)
+ })
+})
diff --git a/src/core/remote-ssh/ssh-child-gate.ts b/src/core/remote-ssh/ssh-child-gate.ts
new file mode 100644
index 000000000..9b9874dd4
--- /dev/null
+++ b/src/core/remote-ssh/ssh-child-gate.ts
@@ -0,0 +1,131 @@
+// A per-ControlMaster ceiling on how many ssh EXEC children may be in flight at once.
+//
+// WHY (measured against a real sshd, 2026-09-14). Every non-pty ssh command an SSH project runs is
+// an exec child multiplexed over that project's one ControlMaster, and a stock sshd allows
+// `MaxSessions 10` channels on one connection. A connect fans out ~23 hook-install round trips plus
+// four fire-and-forget chains, a Source Control refresh is 13 concurrent `git` children, and every
+// live remote terminal is already holding one of those channels for its whole life. When the burst
+// overruns the limit each excess child is told `Session open refused by peer`, falls back to
+// becoming a master, cannot bind the socket the live master owns (`ControlSocket … already exists,
+// disabling multiplexing`) and performs a FULL TCP+KEX+auth login.
+//
+// A flood of simultaneous logins then trips the host's `MaxStartups`, and sshd resets the losers —
+// `kex_exchange_identification: read: Connection reset by peer`. That is the expensive outcome: a
+// reset is not a slow command, it is a FAILED one, and in the app it becomes a dropped terminal and
+// the reconnect backoff (1s, 2s, 4s, 8s, 15s).
+//
+// Lab: one switch to a 10-terminal project with a 20-probe burst, MaxSessions 10, 20 ms RTT.
+// ungated (4 runs): 4, 6, 6, 8 connections RESET; once a pane never painted at all (9/10)
+// gated at 6 (3 runs): 0, 0, 0 resets; every pane painted
+// Wall time is a wash (0.62–1.71 s gated, 0.68–1.14 s ungated) — the gate paces logins rather than
+// preventing them once the terminals alone fill the host's limit. What it removes is the resets.
+//
+// Two things it deliberately does NOT gate:
+// - **Mux CONTROL commands** (`ssh -O check/exit/forward/cancel`). They talk to the master's
+// control socket instead of opening a session channel, so they cost the host nothing and cannot
+// be refused for a channel limit — and queueing the 45 s watchdog's `-O check` behind six 5 MB
+// transcript reads would make the health probe report on a connection it never looked at.
+// - **The terminals themselves.** A pty is spawned through node-pty, not through this runner, so
+// a terminal never waits on a queue for a screen the user is looking at.
+//
+// The ceiling is per CONTROL PATH because that is what a `MaxSessions` limit is scoped to: one
+// connection. Two SSH projects on the same host have two masters and two independent budgets.
+
+/** Concurrent exec children allowed per ControlMaster. Deliberately below a stock `MaxSessions 10`
+ * while leaving room for the terminals, which hold their channels for life and are never queued. */
+export const SSH_CHILD_CONCURRENCY = 6
+
+/**
+ * The `ControlPath` this argv addresses, or `undefined` when it names none.
+ *
+ * Every builder in `control-master.ts` emits it as the two tokens `-o` `ControlPath=<p>`; the
+ * single-token `-oControlPath=<p>` spelling is accepted too, because ssh does and a future caller
+ * may. Children with no control path share one bucket: they are not multiplexed, so they cannot
+ * exhaust a master's channels, but they are still ssh logins and a burst of them is still a burst.
+ */
+export function controlPathOf(args: readonly string[]): string | undefined {
+ for (let i = 0; i < args.length; i++) {
+ const a = args[i]
+ if (a === '-o' && i + 1 < args.length) {
+ const v = args[i + 1]
+ if (v.startsWith('ControlPath=')) return v.slice('ControlPath='.length)
+ } else if (a.startsWith('-oControlPath=')) {
+ return a.slice('-oControlPath='.length)
+ }
+ }
+ return undefined
+}
+
+/**
+ * Is this a mux CONTROL command (`-O check|exit|forward|cancel`) rather than a session channel?
+ *
+ * Matched on the flag alone: `-O` takes a command argument and ssh has no other use for it, so the
+ * flag's presence is the whole test. These are never queued (see the header).
+ */
+export function isMuxControlCommand(args: readonly string[]): boolean {
+ return args.includes('-O')
+}
+
+/**
+ * Per-control-path concurrency limiter for ssh exec children.
+ *
+ * FIFO within a bucket, so a queued read cannot be starved by later arrivals. A slot is always
+ * released, including when the work throws — the runner this wraps resolves rather than rejects,
+ * but a limiter that can leak a permit is a limiter that eventually wedges a whole project.
+ */
+export class SshChildGate {
+ private active = new Map<string, number>()
+ private waiting = new Map<string, (() => void)[]>()
+
+ constructor(private limit: number = SSH_CHILD_CONCURRENCY) {}
+
+ /** Run `fn` under this argv's budget. Mux control commands bypass the queue entirely. */
+ async run<T>(args: readonly string[], fn: () => Promise<T>): Promise<T> {
+ if (isMuxControlCommand(args)) return fn()
+ const key = controlPathOf(args) ?? ''
+ await this.acquire(key)
+ try {
+ return await fn()
+ } finally {
+ this.release(key)
+ }
+ }
+
+ /** In-flight children for a control path — for tests and diagnostics. */
+ inFlight(controlPath: string): number {
+ return this.active.get(controlPath) ?? 0
+ }
+
+ /** Children queued behind the limit for a control path. */
+ queued(controlPath: string): number {
+ return this.waiting.get(controlPath)?.length ?? 0
+ }
+
+ private acquire(key: string): Promise<void> {
+ const n = this.active.get(key) ?? 0
+ if (n < this.limit) {
+ this.active.set(key, n + 1)
+ return Promise.resolve()
+ }
+ return new Promise<void>((resolve) => {
+ const q = this.waiting.get(key)
+ if (q) q.push(resolve)
+ else this.waiting.set(key, [resolve])
+ })
+ }
+
+ private release(key: string): void {
+ const q = this.waiting.get(key)
+ const next = q?.shift()
+ if (next) {
+ if (q && q.length === 0) this.waiting.delete(key)
+ // The permit is handed straight to the waiter — `active` never dips, so a burst cannot
+ // slip past the limit in the gap between one child finishing and the next starting.
+ next()
+ return
+ }
+ const n = (this.active.get(key) ?? 1) - 1
+ if (n <= 0) this.active.delete(key)
+ else this.active.set(key, n)
+ }
+}
diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.ts
index 35dbf75be..3e2386798 100644
--- a/src/main/remote-ssh/ssh-project.ts
+++ b/src/main/remote-ssh/ssh-project.ts
@@ -40,6 +40,7 @@ import {
scpDownArgs,
RMT_TMUX_SOCKET
} from '../../core/remote-ssh/control-master'
+import { SshChildGate } from '../../core/remote-ssh/ssh-child-gate'
import { claudeVersionProbeCommand, parseClaudeVersionProbe } from '../../core/remote-ssh/claude-version-probe'
import { RemoteHooks } from './remote-hooks'
import {
@@ -2315,6 +2316,12 @@ export function resolvePassphrasePrompt(requestId: string, value: string | null)
pendingPassphrasePrompts.delete(requestId)
}
+/** One concurrency budget per ControlMaster, shared by every ssh exec child this process runs
+ * (the manager's own commands and every `sshRun` caller — transcript reads, remote git, the
+ * board-log poll, the session-memory sweep). Module-level so a manager rebuilt in a test does
+ * not hand a host two budgets. */
+const sshChildGate = new SshChildGate()
+
export function initSshProject(
onConnected?: (projectId: string) => void,
askpassScriptPath?: string,
@@ -2401,34 +2408,41 @@ export function initSshProject(
// The master may already be gone, the host unreachable, or the timeout hit. Best effort.
}
},
+ // Bounded per ControlMaster: a connect's install fan-out, a Source Control refresh and a
+ // switch's mount burst all land on one multiplexed connection, and past the host's
+ // `MaxSessions` every excess child silently becomes a full login — enough of those at once
+ // and sshd's `MaxStartups` resets some outright, which the app sees as a dropped terminal.
+ // Mux control commands and the terminals themselves are never queued (see ssh-child-gate.ts).
run: (args, stdin) =>
- new Promise((resolve) => {
- // 16 MB ceiling: remote transcript reads pull up to REMOTE_TRANSCRIPT_CAP (5 MB) via
- // RemoteFile; the default 1 MB maxBuffer would kill the child and silently break the
- // remote context meter / subagent transcript / content search for large transcripts.
- // (cf. pty-manager tmux capture 50 MB, git-service 20–50 MB.) Just a ceiling, safe for
- // the small Phase-1/2a control commands too.
- // The agent env rides along here too: `childArgs` uses `ControlMaster=auto`, so with the
- // master down the first child ssh authenticates for real, and the only place the unlocked
- // key lives is the app-private agent.
- const child = execFile(
- ssh,
- args,
- { timeout: 15000, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...appSshAgent.env() } },
- (err, stdout) =>
- resolve({ code: err ? ((err as { code?: number }).code ?? 1) : 0, stdout: stdout ?? '' })
- )
- if (stdin !== undefined) {
- // ssh can die before draining stdin (unreachable host, bad option, instant auth
- // refusal) — that EPIPE is an async 'error' EVENT on the pipe, not a throw here, and
- // unhandled it kills the main process (issue #382's class). The execFile callback
- // above already reports the child's exit; log and stand by.
- child.stdin?.on('error', (e: NodeJS.ErrnoException) => {
- console.warn(`[ssh-project] ssh stdin write failed (${e.code ?? e})`)
- })
- child.stdin?.end(stdin)
- }
- }),
+ sshChildGate.run(args, () =>
+ new Promise<{ code: number; stdout: string }>((resolve) => {
+ // 16 MB ceiling: remote transcript reads pull up to REMOTE_TRANSCRIPT_CAP (5 MB) via
+ // RemoteFile; the default 1 MB maxBuffer would kill the child and silently break the
+ // remote context meter / subagent transcript / content search for large transcripts.
+ // (cf. pty-manager tmux capture 50 MB, git-service 20–50 MB.) Just a ceiling, safe for
+ // the small Phase-1/2a control commands too.
+ // The agent env rides along here too: `childArgs` uses `ControlMaster=auto`, so with the
+ // master down the first child ssh authenticates for real, and the only place the unlocked
+ // key lives is the app-private agent.
+ const child = execFile(
+ ssh,
+ args,
+ { timeout: 15000, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...appSshAgent.env() } },
+ (err, stdout) =>
+ resolve({ code: err ? ((err as { code?: number }).code ?? 1) : 0, stdout: stdout ?? '' })
+ )
+ if (stdin !== undefined) {
+ // ssh can die before draining stdin (unreachable host, bad option, instant auth
+ // refusal) — that EPIPE is an async 'error' EVENT on the pipe, not a throw here, and
+ // unhandled it kills the main process (issue #382's class). The execFile callback
+ // above already reports the child's exit; log and stand by.
+ child.stdin?.on('error', (e: NodeJS.ErrnoException) => {
+ console.warn(`[ssh-project] ssh stdin write failed (${e.code ?? e})`)
+ })
+ child.stdin?.end(stdin)
+ }
+ })
+ ),
runScp: (args) =>
new Promise((resolve) => {
// Same reason as `run`: scp re-authenticates when the master socket is gone.