Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.
TypeScript adhdagent-orchestrationai-agentscanvasclaude-codecodexcoding-agentsdeveloper-toolselectrongeminilinuxmacosmultiplexeropencodeparallel-agentsterminalterminal-emulatorterminal-multiplexertmuxworkspace-manager
perf(ssh): cap concurrent ssh exec children per ControlMaster
enes9e9907b4f584
diff --git a/src/core/remote-ssh/ssh-child-gate.test.ts b/src/core/remote-ssh/ssh-child-gate.test.tsnew file mode 100644index 000000000..44084954d--- /dev/null+++ b/src/core/remote-ssh/ssh-child-gate.test.ts@@ -0,0 +1,162 @@+import { describe, it, expect } from 'vitest'+import {+ SshChildGate,+ controlPathOf,+ isMuxControlCommand,+ SSH_CHILD_CONCURRENCY+} from './ssh-child-gate'+import { childArgs, checkMasterArgs, exitMasterArgs, hookForwardArgs } from './control-master'+import type { SshConnection } from '../../shared/ssh'++const conn: SshConnection = { user: 'deploy', host: 'h.example.com' }++function deferred() {+ let resolve!: () => void+ const promise = new Promise<void>((r) => {+ resolve = r+ })+ return { promise, resolve }+}++describe('controlPathOf', () => {+ it('reads the path out of the argv our own builders emit', () => {+ expect(controlPathOf(childArgs(conn, '/cm/p1.sock', 'true'))).toBe('/cm/p1.sock')+ expect(controlPathOf(checkMasterArgs(conn, '/cm/p1.sock'))).toBe('/cm/p1.sock')+ })++ it('accepts the single-token spelling ssh also accepts', () => {+ expect(controlPathOf(['-oControlPath=/cm/x.sock', 'h'])).toBe('/cm/x.sock')+ })++ it('is undefined when no control path is named', () => {+ expect(controlPathOf(['-G', '[email protected]'])).toBeUndefined()+ })++ it('does not mistake another ControlPath-suffixed option for the path', () => {+ expect(controlPathOf(['-o', 'ControlPersist=300', '-o', 'ControlMaster=auto'])).toBeUndefined()+ })+})++describe('isMuxControlCommand', () => {+ it('recognises every -O command we build', () => {+ expect(isMuxControlCommand(checkMasterArgs(conn, '/cm/p.sock'))).toBe(true)+ expect(isMuxControlCommand(exitMasterArgs(conn, '/cm/p.sock'))).toBe(true)+ expect(isMuxControlCommand(hookForwardArgs(conn, '/cm/p.sock', '/tmp/s.sock', 1234))).toBe(true)+ })++ it('an ordinary exec child is not one', () => {+ expect(isMuxControlCommand(childArgs(conn, '/cm/p.sock', 'printf ok'))).toBe(false)+ })+})++describe('SshChildGate', () => {+ it('never runs more than the limit at once for one control path', async () => {+ const gate = new SshChildGate(3)+ const args = childArgs(conn, '/cm/p1.sock', 'true')+ let live = 0+ let peak = 0+ const gates = Array.from({ length: 8 }, () => deferred())++ const runs = gates.map((g) =>+ gate.run(args, async () => {+ live+++ peak = Math.max(peak, live)+ await g.promise+ live--+ })+ )++ await Promise.resolve()+ expect(peak).toBe(3)+ // Release them one at a time; the queue must refill but never overshoot.+ for (const g of gates) {+ g.resolve()+ await Promise.resolve()+ await Promise.resolve()+ }+ await Promise.all(runs)+ expect(peak).toBe(3)+ expect(gate.inFlight('/cm/p1.sock')).toBe(0)+ expect(gate.queued('/cm/p1.sock')).toBe(0)+ })++ it('budgets are per control path — one busy host never throttles another', async () => {+ const gate = new SshChildGate(1)+ const a = deferred()+ const b = deferred()+ let startedB = false++ void gate.run(childArgs(conn, '/cm/a.sock', 'true'), () => a.promise)+ const runB = gate.run(childArgs(conn, '/cm/b.sock', 'true'), async () => {+ startedB = true+ await b.promise+ })++ await Promise.resolve()+ expect(startedB).toBe(true)+ b.resolve()+ a.resolve()+ await runB+ })++ it('mux control commands bypass the queue entirely', async () => {+ const gate = new SshChildGate(1)+ const held = deferred()+ void gate.run(childArgs(conn, '/cm/p1.sock', 'true'), () => held.promise)+ await Promise.resolve()++ // The watchdog's health probe must not wait behind a 5 MB transcript read.+ let checked = false+ await gate.run(checkMasterArgs(conn, '/cm/p1.sock'), async () => {+ checked = true+ })+ expect(checked).toBe(true)+ held.resolve()+ })++ it('releases the slot when the work throws', async () => {+ const gate = new SshChildGate(1)+ const args = childArgs(conn, '/cm/p1.sock', 'true')+ await expect(+ gate.run(args, async () => {+ throw new Error('spawn failed')+ })+ ).rejects.toThrow('spawn failed')+ expect(gate.inFlight('/cm/p1.sock')).toBe(0)++ let ran = false+ await gate.run(args, async () => {+ ran = true+ })+ expect(ran).toBe(true)+ })++ it('runs FIFO, so a queued read cannot be starved by later arrivals', async () => {+ const gate = new SshChildGate(1)+ const args = childArgs(conn, '/cm/p1.sock', 'true')+ const order: number[] = []+ const first = deferred()++ const runs = [+ gate.run(args, async () => {+ order.push(0)+ await first.promise+ }),+ gate.run(args, async () => {+ order.push(1)+ }),+ gate.run(args, async () => {+ order.push(2)+ })+ ]+ await Promise.resolve()+ first.resolve()+ await Promise.all(runs)+ expect(order).toEqual([0, 1, 2])+ })++ it('ships a ceiling below a stock sshd MaxSessions', () => {+ expect(SSH_CHILD_CONCURRENCY).toBeLessThan(10)+ expect(SSH_CHILD_CONCURRENCY).toBeGreaterThan(1)+ })+})diff --git a/src/core/remote-ssh/ssh-child-gate.ts b/src/core/remote-ssh/ssh-child-gate.tsnew file mode 100644index 000000000..9b9874dd4--- /dev/null+++ b/src/core/remote-ssh/ssh-child-gate.ts@@ -0,0 +1,131 @@+// A per-ControlMaster ceiling on how many ssh EXEC children may be in flight at once.+//+// WHY (measured against a real sshd, 2026-09-14). Every non-pty ssh command an SSH project runs is+// an exec child multiplexed over that project's one ControlMaster, and a stock sshd allows+// `MaxSessions 10` channels on one connection. A connect fans out ~23 hook-install round trips plus+// four fire-and-forget chains, a Source Control refresh is 13 concurrent `git` children, and every+// live remote terminal is already holding one of those channels for its whole life. When the burst+// overruns the limit each excess child is told `Session open refused by peer`, falls back to+// becoming a master, cannot bind the socket the live master owns (`ControlSocket … already exists,+// disabling multiplexing`) and performs a FULL TCP+KEX+auth login.+//+// A flood of simultaneous logins then trips the host's `MaxStartups`, and sshd resets the losers —+// `kex_exchange_identification: read: Connection reset by peer`. That is the expensive outcome: a+// reset is not a slow command, it is a FAILED one, and in the app it becomes a dropped terminal and+// the reconnect backoff (1s, 2s, 4s, 8s, 15s).+//+// Lab: one switch to a 10-terminal project with a 20-probe burst, MaxSessions 10, 20 ms RTT.+// ungated (4 runs): 4, 6, 6, 8 connections RESET; once a pane never painted at all (9/10)+// gated at 6 (3 runs): 0, 0, 0 resets; every pane painted+// Wall time is a wash (0.62–1.71 s gated, 0.68–1.14 s ungated) — the gate paces logins rather than+// preventing them once the terminals alone fill the host's limit. What it removes is the resets.+//+// Two things it deliberately does NOT gate:+// - **Mux CONTROL commands** (`ssh -O check/exit/forward/cancel`). They talk to the master's+// control socket instead of opening a session channel, so they cost the host nothing and cannot+// be refused for a channel limit — and queueing the 45 s watchdog's `-O check` behind six 5 MB+// transcript reads would make the health probe report on a connection it never looked at.+// - **The terminals themselves.** A pty is spawned through node-pty, not through this runner, so+// a terminal never waits on a queue for a screen the user is looking at.+//+// The ceiling is per CONTROL PATH because that is what a `MaxSessions` limit is scoped to: one+// connection. Two SSH projects on the same host have two masters and two independent budgets.++/** Concurrent exec children allowed per ControlMaster. Deliberately below a stock `MaxSessions 10`+ * while leaving room for the terminals, which hold their channels for life and are never queued. */+export const SSH_CHILD_CONCURRENCY = 6++/**+ * The `ControlPath` this argv addresses, or `undefined` when it names none.+ *+ * Every builder in `control-master.ts` emits it as the two tokens `-o` `ControlPath=<p>`; the+ * single-token `-oControlPath=<p>` spelling is accepted too, because ssh does and a future caller+ * may. Children with no control path share one bucket: they are not multiplexed, so they cannot+ * exhaust a master's channels, but they are still ssh logins and a burst of them is still a burst.+ */+export function controlPathOf(args: readonly string[]): string | undefined {+ for (let i = 0; i < args.length; i++) {+ const a = args[i]+ if (a === '-o' && i + 1 < args.length) {+ const v = args[i + 1]+ if (v.startsWith('ControlPath=')) return v.slice('ControlPath='.length)+ } else if (a.startsWith('-oControlPath=')) {+ return a.slice('-oControlPath='.length)+ }+ }+ return undefined+}++/**+ * Is this a mux CONTROL command (`-O check|exit|forward|cancel`) rather than a session channel?+ *+ * Matched on the flag alone: `-O` takes a command argument and ssh has no other use for it, so the+ * flag's presence is the whole test. These are never queued (see the header).+ */+export function isMuxControlCommand(args: readonly string[]): boolean {+ return args.includes('-O')+}++/**+ * Per-control-path concurrency limiter for ssh exec children.+ *+ * FIFO within a bucket, so a queued read cannot be starved by later arrivals. A slot is always+ * released, including when the work throws — the runner this wraps resolves rather than rejects,+ * but a limiter that can leak a permit is a limiter that eventually wedges a whole project.+ */+export class SshChildGate {+ private active = new Map<string, number>()+ private waiting = new Map<string, (() => void)[]>()++ constructor(private limit: number = SSH_CHILD_CONCURRENCY) {}++ /** Run `fn` under this argv's budget. Mux control commands bypass the queue entirely. */+ async run<T>(args: readonly string[], fn: () => Promise<T>): Promise<T> {+ if (isMuxControlCommand(args)) return fn()+ const key = controlPathOf(args) ?? ''+ await this.acquire(key)+ try {+ return await fn()+ } finally {+ this.release(key)+ }+ }++ /** In-flight children for a control path — for tests and diagnostics. */+ inFlight(controlPath: string): number {+ return this.active.get(controlPath) ?? 0+ }++ /** Children queued behind the limit for a control path. */+ queued(controlPath: string): number {+ return this.waiting.get(controlPath)?.length ?? 0+ }++ private acquire(key: string): Promise<void> {+ const n = this.active.get(key) ?? 0+ if (n < this.limit) {+ this.active.set(key, n + 1)+ return Promise.resolve()+ }+ return new Promise<void>((resolve) => {+ const q = this.waiting.get(key)+ if (q) q.push(resolve)+ else this.waiting.set(key, [resolve])+ })+ }++ private release(key: string): void {+ const q = this.waiting.get(key)+ const next = q?.shift()+ if (next) {+ if (q && q.length === 0) this.waiting.delete(key)+ // The permit is handed straight to the waiter — `active` never dips, so a burst cannot+ // slip past the limit in the gap between one child finishing and the next starting.+ next()+ return+ }+ const n = (this.active.get(key) ?? 1) - 1+ if (n <= 0) this.active.delete(key)+ else this.active.set(key, n)+ }+}diff --git a/src/main/remote-ssh/ssh-project.ts b/src/main/remote-ssh/ssh-project.tsindex 35dbf75be..3e2386798 100644--- a/src/main/remote-ssh/ssh-project.ts+++ b/src/main/remote-ssh/ssh-project.ts@@ -40,6 +40,7 @@ import {scpDownArgs,RMT_TMUX_SOCKET} from '../../core/remote-ssh/control-master'+import { SshChildGate } from '../../core/remote-ssh/ssh-child-gate'import { claudeVersionProbeCommand, parseClaudeVersionProbe } from '../../core/remote-ssh/claude-version-probe'import { RemoteHooks } from './remote-hooks'import {@@ -2315,6 +2316,12 @@ export function resolvePassphrasePrompt(requestId: string, value: string | null)pendingPassphrasePrompts.delete(requestId)}+/** One concurrency budget per ControlMaster, shared by every ssh exec child this process runs+ * (the manager's own commands and every `sshRun` caller — transcript reads, remote git, the+ * board-log poll, the session-memory sweep). Module-level so a manager rebuilt in a test does+ * not hand a host two budgets. */+const sshChildGate = new SshChildGate()+export function initSshProject(onConnected?: (projectId: string) => void,askpassScriptPath?: string,@@ -2401,34 +2408,41 @@ export function initSshProject(// The master may already be gone, the host unreachable, or the timeout hit. Best effort.}},+ // Bounded per ControlMaster: a connect's install fan-out, a Source Control refresh and a+ // switch's mount burst all land on one multiplexed connection, and past the host's+ // `MaxSessions` every excess child silently becomes a full login — enough of those at once+ // and sshd's `MaxStartups` resets some outright, which the app sees as a dropped terminal.+ // Mux control commands and the terminals themselves are never queued (see ssh-child-gate.ts).run: (args, stdin) =>- new Promise((resolve) => {- // 16 MB ceiling: remote transcript reads pull up to REMOTE_TRANSCRIPT_CAP (5 MB) via- // RemoteFile; the default 1 MB maxBuffer would kill the child and silently break the- // remote context meter / subagent transcript / content search for large transcripts.- // (cf. pty-manager tmux capture 50 MB, git-service 20–50 MB.) Just a ceiling, safe for- // the small Phase-1/2a control commands too.- // The agent env rides along here too: `childArgs` uses `ControlMaster=auto`, so with the- // master down the first child ssh authenticates for real, and the only place the unlocked- // key lives is the app-private agent.- const child = execFile(- ssh,- args,- { timeout: 15000, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...appSshAgent.env() } },- (err, stdout) =>- resolve({ code: err ? ((err as { code?: number }).code ?? 1) : 0, stdout: stdout ?? '' })- )- if (stdin !== undefined) {- // ssh can die before draining stdin (unreachable host, bad option, instant auth- // refusal) — that EPIPE is an async 'error' EVENT on the pipe, not a throw here, and- // unhandled it kills the main process (issue #382's class). The execFile callback- // above already reports the child's exit; log and stand by.- child.stdin?.on('error', (e: NodeJS.ErrnoException) => {- console.warn(`[ssh-project] ssh stdin write failed (${e.code ?? e})`)- })- child.stdin?.end(stdin)- }- }),+ sshChildGate.run(args, () =>+ new Promise<{ code: number; stdout: string }>((resolve) => {+ // 16 MB ceiling: remote transcript reads pull up to REMOTE_TRANSCRIPT_CAP (5 MB) via+ // RemoteFile; the default 1 MB maxBuffer would kill the child and silently break the+ // remote context meter / subagent transcript / content search for large transcripts.+ // (cf. pty-manager tmux capture 50 MB, git-service 20–50 MB.) Just a ceiling, safe for+ // the small Phase-1/2a control commands too.+ // The agent env rides along here too: `childArgs` uses `ControlMaster=auto`, so with the+ // master down the first child ssh authenticates for real, and the only place the unlocked+ // key lives is the app-private agent.+ const child = execFile(+ ssh,+ args,+ { timeout: 15000, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, ...appSshAgent.env() } },+ (err, stdout) =>+ resolve({ code: err ? ((err as { code?: number }).code ?? 1) : 0, stdout: stdout ?? '' })+ )+ if (stdin !== undefined) {+ // ssh can die before draining stdin (unreachable host, bad option, instant auth+ // refusal) — that EPIPE is an async 'error' EVENT on the pipe, not a throw here, and+ // unhandled it kills the main process (issue #382's class). The execFile callback+ // above already reports the child's exit; log and stand by.+ child.stdin?.on('error', (e: NodeJS.ErrnoException) => {+ console.warn(`[ssh-project] ssh stdin write failed (${e.code ?? e})`)+ })+ child.stdin?.end(stdin)+ }+ })+ ),runScp: (args) =>new Promise((resolve) => {// Same reason as `run`: scp re-authenticates when the master socket is gone.