Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.
TypeScript adhdagent-orchestrationai-agentscanvasclaude-codecodexcoding-agentsdeveloper-toolselectrongeminilinuxmacosmultiplexeropencodeparallel-agentsterminalterminal-emulatorterminal-multiplexertmuxworkspace-manager
fix(ssh): spill channels past the server's MaxSessions onto more connections
enesca0dd37f4810
diff --git a/src/core/remote-ssh/native/native-invoke.ts b/src/core/remote-ssh/native/native-invoke.tsindex 292a1d9a7..6e427954e 100644--- a/src/core/remote-ssh/native/native-invoke.ts+++ b/src/core/remote-ssh/native/native-invoke.ts@@ -231,6 +231,7 @@ export function spawnSshArgvStream(mux: NativeMux, argv: string[]): NativeStreamch.pipe(child.stdout, { end: false })ch.stderr.pipe(child.stderr, { end: false })ch.on('close', () => finish(channelExit(ch).code ?? 255))+ if (channelExit(ch).closed) queueMicrotask(() => finish(channelExit(ch).code ?? 255))ch.on('error', () => {})ch.end()},diff --git a/src/core/remote-ssh/native/native-mux.ts b/src/core/remote-ssh/native/native-mux.tsindex ce310c5ff..f377b7b46 100644--- a/src/core/remote-ssh/native/native-mux.ts+++ b/src/core/remote-ssh/native/native-mux.ts@@ -68,6 +68,24 @@ interface Conn {alive: booleanclosed: Promise<string>forwards: Map<string, ReverseForward>+ target: SshTarget+ options: SshOptions+ /** Channels open on this connection right now. */+ channels: number+ /** The server refused a channel here (its MaxSessions): skip it until one closes. */+ full: boolean+}++/**+ * Connections a group may grow to when the server refuses channels (MaxSessions — 10 on a stock+ * sshd). OpenSSH's answer to the same refusal is a full login per refused client with no bound at+ * all; this one is bounded so a runaway cannot trip the host's MaxStartups.+ */+export const MAX_OVERFLOW_CONNECTIONS = 12++/** ssh2's error for a channel the server refused to open (MaxSessions reached). */+function isChannelRefused(e: unknown): boolean {+ return e instanceof Error && /Channel open failure/i.test(e.message)}function targetKey(t: SshTarget): string {@@ -98,7 +116,20 @@ function defaultConnectSocket(host: string, port: number, timeoutMs: number): Pr}export class NativeMux {+ /** The PRIMARY connection per ControlPath — the one -O check/exit and forwards speak about. */private conns = new Map<string, Conn>()+ /**+ * Extra connections a ControlPath grew because its primary's server refused channels. They live+ * and die with the primary, exactly as mux clients die with their ControlMaster.+ */+ private overflow = new Map<string, Conn[]>()+ /**+ * Keys unlocked with a passphrase during this process, so an overflow connection (never+ * interactive) can authenticate without asking again. Held only while some connection is alive —+ * the "decrypted key in memory for the connection's life" tradeoff of this transport on Windows,+ * where there is no app-private ssh-agent to hold it instead.+ */+ private unlocked = new Map<string, ParsedKey>()constructor(private deps: NativeMuxDeps) {}@@ -114,6 +145,7 @@ export class NativeMux {this.conns.delete(controlPath)c.alive = falsec.client.end()+ this.endOverflow(controlPath)await Promise.race([c.closed, new Promise((r) => setTimeout(r, 2000))])return true}@@ -124,7 +156,76 @@ export class NativeMux {this.conns.delete(cp)c.alive = falsec.client.end()+ this.endOverflow(cp)+ }+ this.unlocked.clear()+ }++ /** Connections a ControlPath has open (primary + overflow) — diagnostics and tests. */+ connectionCount(controlPath: string): number {+ return (this.conns.get(controlPath)?.alive ? 1 : 0) + (this.overflow.get(controlPath)?.length ?? 0)+ }++ private endOverflow(controlPath: string): void {+ for (const o of this.overflow.get(controlPath) ?? []) {+ o.alive = false+ o.client.end()+ }+ this.overflow.delete(controlPath)+ }++ /**+ * Open a channel on the ControlPath's group: the first connection not known to be full; on a+ * refusal, mark it full and move on, growing an overflow connection when every one is. `open`+ * is the ssh2 call that opens the channel on a given client.+ */+ private async openOn<T extends NodeJS.EventEmitter>(+ controlPath: string,+ primary: Conn,+ open: (client: Client, cb: (err: Error | undefined, ch: T) => void) => void,+ /** Runs INSIDE ssh2's callback, before any await: a consumer attached later can miss the+ * channel's exit and close, which may arrive in the same read as the open confirmation. */+ onOpen?: (ch: T) => void+ ): Promise<T> {+ for (let attempt = 0; attempt <= MAX_OVERFLOW_CONNECTIONS + 1; attempt++) {+ if (!primary.alive) throw new Error(noMasterMessage(controlPath))+ const group = [primary, ...(this.overflow.get(controlPath) ?? [])].filter((c) => c.alive)+ let conn = group.find((c) => !c.full)+ if (!conn) {+ const extra = this.overflow.get(controlPath) ?? []+ if (extra.length >= MAX_OVERFLOW_CONNECTIONS) {+ throw new Error('ssh: channel open failed: the server refused more sessions (MaxSessions)')+ }+ conn = this.open(`${controlPath}#${extra.length + 1}`, primary.target, primary.options, { interactive: false })+ const added = conn+ extra.push(added)+ this.overflow.set(controlPath, extra)+ void added.closed.then(() => {+ const list = this.overflow.get(controlPath)+ if (list) this.overflow.set(controlPath, list.filter((o) => o !== added))+ })+ }+ await conn.ready+ const c = conn+ try {+ return await new Promise<T>((resolve, reject) =>+ open(c.client, (err, ch) => {+ if (err) return reject(err)+ c.channels+++ ch.once('close', () => {+ c.channels--+ c.full = false+ })+ onOpen?.(ch)+ resolve(ch)+ })+ )+ } catch (e) {+ if (!isChannelRefused(e)) throw e+ c.full = true+ }}+ throw new Error('ssh: channel open failed')}/**@@ -170,7 +271,24 @@ export class NativeMux {}if (!conn) throw new Error(cp ? noMasterMessage(cp) : 'ssh: no connection')await conn.ready- return await this.runCommand(conn, p.command ?? '', io)+ const c = conn+ let result!: Promise<ExecResult>+ const start = (ch: ClientChannel): void => {+ recordExit(ch)+ result = this.collect(ch, io)+ }+ if (cp && !oneOff) {+ await this.openOn<ClientChannel>(cp, c, (client, cb) => client.exec(p.command ?? '', cb), start)+ } else {+ await new Promise<void>((resolve, reject) =>+ c.client.exec(p.command ?? '', (err, ch) => {+ if (err) return reject(err)+ start(ch)+ resolve()+ })+ )+ }+ return await result} catch (e) {return fail(e)} finally {@@ -202,19 +320,16 @@ export class NativeMux {}const conn = this.conns.get(cp)!await conn.ready- return new Promise((resolve, reject) => {- const cb = (err: Error | undefined, ch: ClientChannel): void => {- if (err) return reject(err)- recordExit(ch)- resolve(ch)- }- if (!pty) {- conn.client.exec(p.command ?? '', cb)- return+ const ptyOpts = pty ? { cols: pty.cols, rows: pty.rows, term: pty.term ?? 'xterm-256color' } : undefined+ return this.openOn<ClientChannel>(cp, conn, (client, cb) => {+ // recordExit runs INSIDE ssh2's callback, before anything can be awaited (see recordExit).+ const done = (err: Error | undefined, ch: ClientChannel): void => {+ if (!err) recordExit(ch)+ cb(err, ch)}- const ptyOpts = { cols: pty.cols, rows: pty.rows, term: pty.term ?? 'xterm-256color' }- if (p.command) conn.client.exec(p.command, { pty: ptyOpts }, cb)- else conn.client.shell(ptyOpts, cb)+ if (!ptyOpts) client.exec(p.command ?? '', done)+ else if (p.command) client.exec(p.command, { pty: ptyOpts }, done)+ else client.shell(ptyOpts, done)})}@@ -233,9 +348,12 @@ export class NativeMux {const c = conntry {await c.ready- const s = await new Promise<SFTPWrapper>((resolve, reject) =>- c.client.sftp((err, sftp) => (err ? reject(err) : resolve(sftp)))- )+ const s =+ controlPath && !oneOff+ ? await this.openOn<SFTPWrapper>(controlPath, c, (client, cb) => client.sftp(cb))+ : await new Promise<SFTPWrapper>((resolve, reject) =>+ c.client.sftp((err, sftp) => (err ? reject(err) : resolve(sftp)))+ )if (oneOff) s.on('close', () => c.client.end())return s} catch (e) {@@ -267,38 +385,31 @@ export class NativeMux {// ---------------------------------------------------------------------------------------------- private runCommand(conn: Conn, command: string, io: { stdin?: string | Buffer; timeoutMs?: number }): Promise<ExecResult> {+ /** Drain an exec channel into an ExecResult (stdin written, timeout enforced). */+ private collect(ch: ClientChannel, io: { stdin?: string | Buffer; timeoutMs?: number }): Promise<ExecResult> {return new Promise((resolve) => {- conn.client.exec(command, (err, ch) => {- if (err) return resolve(fail(err))- const out: Buffer[] = []- const errOut: Buffer[] = []- let code: number | null = null- let signal: string | null = null- let timedOut = false- let timer: ReturnType<typeof setTimeout> | undefined- if (io.timeoutMs && io.timeoutMs > 0) {- timer = setTimeout(() => {- timedOut = true- ch.close()- }, io.timeoutMs)- }- ch.on('data', (d: Buffer) => out.push(d))- ch.stderr.on('data', (d: Buffer) => errOut.push(d))- ch.on('exit', (c: number | null, s?: string) => {- code = c- signal = s ?? null- })- ch.on('close', () => {- if (timer) clearTimeout(timer)- // A channel that closed with no exit status was cut off (connection dropped, or our own- // timeout): report it the way a mux'd ssh does, 255.- if (code === null && signal === null) code = 255- resolve({ code, signal, stdout: Buffer.concat(out), stderr: Buffer.concat(errOut), timedOut })- })- if (io.stdin !== undefined) ch.end(io.stdin)- else ch.end()+ const out: Buffer[] = []+ const errOut: Buffer[] = []+ let timedOut = false+ let timer: ReturnType<typeof setTimeout> | undefined+ if (io.timeoutMs && io.timeoutMs > 0) {+ timer = setTimeout(() => {+ timedOut = true+ ch.close()+ }, io.timeoutMs)+ }+ ch.on('data', (d: Buffer) => out.push(d))+ ch.stderr.on('data', (d: Buffer) => errOut.push(d))+ ch.on('close', () => {+ if (timer) clearTimeout(timer)+ const { code: c, signal } = channelExit(ch)+ // A channel that closed with no exit status was cut off (connection dropped, or our own+ // timeout): report it the way a mux'd ssh does, 255.+ const code = c === null && signal === null ? 255 : c+ resolve({ code, signal, stdout: Buffer.concat(out), stderr: Buffer.concat(errOut), timedOut })})+ if (io.stdin !== undefined) ch.end(io.stdin)+ else ch.end()})}@@ -312,7 +423,11 @@ export class NativeMux {ready: Promise.resolve(),alive: true,closed,- forwards: new Map()+ forwards: new Map(),+ target,+ options,+ channels: 0,+ full: false}let lastError = ''client.on('error', (e: Error & { level?: string }) => {@@ -321,7 +436,12 @@ export class NativeMux {})client.on('close', () => {conn.alive = false- if (this.conns.get(controlPath) === conn) this.conns.delete(controlPath)+ if (this.conns.get(controlPath) === conn) {+ this.conns.delete(controlPath)+ // The group dies with its primary, as mux clients die with their ControlMaster.+ this.endOverflow(controlPath)+ }+ if (this.conns.size === 0) this.unlocked.clear()resolveClosed(lastError || 'connection closed')})client.on('unix connection', (info: UNIXConnectionDetails, accept: () => Channel) => {@@ -441,13 +561,17 @@ export class NativeMux {}for (const file of files) {steps.push(async () => {+ const cachedKey = this.unlocked.get(file)+ if (cachedKey) return { type: 'publickey', username, key: cachedKey }const data = read(file)if (!data) return nulllet key = utils.parseKey(data)+ let prompted = falseif (key instanceof Error && /encrypted|passphrase/i.test(key.message)) {if (!mode.interactive || options.batchMode || !this.deps.askPassphrase) return null// Up to three tries, like ssh's NumberOfPasswordPrompts default.for (let i = 0; i < 3 && key instanceof Error; i++) {+ prompted = trueconst pass = await this.deps.askPassphrase(file, { retry: i > 0, target: `${target.user}@${target.host}` })if (pass === null) {cancelled.value = true@@ -457,7 +581,9 @@ export class NativeMux {}}if (key instanceof Error) return null- return { type: 'publickey', username, key: Array.isArray(key) ? key[0] : key }+ const parsed = Array.isArray(key) ? key[0] : key+ if (prompted) this.unlocked.set(file, parsed)+ return { type: 'publickey', username, key: parsed }})}let i = 0@@ -519,7 +645,7 @@ function filteredAgent(agentPath: string, pubFiles: Buffer[]): ReturnType<typeofreturn base}-const exits = new WeakMap<ClientChannel, { code: number | null; signal: string | null }>()+const exits = new WeakMap<ClientChannel, { code: number | null; signal: string | null; closed: boolean }>()/*** Record a channel's exit status the moment it is opened, synchronously. The open confirmation and@@ -529,17 +655,21 @@ const exits = new WeakMap<ClientChannel, { code: number | null; signal: string |* the reconnector after a terminal that simply ended.*/function recordExit(ch: ClientChannel): void {- const rec = { code: null as number | null, signal: null as string | null }+ const rec = { code: null as number | null, signal: null as string | null, closed: false }exits.set(ch, rec)ch.on('exit', (code: number | null, signal?: string) => {rec.code = coderec.signal = signal ?? null})+ ch.once('close', () => {+ rec.closed = true+ })}-/** The exit status recorded for a channel from `channel()` / `shell()` (null until it arrives). */-export function channelExit(ch: ClientChannel): { code: number | null; signal: string | null } {- return exits.get(ch) ?? { code: null, signal: null }+/** The exit status recorded for a channel from `channel()` / `shell()` (null until it arrives),+ * and whether it has already closed — a consumer that attached late checks this first. */+export function channelExit(ch: ClientChannel): { code: number | null; signal: string | null; closed: boolean } {+ return exits.get(ch) ?? { code: null, signal: null, closed: false }}function fail(e: unknown): ExecResult {diff --git a/src/core/remote-ssh/native/native-overflow.test.ts b/src/core/remote-ssh/native/native-overflow.test.tsnew file mode 100644index 000000000..2fe891df6--- /dev/null+++ b/src/core/remote-ssh/native/native-overflow.test.ts@@ -0,0 +1,142 @@+// A server that refuses channels past a per-connection limit (sshd's MaxSessions — 10 on a stock+// host). MEASURED against a real host (MaxSessions 64): channel 65 onwards is answered at once with+// `(SSH) Channel open failure: open failed`. On the first live run of the native transport those+// terminals sat blank; OpenSSH answers the same refusal by logging in again per refused client.+// The transport now spills onto another connection instead — and is bounded.++import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'+import fs from 'fs'+import os from 'os'+import path from 'path'+import type { AddressInfo } from 'net'+import { Server, utils, type Connection } from 'ssh2'+import { NativeMux } from './native-mux'+import { parseSshArgv } from './ssh-argv'+import { childArgs, remoteTmuxPtyArgs } from '../control-master'+import type { SshConnection } from '../../../shared/ssh'++const LIMIT = 3+const hostKey = utils.generateKeyPairSync('ed25519')+const clientKey = utils.generateKeyPairSync('ed25519')+let dir: string+let server: Server+let port: number+let logins = 0+const clients: Connection[] = []++beforeAll(async () => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nt-native-ovf-'))+ fs.writeFileSync(path.join(dir, 'id'), clientKey.private)+ server = new Server({ hostKeys: [hostKey.private] }, (client) => {+ clients.push(client)+ let open = 0+ client.on('authentication', (ctx) => (ctx.method === 'publickey' ? ctx.accept() : ctx.reject(['publickey'])))+ client.on('ready', () => {+ logins+++ client.on('session', (accept, reject) => {+ if (open >= LIMIT) return reject()+ open+++ const session = accept()+ session.on('close', () => open--)+ session.on('pty', (a) => a?.())+ session.on('exec', (a, _r, info) => {+ const ch = a()+ if (info.command.startsWith('hold')) {+ ch.write('held\r\n') // a terminal: stays open until the client ends+ return+ }+ ch.write(`ran: ${info.command}\n`)+ ch.exit(0)+ ch.end()+ })+ })+ })+ client.on('error', () => {})+ })+ await new Promise<void>((r) => server.listen(0, '127.0.0.1', () => r()))+ port = (server.address() as AddressInfo).port+})+afterAll(() => {+ server.close()+ fs.rmSync(dir, { recursive: true, force: true })+})+let m: NativeMux | null = null+afterEach(() => {+ m?.exitAll()+ m = null+ for (const c of clients.splice(0)) c.end()+})++function mux(): NativeMux {+ m = new NativeMux({+ defaultAgent: () => undefined,+ resolveHost: async () => ({+ hostname: '127.0.0.1',+ port,+ user: 'dev',+ identityFiles: [path.join(dir, 'id')],+ identitiesOnly: true,+ userKnownHostsFiles: [path.join(dir, 'known_hosts')],+ globalKnownHostsFiles: [],+ strictHostKeyChecking: 'ask'+ })+ })+ return m+}+const conn = (): SshConnection => ({ user: 'dev', host: '127.0.0.1', port, identityFile: path.join(dir, 'id') })+const CP = (): string => path.join(dir, 'cm.sock')++describe('channels past the server MaxSessions spill onto more connections', () => {+ it('9 terminals on a limit-3 server all open, on 3 connections, and commands still run', async () => {+ const x = mux()+ logins = 0+ const pty = parseSshArgv(remoteTmuxPtyArgs(conn(), CP(), 'nt-a', '~'))+ if (pty.kind !== 'exec') throw new Error()+ const holdPty = { ...pty, command: 'hold' }+ const chans = await Promise.all(Array.from({ length: 9 }, () => x.shell(holdPty, { cols: 80, rows: 24 })))+ chans.forEach((c) => c.resume())+ expect(chans).toHaveLength(9)+ expect(x.connectionCount(CP())).toBe(3)+ // Every connection is full of terminals: a command must still run (a 4th connection).+ const p = parseSshArgv(childArgs(conn(), CP(), 'uname'))+ if (p.kind !== 'exec') throw new Error()+ const r = await x.exec(p)+ expect(r.code).toBe(0)+ expect(r.stdout.toString()).toBe('ran: uname\n')+ expect(x.connectionCount(CP())).toBe(4)+ expect(logins).toBe(4)+ })++ it('a closed channel frees its connection for the next open (no extra login)', async () => {+ const x = mux()+ const pty = parseSshArgv(remoteTmuxPtyArgs(conn(), CP(), 'nt-b', '~'))+ if (pty.kind !== 'exec') throw new Error()+ const hold = { ...pty, command: 'hold' }+ const chans = await Promise.all(Array.from({ length: LIMIT }, () => x.shell(hold, { cols: 80, rows: 24 })))+ // Consume, as NativeSshPty always does: a paused stream never emits 'close'.+ chans.forEach((c) => c.resume())+ expect(x.connectionCount(CP())).toBe(1)+ const closed = new Promise((r) => chans[0].once('close', r))+ chans[0].close()+ await closed+ const before = logins+ ;(await x.shell(hold, { cols: 80, rows: 24 })).resume()+ expect(x.connectionCount(CP())).toBe(1)+ expect(logins).toBe(before)+ })++ it('the overflow connections end with the primary (-O exit), as mux clients die with their master', async () => {+ const x = mux()+ const pty = parseSshArgv(remoteTmuxPtyArgs(conn(), CP(), 'nt-c', '~'))+ if (pty.kind !== 'exec') throw new Error()+ const hold = { ...pty, command: 'hold' }+ const chans = await Promise.all(Array.from({ length: 5 }, () => x.shell(hold, { cols: 80, rows: 24 })))+ // Consume, as NativeSshPty always does: a paused stream never emits 'close'.+ chans.forEach((c) => c.resume())+ expect(x.connectionCount(CP())).toBe(2)+ const allClosed = Promise.all(chans.map((c) => new Promise((r) => c.once('close', r))))+ await x.exit(CP())+ await allClosed+ expect(x.connectionCount(CP())).toBe(0)+ })+})diff --git a/src/core/remote-ssh/native/native-pty.ts b/src/core/remote-ssh/native/native-pty.tsindex 18b1187a8..2ef7e65d5 100644--- a/src/core/remote-ssh/native/native-pty.ts+++ b/src/core/remote-ssh/native/native-pty.ts@@ -61,6 +61,8 @@ export class NativeSshPty {ch.on('data', (d: Buffer) => this.emitData(out.write(d)))ch.stderr.on('data', (d: Buffer) => this.emitData(err.write(d)))ch.on('close', () => this.exit(channelExit(ch).code ?? 255))+ // Closed before we got here (a remote side that exited at once): report it now.+ if (channelExit(ch).closed) queueMicrotask(() => this.exit(channelExit(ch).code ?? 255))ch.on('error', () => {})if (this.paused) ch.pause()for (const q of this.queue.splice(0)) ch.write(q)