Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

tmux before 3.7: a terminal attaching never meets a notification that crashes the server (#950)

Deedabdbae1031e
diff --git a/cli/src/architecture.spec.ts b/cli/src/architecture.spec.ts
index c299b8163..4b008108f 100644
--- a/cli/src/architecture.spec.ts
+++ b/cli/src/architecture.spec.ts
@@ -299,7 +299,13 @@ const SERVICE_MAY_IMPORT: Record<string, string> = {
// it is the core's (core/devicesWake.ts, 146: a dial's port in /dev every two seconds, a paired Wi-Fi
// device); a Wi-Fi device's requests are held while it starts (core/wifi.ts, +50), and a process on demand
// that did not come in time is answered at once (core/serviceLinks.ts).
-const CORE_CLOSURE_BUDGET = 73_355
+//
+// Grew by 157 to 73,411 in 338, from 73,254 in 337: the gate that keeps a terminal attaching apart from what
+// tmux tells every terminal (lib/tmuxControlGate.ts and its uses in the terminal stream, the pastes, and
+// session create, kill and rename). Before tmux 3.7 the two meeting crashed the tmux server, and every agent
+// with it (windows.e2e.ts, 7 of 27 CI runs on Ubuntu's 3.4). It guards the core's own terminals, so it
+// cannot move to a service. The budget keeps the 101 lines of room it had.
+const CORE_CLOSURE_BUDGET = 73_512
/** What is not the core's, by path: each goes to a service or its own process, in the plan's order. */
const EDGE: RegExp[] = [
diff --git a/cli/src/lib/tmux.spec.ts b/cli/src/lib/tmux.spec.ts
index f7235869f..ee9a4a52d 100644
--- a/cli/src/lib/tmux.spec.ts
+++ b/cli/src/lib/tmux.spec.ts
@@ -20,6 +20,8 @@ import {
sendToTmux,
tmuxCaptureArgs,
} from './tmux.js'
+import { tmuxControlGate } from './tmuxControlGate.js'
+import { assumeTmuxVersion, resetTmuxVersionCache } from './tmuxVersion.js'
const ownership = (cursor: string[] = [], grok: string[] = []): AgentCommandOwnershipSnapshot => ({
cursorFileKeys: new Set(cursor),
@@ -506,6 +508,7 @@ describe('tmux process primitives', () => {
})
it('carries prompt and literal bytes only over stdin, never child argv or diagnostics', async () => {
+ assumeTmuxVersion(null)
const dir = mkdtempSync(join(tmpdir(), 'harness-tmux-input-'))
const argsFile = join(dir, 'args')
const stdinFile = join(dir, 'stdin')
@@ -561,6 +564,8 @@ fi
})
it('brackets every submitted message and sends one separate Enter only after a successful paste', async () => {
+ // Known up front: a `tmux -V` asked through the fake tmux would land in the commands compared below.
+ assumeTmuxVersion(null)
const dir = mkdtempSync(join(tmpdir(), 'harness-tmux-submit-'))
const argsFile = join(dir, 'args')
const fakeTmux = join(dir, 'tmux')
@@ -662,3 +667,47 @@ fi
}
})
})
+
+describe('a paste on a tmux that crashes when one attaches as a buffer changes (tmuxControlGate.ts)', () => {
+ it('waits for a terminal attaching before it sets its buffer, and goes straight through on tmux 3.7', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'harness-tmux-gate-'))
+ const argsFile = join(dir, 'args')
+ writeFileSync(join(dir, 'tmux'), `#!/bin/sh
+printf '%s\\n' "$1" >> "$TMUX_GATE_ARGS"
+if [ "$1" = "load-buffer" ]; then cat > /dev/null; fi
+`)
+ chmodSync(join(dir, 'tmux'), 0o700)
+ writeFileSync(argsFile, '')
+ const previous = { path: process.env.PATH, args: process.env.TMUX_GATE_ARGS }
+ const asked = () => readFileSync(argsFile, 'utf8').trim().split('\n').filter(Boolean)
+ try {
+ process.env.PATH = `${dir}:${previous.path ?? ''}`
+ process.env.TMUX_GATE_ARGS = argsFile
+ assumeTmuxVersion({ major: 3, minor: 4 })
+ const attaching = await tmuxControlGate.enter('attach')
+ const pasted = pasteRawIntoTmux('%7', 'clipboard')
+ await new Promise((resolve) => setTimeout(resolve, 100))
+ expect(asked()).toEqual([])
+ attaching()
+ expect(await pasted).toBe(true)
+ expect(asked()).toEqual(['load-buffer', 'paste-buffer'])
+
+ writeFileSync(argsFile, '')
+ assumeTmuxVersion({ major: 3, minor: 7 })
+ const held = await tmuxControlGate.enter('attach')
+ try {
+ expect(await pasteRawIntoTmux('%7', 'clipboard')).toBe(true)
+ expect(asked()).toEqual(['load-buffer', 'paste-buffer'])
+ } finally {
+ held()
+ }
+ } finally {
+ resetTmuxVersionCache()
+ if (previous.path === undefined) delete process.env.PATH
+ else process.env.PATH = previous.path
+ if (previous.args === undefined) delete process.env.TMUX_GATE_ARGS
+ else process.env.TMUX_GATE_ARGS = previous.args
+ rmSync(dir, { recursive: true, force: true })
+ }
+ })
+})
diff --git a/cli/src/lib/tmux.ts b/cli/src/lib/tmux.ts
index b08e5270d..d1ea7ece7 100644
--- a/cli/src/lib/tmux.ts
+++ b/cli/src/lib/tmux.ts
@@ -19,6 +19,7 @@ import { psEnv } from './childLocale.js'
import { nativeProcessImages } from './nativeProcessImages.js'
import { neutralizePasteControls } from './pasteText.js'
import { patientExec } from './patientExec.js'
+import { inTmuxRoom } from './tmuxControlGate.js'
import { tmuxFeatures, type TmuxFeatures } from './tmuxVersion.js'
export { captureTmuxPane, tmuxCaptureArgs } from './tmuxCapture.js'
@@ -1337,8 +1338,15 @@ function tmuxDeleteBuffer(name: string): Promise<void> {
/** Paste text from a uniquely named stdin-loaded buffer so its bytes never enter argv or errors. A
* bracketed paste carries text and nothing else (pasteText.ts), not even its own end marker; an
- * unbracketed one is typing, and is typed as it was given. */
-async function tmuxPasteText(pane: string, content: string, bracketed: boolean): Promise<boolean> {
+ * unbracketed one is typing, and is typed as it was given.
+ *
+ * A buffer set and deleted is a notification to every control client, which on a tmux before 3.7
+ * crashed the server while one was attaching: the paste waits for none to be (tmuxControlGate.ts). */
+function tmuxPasteText(pane: string, content: string, bracketed: boolean): Promise<boolean> {
+ return inTmuxRoom('notify', () => pasteThroughBuffer(pane, content, bracketed))
+}
+
+async function pasteThroughBuffer(pane: string, content: string, bracketed: boolean): Promise<boolean> {
const bufferName = `machinemsg-${process.pid}-${++injectBufferSequence}`
if (!(await tmuxLoadBuffer(bufferName, bracketed ? neutralizePasteControls(content) : content))) {
await tmuxDeleteBuffer(bufferName)
diff --git a/cli/src/lib/tmuxAgentDiscovery.ts b/cli/src/lib/tmuxAgentDiscovery.ts
index 5c46d5c5d..0b38c09ee 100644
--- a/cli/src/lib/tmuxAgentDiscovery.ts
+++ b/cli/src/lib/tmuxAgentDiscovery.ts
@@ -10,6 +10,7 @@ import { env } from '../config/env.js'
import {
buildHarnessSessionLabel, harnessPaneOwner, isLegacyHarnessSession, ownedHere, paneOwnerFormat, paneOwnerOf,
} from './harnessSessionLabel.js'
+import { inTmuxRoom } from './tmuxControlGate.js'
import { tmuxFeatures } from './tmuxVersion.js'
import { isNoTmuxServerError, rememberTmuxServer, reviveRemovedTmuxSocket } from './tmux.js'
@@ -146,7 +147,9 @@ export async function adoptLegacyHarnessSessions(
// to the next one.
const to = buildHarnessSessionLabel(engine, now + seen.size - 1)
// `=name` is tmux's exact match; a bare name may also be read as a prefix or a pane target.
- const renamed = await execText('tmux', ['rename-session', '-t', `=${pane.tmuxSessionName}`, to], 2_000)
+ // A session renamed is a notification to every control client: on a tmux before 3.7, not while one
+ // attaches (tmuxControlGate.ts).
+ const renamed = await inTmuxRoom('notify', () => execText('tmux', ['rename-session', '-t', `=${pane.tmuxSessionName}`, to], 2_000))
if (renamed.ok) adopted.push({ from: pane.tmuxSessionName, to, paneId: pane.tmuxPane })
}
return adopted
diff --git a/cli/src/lib/tmuxBackend.spec.ts b/cli/src/lib/tmuxBackend.spec.ts
index c2b7b21bf..9c49d67f0 100644
--- a/cli/src/lib/tmuxBackend.spec.ts
+++ b/cli/src/lib/tmuxBackend.spec.ts
@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ExecFileException } from 'node:child_process'
import { TmuxBackend, clearEnvArgs } from './tmuxBackend.js'
import { clearPaneRemainOnExit, forgetTmuxServer } from './tmux.js'
+import { tmuxControlGate } from './tmuxControlGate.js'
import { assumeTmuxVersion, resetTmuxVersionCache } from './tmuxVersion.js'
/**
@@ -132,6 +133,39 @@ esac
])
})
+ it('on a tmux before 3.7, makes and closes a session only while no terminal is attaching (tmuxControlGate.ts)', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'tmux-backend-gate-'))
+ dirs.push(dir)
+ const calls = join(dir, 'calls')
+ writeFileSync(join(dir, 'tmux'), `#!/bin/sh\nprintf '%s\\n' "$1" >> "$TMUX_BACKEND_CALLS"\n[ "$1" = new-session ] && printf '%%42\\n'\nexit 0\n`, { mode: 0o700 })
+ process.env.PATH = `${dir}${delimiter}${originalPath ?? ''}`
+ process.env.TMUX_BACKEND_CALLS = calls
+ assumeTmuxVersion({ major: 3, minor: 4 })
+ const backend = new TmuxBackend(undefined, () => 'daemon-a')
+ const asked = () => existsSync(calls) ? readFileSync(calls, 'utf8').trim().split('\n') : []
+ const wait = () => new Promise((resolve) => setTimeout(resolve, 100))
+
+ let attaching = await tmuxControlGate.enter('attach')
+ const created = backend.create({ cwd: '/tmp/work' })
+ await wait()
+ expect(asked()).toEqual([])
+ attaching()
+ const made = await created
+ expect(made.state).toBe('succeeded')
+ expect(asked()[0]).toBe('new-session')
+ if (made.state !== 'succeeded') return
+ await settled()
+
+ attaching = await tmuxControlGate.enter('attach')
+ const before = asked().length
+ const killed = backend.kill(made.runtime)
+ await wait()
+ expect(asked()).toHaveLength(before)
+ attaching()
+ await expect(killed).resolves.toEqual({ state: 'succeeded', dispatch: 'executed' })
+ expect(asked().at(-1)).toBe('kill-pane')
+ })
+
it('reads the machine\'s name before tmux makes a pane, the name tmux titles it with', async () => {
const dir = mkdtempSync(join(tmpdir(), 'tmux-backend-name-'))
dirs.push(dir)
diff --git a/cli/src/lib/tmuxBackend.ts b/cli/src/lib/tmuxBackend.ts
index dcf6c258a..cb18c02d3 100644
--- a/cli/src/lib/tmuxBackend.ts
+++ b/cli/src/lib/tmuxBackend.ts
@@ -22,6 +22,7 @@ import {
type TmuxRuntimeRef,
type RuntimeValidation,
} from './terminalTypes.js'
+import { inTmuxRoom } from './tmuxControlGate.js'
import { TmuxControlStream } from './tmuxStream.js'
import {
captureTmuxPane,
@@ -178,13 +179,15 @@ export class TmuxBackend implements TerminalBackend<TmuxRuntimeRef> {
args.push(';', 'set-option', scope, HARNESS_OWNER_OPTION, owner)
// `killed`/`signal` come from execFile's own error shape, which ErrnoException alone does not declare.
type ExecError = NodeJS.ErrnoException & { killed?: boolean; signal?: NodeJS.Signals | null }
- const result = await new Promise<{ error: ExecError | null; stdout: string; stderr: string }>((resolve) => {
+ // A new session is a notification to every control client: on a tmux before 3.7, not while one
+ // attaches (tmuxControlGate.ts).
+ const result = await inTmuxRoom('notify', () => new Promise<{ error: ExecError | null; stdout: string; stderr: string }>((resolve) => {
run('tmux', args, { timeout: 5_000 }, (error, stdout, stderr) => resolve({
error: error as ExecError | null,
stdout,
stderr,
}))
- })
+ }), features)
if (result.error) {
if (result.error.code === 'ENOENT') return terminalActionNotStarted('tmux is unavailable')
// tmux says exactly why it refused — "duplicate session", "protocol version mismatch",
@@ -270,9 +273,11 @@ export class TmuxBackend implements TerminalBackend<TmuxRuntimeRef> {
if (!/^%\d+$/.test(runtime.paneId)) return terminalActionNotStarted('invalid tmux pane identity')
// Discovered harnesses can share a tmux session with unrelated work. The
// canonical pane id is the entire target; never widen this to kill-session.
- const ok = await new Promise<boolean>((resolve) => {
+ // The last pane gone closes its session, a notification to every control client: on a tmux before
+ // 3.7, not while one attaches (tmuxControlGate.ts).
+ const ok = await inTmuxRoom('notify', () => new Promise<boolean>((resolve) => {
run('tmux', ['kill-pane', '-t', runtime.paneId], { timeout: 5_000 }, (error) => resolve(!error))
- })
+ }))
if (ok) return TERMINAL_ACTION_SUCCEEDED
// The engine may have exited and removed its pane before the parallel PID
// check completed. Only authoritative inventory makes that an idempotent success.
diff --git a/cli/src/lib/tmuxControlGate.spec.ts b/cli/src/lib/tmuxControlGate.spec.ts
new file mode 100644
index 000000000..53655d1d4
--- /dev/null
+++ b/cli/src/lib/tmuxControlGate.spec.ts
@@ -0,0 +1,107 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { enterTmuxRoom, inTmuxRoom, needsControlGate, tmuxControlGate, TwoRoomGate } from './tmuxControlGate.js'
+import { assumeTmuxVersion, resetTmuxVersionCache, tmuxFeaturesOf } from './tmuxVersion.js'
+
+const tick = () => new Promise((resolve) => setImmediate(resolve))
+
+describe('TwoRoomGate', () => {
+ it('lets any number into one room and none into the other until it is empty', async () => {
+ const gate = new TwoRoomGate()
+ const pasteOne = await gate.enter('notify')
+ const pasteTwo = await gate.enter('notify')
+ let attached = false
+ const attaching = gate.enter('attach').then((leave) => { attached = true; return leave })
+ await tick()
+ expect(attached).toBe(false)
+ expect(gate.state).toEqual({ room: 'notify', inside: 2, waiting: 1 })
+ pasteOne()
+ await tick()
+ expect(attached).toBe(false)
+ pasteTwo()
+ const leave = await attaching
+ expect(gate.state).toEqual({ room: 'attach', inside: 1, waiting: 0 })
+ leave()
+ expect(gate.state).toEqual({ room: null, inside: 0, waiting: 0 })
+ })
+
+ it('serves in arrival order: one arriving for the open room waits behind one waiting for the other', async () => {
+ const gate = new TwoRoomGate()
+ const order: string[] = []
+ const first = await gate.enter('notify')
+ const attaches = [gate.enter('attach'), gate.enter('attach')].map((entered, i) => entered.then((leave) => { order.push(`attach ${i}`); return leave }))
+ // The notify room is open, but an attach is waiting: a stream of pastes must not keep it waiting forever.
+ const later = gate.enter('notify').then((leave) => { order.push('notify'); return leave })
+ await tick()
+ expect(order).toEqual([])
+ first()
+ // Both attaches go in together; the paste after them waits for both to leave.
+ const [one, two] = await Promise.all(attaches)
+ expect(order).toEqual(['attach 0', 'attach 1'])
+ one()
+ await tick()
+ expect(order).toEqual(['attach 0', 'attach 1'])
+ two()
+ ;(await later)()
+ expect(order).toEqual(['attach 0', 'attach 1', 'notify'])
+ expect(gate.state).toEqual({ room: null, inside: 0, waiting: 0 })
+ })
+
+ it('takes a second way out as nothing', async () => {
+ const gate = new TwoRoomGate()
+ const one = await gate.enter('attach')
+ const two = await gate.enter('attach')
+ one()
+ one()
+ expect(gate.state).toEqual({ room: 'attach', inside: 1, waiting: 0 })
+ two()
+ expect(gate.state).toEqual({ room: null, inside: 0, waiting: 0 })
+ })
+})
+
+describe('the gate, by the tmux it runs on', () => {
+ afterEach(() => resetTmuxVersionCache())
+
+ it('is needed before tmux 3.7, and not from it or on a tmux that prints no version', () => {
+ expect(needsControlGate(tmuxFeaturesOf({ major: 3, minor: 4 }))).toBe(true)
+ expect(needsControlGate(tmuxFeaturesOf({ major: 3, minor: 6 }))).toBe(true)
+ expect(needsControlGate(tmuxFeaturesOf({ major: 2, minor: 8 }))).toBe(true)
+ expect(needsControlGate(tmuxFeaturesOf({ major: 3, minor: 7 }))).toBe(false)
+ expect(needsControlGate(tmuxFeaturesOf(null))).toBe(false)
+ })
+
+ it('holds nothing on a tmux that needs no gate', async () => {
+ assumeTmuxVersion({ major: 3, minor: 7 })
+ const held = await tmuxControlGate.enter('attach')
+ try {
+ // A paste on 3.7 goes straight through, even while a terminal attaches.
+ expect(await inTmuxRoom('notify', async () => 'pasted')).toBe('pasted')
+ const leave = await enterTmuxRoom('notify', tmuxFeaturesOf(null))
+ expect(tmuxControlGate.state).toEqual({ room: 'attach', inside: 1, waiting: 0 })
+ leave()
+ expect(tmuxControlGate.state).toEqual({ room: 'attach', inside: 1, waiting: 0 })
+ } finally {
+ held()
+ }
+ })
+
+ it('on an older tmux, runs the work in its room and leaves it however the work ends', async () => {
+ assumeTmuxVersion({ major: 3, minor: 4 })
+ const seen: unknown[] = []
+ await expect(inTmuxRoom('notify', async () => {
+ seen.push(tmuxControlGate.state)
+ throw new Error('tmux refused')
+ })).rejects.toThrow('tmux refused')
+ expect(seen).toEqual([{ room: 'notify', inside: 1, waiting: 0 }])
+ expect(tmuxControlGate.state).toEqual({ room: null, inside: 0, waiting: 0 })
+
+ // And waits while the other room is held.
+ const attaching = await enterTmuxRoom('attach')
+ let ran = false
+ const pasted = inTmuxRoom('notify', async () => { ran = true })
+ await tick()
+ expect(ran).toBe(false)
+ attaching()
+ await pasted
+ expect(ran).toBe(true)
+ })
+})
diff --git a/cli/src/lib/tmuxControlGate.ts b/cli/src/lib/tmuxControlGate.ts
new file mode 100644
index 000000000..4264d11fb
--- /dev/null
+++ b/cli/src/lib/tmuxControlGate.ts
@@ -0,0 +1,94 @@
+/**
+ * Keeps a terminal attaching apart from what tmux tells every terminal, on a tmux that crashes when the
+ * two meet. Every open terminal is a tmux control client (`tmux -C attach-session`, tmuxStream.ts).
+ * Before 3.7 tmux marks a control client at its first identify message but builds the state it writes
+ * notifications into only at its last, a few turns of its loop later. A notification for every control
+ * client in between (a client detaching; a session created, closed or renamed; a paste buffer set or
+ * deleted) dereferences NULL, and the server dies with every agent's pane (tmux issue 4980, fixed in 3.7
+ * by e5a2a25). Ubuntu 24.04 ships 3.4, Debian 12 3.3a, Fedora 3.5a.
+ *
+ * Found by windows.e2e.ts on Ubuntu 24.04: 7 of 27 CI runs failed with `tmux pane metadata is
+ * unavailable`; the server had segfaulted. Measured, 2026-10-06: control clients attaching beside others
+ * detaching killed Ubuntu's 3.4 (SIGSEGV) in each of 6 runs, gdb in `control_write` from
+ * `control_notify_client_detached` with `control_state` NULL; attaches beside paste buffers, or beside
+ * sessions made and killed, killed it every run; 3.7 survived all. The churn is in tmuxStream.real.spec.ts:
+ * without this gate it failed on 3.4 within a second, every run.
+ *
+ * So, before 3.7, a control client attaches in the `attach` room until tmux answers its first command,
+ * and what this daemon does that notifies (a control client going, a session made or killed or renamed,
+ * a paste) runs in the `notify` room. Each room holds any number; the two never overlap. What tmux does
+ * on its own (an engine exiting ends its session) and a person's own tmux clients cannot be held here.
+ */
+
+import { tmuxFeatures, type TmuxFeatures } from './tmuxVersion.js'
+
+export type TmuxGateRoom = 'attach' | 'notify'
+
+interface Waiter {
+ room: TmuxGateRoom
+ admit: () => void
+}
+
+/**
+ * Two rooms, any number inside one, never both occupied. First come, first served: one that arrives
+ * while others wait queues behind them even when its room is the one open, so a stream of pastes never
+ * keeps a terminal from opening, nor the other way round.
+ */
+export class TwoRoomGate {
+ private room: TmuxGateRoom | null = null
+ private inside = 0
+ private readonly queue: Waiter[] = []
+
+ /** Resolves once [room] is entered, with the way out (calling it again does nothing). */
+ enter(room: TmuxGateRoom): Promise<() => void> {
+ if (this.queue.length === 0 && (this.inside === 0 || this.room === room)) return Promise.resolve(this.admit(room))
+ return new Promise((resolve) => this.queue.push({ room, admit: () => resolve(this.admit(room)) }))
+ }
+
+ /** Who is inside and who waits, for tests and the log. */
+ get state(): { room: TmuxGateRoom | null; inside: number; waiting: number } {
+ return { room: this.room, inside: this.inside, waiting: this.queue.length }
+ }
+
+ private admit(room: TmuxGateRoom): () => void {
+ this.room = room
+ this.inside++
+ let left = false
+ return () => {
+ if (left) return
+ left = true
+ this.inside--
+ if (this.inside > 0) return
+ this.room = null
+ // Everyone at the head who wants the same room goes in together.
+ const next = this.queue[0]?.room
+ while (next && this.queue[0]?.room === next) this.queue.shift()!.admit()
+ }
+ }
+}
+
+/** The one gate for this process: every control client and every notifying command of this daemon. */
+export const tmuxControlGate = new TwoRoomGate()
+
+/** Whether this tmux needs the gate: before 3.7 (see above). An unknown version is new, as everywhere. */
+export function needsControlGate(features: TmuxFeatures): boolean {
+ return !features.controlNotifyGuard
+}
+
+const OPEN_DOOR = (): void => {}
+
+/** Enter [room] on a tmux that needs it; on one that does not, the way out of a room never entered. */
+export async function enterTmuxRoom(room: TmuxGateRoom, features?: TmuxFeatures): Promise<() => void> {
+ if (!needsControlGate(features ?? await tmuxFeatures())) return OPEN_DOOR
+ return tmuxControlGate.enter(room)
+}
+
+/** Run [work] in [room] (see `enterTmuxRoom`), leaving it however [work] ends. */
+export async function inTmuxRoom<T>(room: TmuxGateRoom, work: () => Promise<T>, features?: TmuxFeatures): Promise<T> {
+ const leave = await enterTmuxRoom(room, features)
+ try {
+ return await work()
+ } finally {
+ leave()
+ }
+}
diff --git a/cli/src/lib/tmuxLegacySessions.spec.ts b/cli/src/lib/tmuxLegacySessions.spec.ts
index fc8e533df..d41db483d 100644
--- a/cli/src/lib/tmuxLegacySessions.spec.ts
+++ b/cli/src/lib/tmuxLegacySessions.spec.ts
@@ -1,14 +1,20 @@
import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { delimiter, join } from 'node:path'
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { isLegacyHarnessSession } from './harnessSessionLabel.js'
import { adoptLegacyHarnessSessions } from './tmuxAgentDiscovery.js'
+import { assumeTmuxVersion, resetTmuxVersionCache } from './tmuxVersion.js'
const originalPath = process.env.PATH
const dirs: string[] = []
+// Known up front: a rename asks which tmux this is (tmuxControlGate.ts), and a `tmux -V` through the fake
+// would land in the calls compared below.
+beforeEach(() => assumeTmuxVersion(null))
+
afterEach(() => {
+ resetTmuxVersionCache()
process.env.PATH = originalPath
delete process.env.TMUX_LEGACY_CALLS
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true })
diff --git a/cli/src/lib/tmuxStream.oldTmux.spec.ts b/cli/src/lib/tmuxStream.oldTmux.spec.ts
index b0e98b56e..d707723de 100644
--- a/cli/src/lib/tmuxStream.oldTmux.spec.ts
+++ b/cli/src/lib/tmuxStream.oldTmux.spec.ts
@@ -9,6 +9,7 @@ import { EventEmitter } from 'node:events'
import { PassThrough, Writable } from 'node:stream'
import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { afterEach, describe, expect, it, vi } from 'vitest'
+import { tmuxControlGate } from './tmuxControlGate.js'
import { TmuxControlStream } from './tmuxStream.js'
import { assumeTmuxVersion, resetTmuxVersionCache, type TmuxVersion } from './tmuxVersion.js'
@@ -24,7 +25,7 @@ afterEach(() => {
})
/** A control client that answers every command, `display-message` with [meta], and records what it was sent. */
-function controlClient(version: TmuxVersion, meta = '$1|@1|1|80|24|80|24|0|0|0|1|0|0|0|0|0') {
+function controlClient(version: TmuxVersion, meta = '$1|@1|1|80|24|80|24|0|0|0|1|0|0|0|0|0', answersAttach = true) {
assumeTmuxVersion(version)
const sent: string[] = []
const child = new EventEmitter() as {
@@ -56,15 +57,15 @@ function controlClient(version: TmuxVersion, meta = '$1|@1|1|80|24|80|24|0|0|0|1
vi.mocked(spawn).mockImplementation(() => {
setImmediate(() => {
child.emit('spawn')
- setImmediate(() => child.stdout.emit('data', Buffer.from('%begin 1 0 0\n%end 1 0 0\n')))
+ if (answersAttach) setImmediate(() => child.stdout.emit('data', Buffer.from('%begin 1 0 0\n%end 1 0 0\n')))
})
return child
})
return { sent }
}
-async function open(size = { cols: 80, rows: 24 }) {
- const opened = await TmuxControlStream.open('%1', size, { onData: () => {}, onClose: () => {} })
+async function open(size = { cols: 80, rows: 24 }, readOnly = false) {
+ const opened = await TmuxControlStream.open('%1', size, { onData: () => {}, onClose: () => {} }, readOnly)
expect(opened.state).toBe('succeeded')
if (opened.state !== 'succeeded') throw new Error('stream did not open')
return opened.value
@@ -132,3 +133,73 @@ describe('the terminal on an older tmux', () => {
}
})
})
+
+describe('a terminal on a tmux that crashes when one attaches as another goes (tmuxControlGate.ts)', () => {
+ const tick = () => new Promise((resolve) => setImmediate(resolve))
+ const empty = { room: null, inside: 0, waiting: 0 }
+ afterEach(() => { vi.useRealTimers() })
+
+ it('attaches in the attach room until tmux answers, and goes in the other, asking tmux once more before it leaves', async () => {
+ const client = controlClient({ major: 3, minor: 4 })
+ const atSpawn: unknown[] = []
+ const attach = vi.mocked(spawn).getMockImplementation()!
+ vi.mocked(spawn).mockImplementation(((...args: Parameters<typeof spawn>) => {
+ atSpawn.push(tmuxControlGate.state)
+ return attach(...args)
+ }) as typeof spawn)
+ const stream = await open()
+ await stream.attached
+ expect(atSpawn).toEqual([{ room: 'attach', inside: 1, waiting: 0 }])
+ expect(tmuxControlGate.state).toEqual(empty)
+
+ // Another terminal attaching: this one's `detach-client` waits for it.
+ const attaching = await tmuxControlGate.enter('attach')
+ vi.mocked(execFile).mockClear()
+ const closing = stream.close()
+ await tick()
+ expect(client.sent).not.toContain('detach-client')
+ attaching()
+ await closing
+ expect(client.sent.at(-1)).toBe('detach-client')
+ // tmux tells the others it went only once it reads the socket close: one more answer comes after that.
+ expect(vi.mocked(execFile).mock.calls.map((call) => call[1])).toEqual([['display-message', '-p', '#{pid}']])
+ expect(tmuxControlGate.state).toEqual(empty)
+ })
+
+ it('on tmux 3.7 neither waits nor asks again', async () => {
+ const client = controlClient({ major: 3, minor: 7 })
+ const stream = await open()
+ const attaching = await tmuxControlGate.enter('attach')
+ vi.mocked(execFile).mockClear()
+ try {
+ await stream.close()
+ expect(client.sent.at(-1)).toBe('detach-client')
+ expect(vi.mocked(execFile)).not.toHaveBeenCalled()
+ } finally {
+ attaching()
+ }
+ })
+
+ it('leaves the attach room when its client cannot start', async () => {
+ controlClient({ major: 3, minor: 4 })
+ vi.mocked(spawn).mockImplementation((() => {
+ const child = new EventEmitter()
+ setImmediate(() => child.emit('error', new Error('spawn tmux ENOENT')))
+ return child
+ }) as unknown as typeof spawn)
+ const opened = await TmuxControlStream.open('%1', { cols: 80, rows: 24 }, { onData: () => {}, onClose: () => {} })
+ expect(opened).toEqual({ state: 'failed', reason: 'tmux control client could not start' })
+ expect(tmuxControlGate.state).toEqual(empty)
+ })
+
+ it('keeps the others waiting no longer than its first command may take, when tmux never answers its attach', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ controlClient({ major: 3, minor: 4 }, undefined, false)
+ const watching = await open(undefined, true)
+ expect(tmuxControlGate.state).toEqual({ room: 'attach', inside: 1, waiting: 0 })
+ vi.advanceTimersByTime(3_000)
+ expect(tmuxControlGate.state).toEqual(empty)
+ await watching.close()
+ expect(tmuxControlGate.state).toEqual(empty)
+ })
+})
diff --git a/cli/src/lib/tmuxStream.real.spec.ts b/cli/src/lib/tmuxStream.real.spec.ts
index a6d338bd1..4aedf6800 100644
--- a/cli/src/lib/tmuxStream.real.spec.ts
+++ b/cli/src/lib/tmuxStream.real.spec.ts
@@ -8,6 +8,7 @@ import { ENGINES } from '../engines/types.js'
import type { RegisteredSession } from './registry.js'
import { TerminalStreamManager } from './terminalStreamManager.js'
import type { TerminalBackendCoordinator } from './terminalBackendCoordinator.js'
+import { pasteRawIntoTmux } from './tmux.js'
import { TmuxControlStream } from './tmuxStream.js'
import { tmuxFeatures } from './tmuxVersion.js'
import { TerminalBinaryKind, type TerminalBinaryClear } from './terminalBinary.js'
@@ -351,4 +352,32 @@ run('TmuxControlStream real tmux', () => {
await opened.value.close()
}
})
+
+ it('keeps the tmux server up while terminals open and close side by side and messages are pasted', async () => {
+ // Before tmux 3.7 a notification for every control client (one going, a paste buffer set or deleted)
+ // that met one still attaching crashed the server, every agent's pane with it (tmux issue 4980,
+ // tmuxControlGate.ts). This is windows.e2e.ts's churn, which found it: on Ubuntu 24.04's tmux 3.4,
+ // without the gate, the server segfaulted within the first few hundred of these rounds.
+ const beside = `${session}-beside`
+ const besidePane = await tmux(['new-session', '-d', '-P', '-F', '#{pane_id}', '-s', beside, 'cat'])
+ const sink = { onData: () => {}, onClose: () => {} }
+ const openAndClose = async (pane: string, readOnly: boolean) => {
+ const opened = await TmuxControlStream.open(pane, { cols: 90, rows: 25 }, sink, readOnly)
+ expect(opened.state, opened.state === 'failed' ? opened.reason : '').toBe('succeeded')
+ if (opened.state === 'succeeded') await opened.value.close()
+ }
+ try {
+ for (let round = 0; round < 300; round++) {
+ await Promise.all([
+ openAndClose(paneId, false),
+ openAndClose(besidePane, true),
+ openAndClose(paneId, true),
+ pasteRawIntoTmux(besidePane, `round ${round}\n`),
+ ])
+ }
+ expect(await tmux(['display-message', '-p', '-t', paneId, '#{pane_id}'])).toBe(paneId)
+ } finally {
+ await tmux(['kill-session', '-t', beside]).catch(() => { /* best effort */ })
+ }
+ }, 120_000)
})
diff --git a/cli/src/lib/tmuxStream.ts b/cli/src/lib/tmuxStream.ts
index ba4257f24..1dbf98bf9 100644
--- a/cli/src/lib/tmuxStream.ts
+++ b/cli/src/lib/tmuxStream.ts
@@ -1,6 +1,7 @@
import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { patientDeadline } from './patientExec.js'
import { pasteRawIntoTmux } from './tmux.js'
+import { enterTmuxRoom, inTmuxRoom, needsControlGate } from './tmuxControlGate.js'
import { tmuxFeatures, tmuxFeaturesOf, type TmuxFeatures } from './tmuxVersion.js'
import {
TERMINAL_ACTION_SUCCEEDED,
@@ -23,6 +24,12 @@ const MAX_ROWS = 120
// line built from 8192 such bytes and rejects 16384 with `%error`, so this leaves a 4x margin.
const INPUT_CHUNK_BYTES = 2 * 1024
const CONTROL_COMMAND_TIMEOUT_MS = 3_000
+/** The longest a control client keeps every other one and every paste waiting while it attaches, on a
+ * tmux before 3.7 (tmuxControlGate.ts): tmux answers an attach in milliseconds, and one that has not in
+ * this long has its first command time out and its stream closed anyway. */
+const ATTACH_HOLD_MAX_MS = CONTROL_COMMAND_TIMEOUT_MS
+/** How long a going control client gets to exit, after `detach-client` and again after SIGTERM. */
+const CLOSE_EXIT_WAIT_MS = 500
const SNAPSHOT_BUFFER_MAX_BYTES = 2 * 1024 * 1024
const SNAPSHOT_QUIET_MS = 8
const SNAPSHOT_QUIET_MAX_MS = 40
@@ -510,6 +517,13 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
private snapshotPostCutBytes = 0
private snapshotLastOutputAt = 0
private lastTuiScrollAt = 0
+ private settleAttach: () => void = () => {}
+ /**
+ * Settles once tmux has answered this client's attach (its first `%end`), or the client is gone. From
+ * then on the server holds the state it writes this client's notifications into: on a tmux before 3.7
+ * one sent before then crashed the server (tmuxControlGate.ts).
+ */
+ readonly attached: Promise<void> = new Promise((resolve) => { this.settleAttach = resolve })
private constructor(
paneId: string,
@@ -554,6 +568,9 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
// the flag was a usage error, the client exited at once, and no terminal opened on 3.1 or older.
// Nothing is lost there: a control client that never sets its own size counts for none (`resize`).
const flags = features.clientFlags ? ['-f', 'ignore-size'] : []
+ // On a tmux before 3.7, nothing tmux tells every control client may happen while this one attaches:
+ // it crashed the server, and every agent with it (tmuxControlGate.ts).
+ const leave = await enterTmuxRoom('attach', features)
const child = spawn('tmux', ['-C', 'attach-session', ...flags, '-t', paneId], {
stdio: ['pipe', 'pipe', 'pipe'],
})
@@ -562,8 +579,17 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
child.once('spawn', () => { if (!settled) { settled = true; resolve(true) } })
child.once('error', () => { if (!settled) { settled = true; resolve(false) } })
})
- if (!spawned) return { state: 'failed', reason: 'tmux control client could not start' }
+ if (!spawned) {
+ leave()
+ return { state: 'failed', reason: 'tmux control client could not start' }
+ }
const stream = new TmuxControlStream(paneId, child, sink, readOnly, features)
+ const held = setTimeout(leave, ATTACH_HOLD_MAX_MS)
+ held.unref()
+ void stream.attached.then(() => {
+ clearTimeout(held)
+ leave()
+ })
const resized = readOnly ? TERMINAL_ACTION_SUCCEEDED : await stream.resize(size)
if (resized.state !== 'succeeded') {
await stream.close()
@@ -600,6 +626,7 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
if (completed) {
if (!this.commands.isReady && this.commands.idle) {
this.commands.markReady()
+ this.settleAttach()
return
}
this.commands.handleCompleted(completed[1] === 'end' ? 'end' : 'error', completed[2])
@@ -656,6 +683,7 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
}
private notifyClose(reason: string): void {
+ this.settleAttach()
if (this.closeNotified) return
this.closeNotified = true
this.failControlCommands()
@@ -879,14 +907,29 @@ export class TmuxControlStream implements TerminalStreamHandle<TmuxRuntimeRef> {
// makes agent switching shrink and immediately re-expand the pane; TUIs
// such as Grok preserve those intermediate repaint fragments in the live
// screen. The next controller will resize only if its grid truly differs.
- if (this.child.stdin.writable) {
- try { this.child.stdin.write('detach-client\n') } catch { /* ignore */ }
- }
- const exited = await new Promise<boolean>((resolve) => {
+ //
+ // A control client going is a notification to every other (`%client-detached`), so on a tmux before
+ // 3.7 it goes in the gate's other room from one that is attaching (tmuxControlGate.ts).
+ await inTmuxRoom('notify', async () => {
+ if (this.child.stdin.writable) {
+ try { this.child.stdin.write('detach-client\n') } catch { /* ignore */ }
+ }
+ if (!(await this.exited())) {
+ this.child.kill('SIGTERM')
+ await this.exited()
+ }
+ // tmux tells the others when it reads this client's socket close, which comes after the client
+ // has exited. A command another client sends after that is answered only once tmux has read it,
+ // so the room is not left while that notification may still be on its way.
+ if (needsControlGate(this.features)) await execTmux(['display-message', '-p', '#{pid}'])
+ }, this.features)
+ }
+
+ private exited(): Promise<boolean> {
+ return new Promise<boolean>((resolve) => {
if (this.child.exitCode != null || this.child.signalCode != null) { resolve(true); return }
- const timer = setTimeout(() => resolve(false), 500)
+ const timer = setTimeout(() => resolve(false), CLOSE_EXIT_WAIT_MS)
this.child.once('close', () => { clearTimeout(timer); resolve(true) })
})
- if (!exited) this.child.kill('SIGTERM')
}
}
diff --git a/cli/src/lib/tmuxVersion.spec.ts b/cli/src/lib/tmuxVersion.spec.ts
index 139dc2a12..74aef9474 100644
--- a/cli/src/lib/tmuxVersion.spec.ts
+++ b/cli/src/lib/tmuxVersion.spec.ts
@@ -39,7 +39,7 @@ describe('tmuxFeaturesOf', () => {
it('answers each feature by the release that brought it', () => {
const none = {
resizeWindow: false, paneOptions: false, sendKeysHex: false, respawnEnv: false,
- captureTrailingSpaces: false, clientFlags: false, sessionEnv: false,
+ captureTrailingSpaces: false, clientFlags: false, sessionEnv: false, controlNotifyGuard: false,
}
// RHEL and Rocky 8 ship 2.7, Debian 10 ships 2.8.
expect(tmuxFeaturesOf({ major: 2, minor: 7 })).toEqual(none)
@@ -49,8 +49,12 @@ describe('tmuxFeaturesOf', () => {
const three = { ...none, resizeWindow: true, paneOptions: true, sendKeysHex: true, respawnEnv: true }
expect(tmuxFeaturesOf({ major: 3, minor: 0 })).toEqual(three)
expect(tmuxFeaturesOf({ major: 3, minor: 1 })).toEqual({ ...three, captureTrailingSpaces: true })
- const all = { ...three, captureTrailingSpaces: true, clientFlags: true, sessionEnv: true }
- expect(tmuxFeaturesOf({ major: 3, minor: 2 })).toEqual(all)
+ const threeTwo = { ...three, captureTrailingSpaces: true, clientFlags: true, sessionEnv: true }
+ expect(tmuxFeaturesOf({ major: 3, minor: 2 })).toEqual(threeTwo)
+ // Ubuntu 24.04 ships 3.4, Fedora 3.5a: a notification can still reach a control client mid-attach.
+ expect(tmuxFeaturesOf({ major: 3, minor: 6 })).toEqual(threeTwo)
+ const all = { ...threeTwo, controlNotifyGuard: true }
+ expect(tmuxFeaturesOf({ major: 3, minor: 7 })).toEqual(all)
expect(tmuxFeaturesOf({ major: 4, minor: 0 })).toEqual(all)
// A build that prints no number is newer than every release.
expect(tmuxFeaturesOf(null)).toEqual(all)
diff --git a/cli/src/lib/tmuxVersion.ts b/cli/src/lib/tmuxVersion.ts
index 4a535ce67..afccc8aa9 100644
--- a/cli/src/lib/tmuxVersion.ts
+++ b/cli/src/lib/tmuxVersion.ts
@@ -38,6 +38,9 @@ export const TMUX_FEATURE_MIN = {
clientFlags: { major: 3, minor: 2 },
/** `new-session -e`, a session's own environment (3.2). */
sessionEnv: { major: 3, minor: 2 },
+ /** No notification written to a control client that has not finished attaching (3.7). Before it, one
+ * crashed the server (tmux issue 4980; tmuxControlGate.ts). */
+ controlNotifyGuard: { major: 3, minor: 7 },
} as const satisfies Record<string, TmuxVersion>
export type TmuxFeature = keyof typeof TMUX_FEATURE_MIN