Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Isolate real tmux discovery tests from live sessions (#563)

Deeca2d7faddf12
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8883260b7..659fdbaa2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -198,6 +198,11 @@ npm run test:tmux-real # RUN_REAL_TMUX_DISCOVERY=1 — drives a real tmux se
npm run test:cursor-e2e # RUN_CURSOR_E2E=1 — needs a real cursor-agent CLI
```
+`test:tmux-real` owns its private tmux server and its cleanup. Run it directly; do not wrap it in
+a bare `tmux kill-server` trap. `TMUX_TMPDIR` alone does not isolate a test launched inside a tmux
+pane, because inherited `TMUX` takes precedence. New tmux fixtures must clear `TMUX` and
+`TMUX_PANE`, use a private socket, and explicitly target that socket with `-S` during cleanup.
+
If your change touches how agents are discovered or driven, run the real multiplexer suite for the
software available on your machine and say exactly which versions and engine rows ran. A missing
binary, credential, or onboarding step is an unavailable row, not a passing one.
diff --git a/cli/README.md b/cli/README.md
index b97f932ea..fd756abe9 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -241,6 +241,11 @@ npm ci
npm run test:tmux-real
```
+The tmux suite creates and cleans up its own private server, including when run from inside a
+Harness pane. No outer `tmux kill-server` cleanup is needed. Setting `TMUX_TMPDIR` alone does not
+isolate tmux: an inherited `TMUX` still selects the parent server. Any separate fixture must clear
+`TMUX`/`TMUX_PANE` and name its private socket explicitly with `tmux -S` for cleanup.
+
The suite uses isolated, test-owned lifecycle fixtures. Its engine matrix explicitly skips commands that are not installed; authentication or
first-run onboarding that prevents a proprietary CLI from running is unavailable evidence and must be
reported as such, not described as exercised.
diff --git a/cli/src/lib/tmuxAgentDiscovery.real.spec.ts b/cli/src/lib/tmuxAgentDiscovery.real.spec.ts
index 54e646eba..84aecb1e3 100644
--- a/cli/src/lib/tmuxAgentDiscovery.real.spec.ts
+++ b/cli/src/lib/tmuxAgentDiscovery.real.spec.ts
@@ -1,9 +1,7 @@
-import { execFile } from 'node:child_process'
import { access, mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
-import { promisify } from 'node:util'
-import { afterAll, describe, expect, it } from 'vitest'
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import type { AgentEngine } from '../engines/types.js'
import {
agentAliasOwner,
@@ -17,14 +15,14 @@ import { TmuxBackend } from './tmuxBackend.js'
import { probeTmuxAgents } from './tmuxAgentDiscovery.js'
import { probeTerminalAgents } from './terminalAgentDiscovery.js'
import { HARNESS_SESSION_PREFIX } from './harnessSessionLabel.js'
+import { isolatedTmux, type IsolatedTmux } from '../testing/isolatedTmux.js'
// Discovery only sees sessions named like agent_create's; a session outside the prefix is
// invisible to it by design, so every session this suite expects to find must wear it.
const SESSION_PREFIX = `${HARNESS_SESSION_PREFIX}real-${process.pid}`
-const exec = promisify(execFile)
const realDescribe = process.env.RUN_REAL_TMUX_DISCOVERY === '1' ? describe : describe.skip
-const createdSessions = new Set<string>()
+let server: IsolatedTmux | undefined
const ownership = agentCommandOwnershipSnapshot()
const engines: Array<[AgentEngine, string, string | null]> = ENGINES.map((engine) => [
@@ -47,7 +45,8 @@ for (const [engine, candidates] of [
}
async function tmux(args: string[]): Promise<string> {
- return (await exec('tmux', args, { timeout: 5_000 })).stdout.trim()
+ if (!server) throw new Error('The private tmux test server has not been initialized')
+ return server.run(...args)
}
async function eventually<T>(read: () => Promise<T | null>, timeoutMs = 20_000): Promise<T | null> {
@@ -61,17 +60,23 @@ async function eventually<T>(read: () => Promise<T | null>, timeoutMs = 20_000):
}
realDescribe.sequential('real installed CLI process discovery', () => {
+ beforeAll(async () => {
+ server = await isolatedTmux()
+ vi.stubEnv('TMUX_TMPDIR', server.root)
+ vi.stubEnv('TMUX', undefined)
+ vi.stubEnv('TMUX_PANE', undefined)
+ // Start with our config before TmuxBackend's first bare new-session call.
+ await server.run('new-session', '-d', '-s', `${SESSION_PREFIX}-keeper`, 'sleep 600')
+ })
+
afterAll(async () => {
- for (const session of createdSessions) {
- await exec('tmux', ['kill-session', '-t', session], { timeout: 5_000 }).catch(() => {})
- }
+ try { await server?.close() } finally { vi.unstubAllEnvs() }
})
it('creates, notifies, and kills a test-owned session through the shared backend contract', async () => {
const dir = await mkdtemp(join(tmpdir(), 'harness-real-tmux-lifecycle-'))
const session = `${SESSION_PREFIX}-lifecycle`
const backend = new TmuxBackend()
- createdSessions.add(session)
try {
const created = await backend.create({ cwd: dir, label: session })
expect(created.state).toBe('succeeded')
@@ -83,10 +88,8 @@ realDescribe.sequential('real installed CLI process discovery', () => {
await expect(backend.kill(created.runtime))
.resolves.toEqual({ state: 'succeeded', dispatch: 'executed' })
await expect(tmux(['has-session', '-t', session])).rejects.toBeTruthy()
- createdSessions.delete(session)
} finally {
- await exec('tmux', ['kill-session', '-t', session], { timeout: 5_000 }).catch(() => {})
- createdSessions.delete(session)
+ await tmux(['kill-session', '-t', session]).catch(() => {})
await rm(dir, { recursive: true, force: true })
}
})
@@ -97,7 +100,6 @@ realDescribe.sequential('real installed CLI process discovery', () => {
matrixIt(`discovers ${engine} once and process-only deletion preserves its pane`, async () => {
const dir = await mkdtemp(join(tmpdir(), `harness-real-${engine}-`))
const session = `${SESSION_PREFIX}-${engine}`
- createdSessions.add(session)
try {
await tmux(['new-session', '-d', '-s', session, '-c', dir])
@@ -147,8 +149,7 @@ realDescribe.sequential('real installed CLI process discovery', () => {
expect(absent, `${bin} remained discoverable after its exact saved PID was terminated`).toBe(true)
await expect(tmux(['display-message', '-p', '-t', pane, '#{pane_id}'])).resolves.toBe(pane)
} finally {
- await exec('tmux', ['kill-session', '-t', session], { timeout: 5_000 }).catch(() => {})
- createdSessions.delete(session)
+ await tmux(['kill-session', '-t', session]).catch(() => {})
await rm(dir, { recursive: true, force: true })
}
}, 35_000)
@@ -157,7 +158,6 @@ realDescribe.sequential('real installed CLI process discovery', () => {
it('types literally and submits short text through stdin-loaded buffers', async () => {
const dir = await mkdtemp(join(tmpdir(), 'harness-real-tmux-input-'))
const session = `${SESSION_PREFIX}-input`
- createdSessions.add(session)
try {
await tmux(['new-session', '-d', '-s', session, '-c', dir])
const pane = await tmux(['list-panes', '-t', session, '-F', '#{pane_id}'])
@@ -179,8 +179,7 @@ realDescribe.sequential('real installed CLI process discovery', () => {
})
expect(await eventually(() => access(submitMarker).then(() => true).catch(() => null), 5_000)).toBe(true)
} finally {
- await exec('tmux', ['kill-session', '-t', session], { timeout: 5_000 }).catch(() => {})
- createdSessions.delete(session)
+ await tmux(['kill-session', '-t', session]).catch(() => {})
await rm(dir, { recursive: true, force: true })
}
}, 15_000)
@@ -188,7 +187,6 @@ realDescribe.sequential('real installed CLI process discovery', () => {
it.each(installedAgentAliases)('classifies installed %s alias named agent from its executable identity', async (engine, path) => {
const dir = await mkdtemp(join(tmpdir(), `harness-real-agent-alias-${engine}-`))
const session = `${SESSION_PREFIX}-${engine}-agent-alias`
- createdSessions.add(session)
try {
await tmux(['new-session', '-d', '-s', session, '-c', dir])
const pane = await tmux(['list-panes', '-t', session, '-F', '#{pane_id}'])
@@ -203,8 +201,7 @@ realDescribe.sequential('real installed CLI process discovery', () => {
const capture = await tmux(['capture-pane', '-p', '-t', pane]).catch(() => '')
expect(discovered, `${path} was not classified as ${engine} in ${pane}\n${capture}`).not.toBeNull()
} finally {
- await exec('tmux', ['kill-session', '-t', session], { timeout: 5_000 }).catch(() => {})
- createdSessions.delete(session)
+ await tmux(['kill-session', '-t', session]).catch(() => {})
await rm(dir, { recursive: true, force: true })
}
}, 35_000)
diff --git a/cli/src/testing/isolatedTmux.spec.ts b/cli/src/testing/isolatedTmux.spec.ts
new file mode 100644
index 000000000..947947178
--- /dev/null
+++ b/cli/src/testing/isolatedTmux.spec.ts
@@ -0,0 +1,41 @@
+import { execFile, execFileSync } from 'node:child_process'
+import { promisify } from 'node:util'
+import { describe, expect, it } from 'vitest'
+import { isolatedTmux } from './isolatedTmux.js'
+
+const exec = promisify(execFile)
+const hasTmux = (() => { try { execFileSync('tmux', ['-V']); return true } catch { return false } })()
+
+describe.skipIf(!hasTmux)('real tmux test isolation', () => {
+ it('isolates backend calls and cleanup from an inherited parent server', async () => {
+ const parent = await isolatedTmux()
+ let child: Awaited<ReturnType<typeof isolatedTmux>> | undefined
+ try {
+ const pane = await parent.run('new-session', '-d', '-P', '-F', '#{pane_id}', '-s', 'sentinel', 'sleep 600')
+ const pid = await parent.run('display-message', '-p', '-t', pane, '#{pid}')
+ const inherited = { ...process.env, TMUX: `${parent.socket},${pid},0`, TMUX_PANE: pane }
+ child = await isolatedTmux(inherited)
+ // Reproduce the incident safely: TMPDIR alone still selects this disposable parent.
+ const wrong = await exec('tmux', ['display-message', '-p', '-t', pane, '#{socket_path}'], {
+ env: { ...inherited, TMUX_TMPDIR: child.root }, timeout: 5_000,
+ })
+ expect(wrong.stdout.trim()).toBe(parent.socket)
+
+ await child.run('new-session', '-d', '-s', 'fixture', 'sleep 600')
+ // TmuxBackend/probeTmuxAgents use bare commands; prove their environment selects the child.
+ const actual = await exec('tmux', ['display-message', '-p', '-t', 'fixture', '#{socket_path}'], {
+ env: child.env, timeout: 5_000,
+ })
+ expect(actual.stdout.trim()).toBe(child.socket)
+ expect(await parent.run('list-sessions', '-F', '#{session_name}')).toBe('sentinel')
+
+ await child.close()
+ child = undefined
+ expect(await parent.run('display-message', '-p', '-t', pane, '#{pid}')).toBe(pid)
+ expect(await parent.run('list-sessions', '-F', '#{session_name}')).toBe('sentinel')
+ } finally {
+ await child?.close()
+ await parent.close()
+ }
+ }, 15_000)
+})
diff --git a/cli/src/testing/isolatedTmux.ts b/cli/src/testing/isolatedTmux.ts
new file mode 100644
index 000000000..358cd1c6e
--- /dev/null
+++ b/cli/src/testing/isolatedTmux.ts
@@ -0,0 +1,37 @@
+import { execFile } from 'node:child_process'
+import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { resolveBinaryOnPath } from '../lib/binaryOnPath.js'
+
+const exec = promisify(execFile)
+
+/** Bare tmux calls in the backend under test must use the same private server as the fixture.
+ * TMUX outranks TMUX_TMPDIR, so changing only the latter still reaches the developer's server.
+ * Cleanup always names the captured socket, even after the caller restores its environment. */
+export async function isolatedTmux(inherited: NodeJS.ProcessEnv = process.env) {
+ const binary = resolveBinaryOnPath('tmux')
+ if (!binary) throw new Error('tmux is required for this test')
+ const root = await realpath(await mkdtemp(join(tmpdir(), 'htmux-')))
+ const env: NodeJS.ProcessEnv = { ...inherited, TMUX_TMPDIR: root }
+ delete env.TMUX
+ delete env.TMUX_PANE
+ const socketDirectory = join(root, `tmux-${process.getuid!()}`)
+ await mkdir(socketDirectory, { mode: 0o700 })
+ const socket = join(socketDirectory, 'default')
+ const run = async (...args: string[]) =>
+ (await exec(binary, ['-S', socket, '-f', '/dev/null', ...args], { env, timeout: 5_000 })).stdout.trim()
+ return {
+ root,
+ socket,
+ env,
+ run,
+ async close() {
+ await run('kill-server').catch(() => { /* this disposable server may already have exited */ })
+ await rm(root, { recursive: true, force: true })
+ },
+ }
+}
+
+export type IsolatedTmux = Awaited<ReturnType<typeof isolatedTmux>>