Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

harnessd: agents made and closed all day, and what a message says (rounds 32 and 33) (#796)

Deed90a5cb94ca2
diff --git a/cli/e2e/churn.e2e.ts b/cli/e2e/churn.e2e.ts
new file mode 100644
index 000000000..ffe267b4f
--- /dev/null
+++ b/cli/e2e/churn.e2e.ts
@@ -0,0 +1,107 @@
+/**
+ * Agents made and closed all day, for Claude Code and Codex, on the real daemon. A person opens an
+ * agent, gives it a task and closes it, again and again, on a daemon that runs for weeks. Whatever the
+ * daemon keeps per agent (watchers, caches, timers, maps, open files, panes) must leave with the agent:
+ * memory and open files settle, no pane or engine is left behind, and the live list ends empty.
+ * `CHURN_ROUNDS` scales it (the suite runs a few; a hundred is a better soak).
+ */
+import { execFile, execFileSync } from 'node:child_process'
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+const exec = promisify(execFile)
+const ROUNDS = Number(process.env.CHURN_ROUNDS ?? 10)
+
+const rows = async (client: LocalClient, includeStopped = true): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
+ const cwd = join(d.projectsDir, folder)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
+ return until(`${folder} to bind its conversation`, async () => {
+ const agent = await row(client, created.agent.id)
+ return agent?.sessionId && agent.status === 'active' ? agent : null
+ }, 60_000, 250)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ await ended
+}
+async function close(client: LocalClient, agentId: string): Promise<void> {
+ const now = (await row(client, agentId))!
+ const answer = await client.request('agent_close', { agentId, sessionId: now.sessionId, createdAt: now.createdAt, mode: 'now' }, 60_000)
+ expect(answer, JSON.stringify(answer)).toMatchObject({ closed: true })
+}
+async function openFiles(pid: number | null): Promise<number> {
+ if (!pid) return 0
+ const { stdout } = await exec('lsof', ['-n', '-P', '-p', String(pid)]).catch(() => ({ stdout: '' }))
+ return stdout.trim().split('\n').length
+}
+/** The lowest of a few samples: what the core holds, not a moment's garbage before a collection. */
+async function settledRss(daemon: IsolatedDaemon): Promise<number> {
+ const samples: number[] = []
+ for (let i = 0; i < 5; i++) { samples.push(await daemon.rssMiB()); await new Promise((resolve) => setTimeout(resolve, 1_000)) }
+ return Math.min(...samples)
+}
+/** The fake engines running for this daemon: an engine-titled process under one of its launch shells. */
+function engines(d: IsolatedDaemon): number {
+ const table = execFileSync('ps', ['-A', '-o', 'pid=,ppid=,command='], { encoding: 'utf8' }).split('\n')
+ .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line)).filter((m): m is RegExpExecArray => !!m)
+ .map((m) => ({ pid: Number(m[1]), ppid: Number(m[2]), command: m[3].trim() }))
+ const launchers = new Set(table.filter((p) => p.command.includes(join(d.root, 'bin'))).map((p) => p.pid))
+ return table.filter((p) => (p.command === 'claude' || p.command === 'codex') && launchers.has(p.ppid)).length
+}
+const panes = async (d: IsolatedDaemon): Promise<number> =>
+ (await d.tmux.run('list-panes', '-a', '-F', '#{pane_id}').catch(() => '')).split('\n').filter(Boolean).length
+
+describe('agents made and closed all day', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ it(`${ROUNDS} rounds of a Claude Code and a Codex agent made, given a task and closed: nothing builds up`, async () => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-120).join('\n')}`) })
+ await d.start()
+ const client = await LocalClient.connect(d)
+ let n = 0
+ const round = async (): Promise<void> => {
+ const made = [await create(d, client, 'claude', `churn-${n}-claude`), await create(d, client, 'codex', `churn-${n}-codex`)]
+ n++
+ await Promise.all(made.map((agent) => turn(client, agent.id, `task ${n} for ${agent.engine}`)))
+ for (const agent of made) await close(client, agent.id)
+ // The client keeps only what it still needs, so its own memory is not the one measured.
+ client.frames.splice(0, client.frames.length)
+ }
+ // Warm: pools, caches and the first transcripts reach their working size.
+ for (let i = 0; i < 3; i++) await round()
+ await until('the warm-up\'s panes and engines to be gone', async () => (await panes(d)) === 0 && engines(d) === 0 || null, 30_000, 500)
+ const rssBefore = await settledRss(d)
+ const filesBefore = await openFiles(d.corePid())
+
+ for (let i = 0; i < ROUNDS; i++) await round()
+
+ await until('every closed agent\'s pane and engine to be gone', async () => (await panes(d)) === 0 && engines(d) === 0 || null, 30_000, 500)
+ const rssAfter = await settledRss(d)
+ const filesAfter = await openFiles(d.corePid())
+ const live = (await rows(client, false)).filter((agent) => agent.status === 'active')
+ const report = `rss ${rssBefore.toFixed(1)} → ${rssAfter.toFixed(1)} MiB · open files ${filesBefore} → ${filesAfter} · ${(ROUNDS + 3) * 2} agents made and closed · ${live.length} left live`
+ if (process.env.CHURN_REPORT) writeFileSync(process.env.CHURN_REPORT, `${report}\n`)
+ expect(live, report).toEqual([])
+ expect(rssAfter - rssBefore, report).toBeLessThan(96)
+ expect(filesAfter - filesBefore, report).toBeLessThan(16)
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ // A round takes several seconds: two binds, two turns and two closes.
+ }, 180_000 + ROUNDS * 20_000)
+})
diff --git a/cli/e2e/content.e2e.ts b/cli/e2e/content.e2e.ts
new file mode 100644
index 000000000..146262de6
--- /dev/null
+++ b/cli/e2e/content.e2e.ts
@@ -0,0 +1,97 @@
+/**
+ * What a message says reaches the engine exactly as one prompt, for Claude Code and Codex. Messages come
+ * from the apps, the phone, hn and other agents, and an agent's words can carry text from anywhere
+ * (a web page it read, a file it opened): emoji joined into one glyph, right-to-left text, tabs, shell
+ * metacharacters, and terminal control sequences. The daemon pastes a message into the engine's pane as
+ * one bracketed paste, then presses Enter. A message must arrive whole and as one prompt, and nothing
+ * in it may act as keystrokes: in particular the paste's own end marker (ESC [201~), which would end
+ * the paste early and turn the rest into typing, and a carriage return inside it.
+ *
+ * Measured (tmux 3.7c): tmux itself defangs control characters inside a bracketed paste (the engine
+ * receives ESC as the two characters `^[`), and that is what keeps the end marker from acting; the
+ * daemon pastes the message as it came. This test is the guard for a tmux that does not.
+ */
+import { execFileSync } from 'node:child_process'
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
+ const cwd = join(d.projectsDir, folder)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
+ return until(`${folder} to bind its conversation`, async () => {
+ const agent = await row(client, created.agent.id)
+ return agent?.sessionId && agent.status === 'active' ? agent : null
+ }, 60_000, 500)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+/** The fake engines running for this daemon: an engine-titled process under one of its launch shells. */
+function engines(d: IsolatedDaemon): number {
+ const table = execFileSync('ps', ['-A', '-o', 'pid=,ppid=,command='], { encoding: 'utf8' }).split('\n')
+ .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line)).filter((m): m is RegExpExecArray => !!m)
+ .map((m) => ({ pid: Number(m[1]), ppid: Number(m[2]), command: m[3].trim() }))
+ const launchers = new Set(table.filter((p) => p.command.includes(join(d.root, 'bin'))).map((p) => p.pid))
+ return table.filter((p) => (p.command === 'claude' || p.command === 'codex') && launchers.has(p.ppid)).length
+}
+
+const ESC = '\x1b'
+// `!exit` ends the fake engine: if any of these leaks out of its paste as typing, the engine is gone.
+const HOSTILE = [
+ ['the paste\'s end marker, then a command', `hello${ESC}[201~\r!exit\r`],
+ ['a carriage return mid-message, then a command', 'first line\r!exit'],
+ ['an escape sequence that clears the screen', `before${ESC}[2J${ESC}[Hafter`],
+] as const
+const FAITHFUL = [
+ 'a family 👨‍👩‍👧‍👦 and a flag 🇻🇳, joined',
+ 'שלום עולם and مرحبا بالعالم, right to left',
+ 'tabs\tbetween\twords and `backticks` $(echo not run) ; | & > <',
+ 'combining: é ä and a zero-width​space',
+] as const
+
+describe('what a message says', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ it.each(['claude', 'codex'] as const)('%s: every message arrives whole, as one prompt, and nothing in it acts as keystrokes', async (engine) => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-120).join('\n')}`) })
+ await d.start()
+ const client = await LocalClient.connect(d)
+ const agent = await create(d, client, engine, `content-${engine}`)
+
+ for (const text of FAITHFUL) {
+ const started = client.next(isTurn('turn_started', agent.id), 45_000, `turn_started (${text})`)
+ const ended = client.next(isTurn('turn_ended', agent.id), 45_000, `turn_ended (${text})`)
+ client.send('message', { agentId: agent.id, content: text })
+ expect((await started).payload?.userMessage, 'arrives exactly as sent').toBe(text)
+ await ended
+ }
+ for (const [what, text] of HOSTILE) {
+ const from = client.frames.length
+ client.send('message', { agentId: agent.id, content: text })
+ await client.waitFor((frame) => isTurn('turn_ended', agent.id)(frame) && client.frames.indexOf(frame) >= from, 45_000, `the turn for ${what}`)
+ // Longer than two reconcile passes, so an engine that typing made exit is seen to be gone.
+ await sleep(12_000)
+ expect(engines(d), `${what}: the engine process is still running`).toBe(1)
+ const starts = client.frames.slice(from).filter(isTurn('turn_started', agent.id))
+ expect(starts, `${what}: one prompt, not two`).toHaveLength(1)
+ if (text.includes('!exit')) expect(String(starts[0].payload?.userMessage), `${what}: the whole message is the one prompt`).toContain('!exit')
+ const now = await row(client, agent.id)
+ expect(now?.status, `${what}: the engine is still there`).toBe('active')
+ expect(now?.engine, `${what}: the pane still runs the engine`).toBe(engine)
+ }
+ client.close()
+ }, 240_000)
+})
diff --git a/cli/e2e/harness/fakeEngine.mjs b/cli/e2e/harness/fakeEngine.mjs
index bd56bebe9..59b3bea53 100644
--- a/cli/e2e/harness/fakeEngine.mjs
+++ b/cli/e2e/harness/fakeEngine.mjs
@@ -388,9 +388,15 @@ export async function run(engine, config) {
process.stdin.setEncoding('utf8')
let buffer = ''
let queue = Promise.resolve()
+ // Inside a bracketed paste a carriage return or newline is a newline in the prompt, as Ink and
+ // ratatui read it; only one typed outside a paste submits. The paste's markers say which.
+ let pasting = false
process.stdin.on('data', (chunk) => {
if (dialog) { dialogKeys(chunk); return }
- for (const part of chunk.replace(/\x1b\[20[01]~/g, '').split(/(\r|\n|\x03)/)) {
+ for (const part of chunk.split(/(\x1b\[20[01]~|\r|\n|\x03)/)) {
+ if (part === '\x1b[200~') { pasting = true; continue }
+ if (part === '\x1b[201~') { pasting = false; continue }
+ if (pasting && (part === '\r' || part === '\n')) { buffer += '\n'; continue }
if (part === '\x03') {
if (open) {
if (engine === 'claude') claude({ type: 'user', message: { role: 'user', content: [{ type: 'text', text: '[Request interrupted by user]' }] } })