Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

harnessd: a full disk loses nothing the daemon answered; rounds 18 and 19 (#768)

Deecd0c6510e5ed
diff --git a/cli/e2e/diskfull.e2e.ts b/cli/e2e/diskfull.e2e.ts
new file mode 100644
index 000000000..bcb51f8a2
--- /dev/null
+++ b/cli/e2e/diskfull.e2e.ts
@@ -0,0 +1,144 @@
+/**
+ * A full disk, for Claude Code and Codex: the daemon's data folder runs out of space while agents are
+ * at work, as a disk filled by logs did on 2026-10-04. The core must keep serving the agents it has,
+ * must say a write failed rather than pretend it held, must not restart over it, and once space is
+ * back must write again: what it is told after that survives a restart.
+ *
+ * The data folder lives on a small disk image of its own (macOS `hdiutil`, mounted where only this
+ * test sees it), so nothing else on the machine runs short. Opt-in, because it mounts a volume:
+ * `DISKFULL=1 npm run test:e2e -- diskfull`.
+ */
+import { execFileSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, openSync, closeSync, rmSync, writeFileSync, writeSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, afterEach, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+const ON = process.env.DISKFULL === '1' && process.platform === 'darwin'
+
+type Row = Record<string, any>
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+
+/** Writes until the volume refuses, and returns the file, to free the space again by removing it. */
+function fill(volume: string): string {
+ const filler = join(volume, 'filler')
+ const fd = openSync(filler, 'w')
+ const chunk = Buffer.alloc(1024 * 1024, 7)
+ try {
+ for (;;) writeSync(fd, chunk)
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== 'ENOSPC') throw error
+ } finally {
+ closeSync(fd)
+ }
+ // The last few blocks a metadata write may still find: take those too.
+ const tail = join(volume, 'filler-tail')
+ try {
+ const small = openSync(tail, 'w')
+ try { for (;;) writeSync(small, Buffer.alloc(4096, 7)) } catch { /* full */ } finally { closeSync(small) }
+ } catch { /* already full */ }
+ return filler
+}
+
+describe.skipIf(!ON)('a full disk', () => {
+ let scratch = ''
+ let volume = ''
+ let daemon: IsolatedDaemon | undefined
+
+ beforeAll(() => {
+ // Short names: the daemon's socket lives in the data folder, and a Unix socket's path has a
+ // 104-byte limit on macOS that a long temporary folder alone nearly uses up.
+ scratch = mkdtempSync(join(tmpdir(), 'hd-'))
+ volume = join(scratch, 'v')
+ mkdirSync(volume)
+ const image = join(scratch, 'data.dmg')
+ execFileSync('hdiutil', ['create', '-size', '64m', '-fs', 'HFS+', '-volname', 'hdtest', '-type', 'UDIF', image], { stdio: 'pipe' })
+ execFileSync('hdiutil', ['attach', image, '-mountpoint', volume, '-nobrowse', '-noverify', '-noautoopen'], { stdio: 'pipe' })
+ }, 120_000)
+
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ afterAll(() => {
+ if (volume) { try { execFileSync('hdiutil', ['detach', volume, '-force'], { stdio: 'pipe' }) } catch { /* already gone */ } }
+ if (scratch) rmSync(scratch, { recursive: true, force: true })
+ })
+
+ it('agents keep working while the data folder is full, a failed write says so, and once there is room again it holds', async () => {
+ const data = join(volume, 'd')
+ mkdirSync(data, { recursive: true })
+ const d = await IsolatedDaemon.create({ dataDir: data })
+ daemon = d
+ onTestFailed(() => {
+ console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`)
+ // The runner swallows console output: DISKFULL_LOG=<file> keeps the daemon's whole log for reading.
+ if (process.env.DISKFULL_LOG) writeFileSync(process.env.DISKFULL_LOG, d.log())
+ })
+ await d.start()
+ let client = await LocalClient.connect(d)
+ const agents: Row[] = []
+ for (const engine of ['claude', 'codex'] as const) {
+ const cwd = join(d.projectsDir, `full-${engine}`)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, JSON.stringify(created)).toBeUndefined()
+ agents.push(await until(`${engine} to bind`, async () => {
+ const now = await row(client, created.agent.id)
+ return now?.sessionId && now.status === 'active' ? now : null
+ }, 60_000, 500))
+ }
+ for (const agent of agents) await turn(client, agent.id, `before the disk filled (${agent.engine})`)
+
+ // The disk fills.
+ const filler = fill(volume)
+ // The agents it has keep working: their turns, their rows.
+ for (const agent of agents) await turn(client, agent.id, `while the disk is full (${agent.engine})`)
+ // A rename is answered, though it cannot be written down yet.
+ const renamed = await client.request('agent_update', { agentId: agents[0].id, name: 'Named while full' }, 30_000)
+ if (process.env.DISKFULL_REPORT) writeFileSync(process.env.DISKFULL_REPORT, `rename while full: ${JSON.stringify(renamed.error ?? renamed.agent?.name)}\n`)
+ expect(renamed.error, JSON.stringify(renamed)).toBeUndefined()
+ // A new agent binds and works: the record it would resume from cannot be saved, and that must not
+ // keep the windows from hearing it bound.
+ const cwd = join(d.projectsDir, 'made-while-full')
+ mkdirSync(cwd, { recursive: true })
+ const made = await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 90_000)
+ expect(made.error, JSON.stringify(made)).toBeUndefined()
+ const synced = client.waitFor((frame) => frame.type === 'session_synced' && frame.payload?.agentId === made.agent.id, 60_000, 'session_synced for the new agent')
+ const madeRow = await until('the agent made while full to bind', async () => {
+ const now = await row(client, made.agent.id)
+ return now?.sessionId && now.status === 'active' ? now : null
+ }, 60_000, 500)
+ await synced
+ await turn(client, madeRow.id, 'made while the disk was full')
+ expect(d.coresStarted()).toBe(1)
+
+ // Space comes back. With no further change asked of it, what the daemon answered while full is
+ // written down within a pass or two, and survives a restart.
+ rmSync(filler, { force: true })
+ rmSync(join(volume, 'filler-tail'), { force: true })
+ await new Promise((resolve) => setTimeout(resolve, 15_000))
+ client.close()
+ await d.restart()
+ client = await LocalClient.connect(d)
+ const everyone = [...agents, madeRow]
+ const after = await until('every agent back after the restart', async () => {
+ const now = await rows(client)
+ return everyone.every((agent) => now.some((one) => one.id === agent.id && one.status === 'active')) ? now : null
+ }, 90_000, 500)
+ expect(after.find((one) => one.id === agents[0].id)?.name).toBe('Named while full')
+ for (const agent of everyone) await turn(client, agent.id, `after the space came back (${agent.engine})`)
+ if (process.env.DISKFULL_LOG) writeFileSync(process.env.DISKFULL_LOG, d.log())
+ client.close()
+ }, 600_000)
+})
diff --git a/cli/e2e/folders.e2e.ts b/cli/e2e/folders.e2e.ts
new file mode 100644
index 000000000..aae3a08b8
--- /dev/null
+++ b/cli/e2e/folders.e2e.ts
@@ -0,0 +1,130 @@
+/**
+ * The folders agents work in, as people name and lose them, for Claude Code and Codex: names with
+ * spaces, quotes, a dollar sign, accents and emoji, which tmux, the shell and the engines' transcript
+ * folders each escape their own way; and a folder that disappears (a worktree swept, a project
+ * deleted) while its agent runs and while the daemon is down. An agent in any of them must be made,
+ * take turns, survive a restart and be closed; one whose folder is gone must not take the daemon or
+ * any other agent with it.
+ */
+import { mkdirSync, rmSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+async function create(client: LocalClient, engine: Engine, cwd: string): Promise<Row> {
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${cwd}: ${JSON.stringify(created)}`).toBeUndefined()
+ return until(`the agent in ${cwd} to bind its conversation`, async () => {
+ const agent = await row(client, created.agent.id)
+ return agent?.sessionId && agent.status === 'active' ? agent : null
+ }, 60_000, 500)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+
+// Names that each break a different layer when escaped wrong: the shell, tmux's -c, and the engines'
+// transcript folders, which Claude Code derives from the path.
+const AWKWARD = [
+ 'My Projects/with spaces',
+ "it's \"quoted\"",
+ 'cost $HOME and `ticks`',
+ 'ünïcødé çafé',
+ 'rocket 🚀 launch',
+]
+
+describe('the folders agents work in', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+ const fresh = async () => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })
+ await d.start()
+ return d
+ }
+
+ it.each(['claude', 'codex'] as const)('%s: a folder of any name holds an agent that works, survives a restart and is closed', async (engine) => {
+ const d = await fresh()
+ let client = await LocalClient.connect(d)
+ const agents: Row[] = []
+ for (const name of AWKWARD) agents.push(await create(client, engine, join(d.projectsDir, name)))
+ for (const [i, agent] of agents.entries()) {
+ expect(agent.project?.cwd ?? agent.cwd, AWKWARD[i]).toContain(AWKWARD[i].split('/').at(-1))
+ await turn(client, agent.id, `in ${AWKWARD[i]}`)
+ }
+ client.close()
+ await d.restart()
+ client = await LocalClient.connect(d)
+ for (const [i, agent] of agents.entries()) {
+ await until(`${AWKWARD[i]} back after the restart`, async () => {
+ const now = await row(client, agent.id)
+ return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null
+ }, 60_000, 500)
+ await turn(client, agent.id, `back in ${AWKWARD[i]}`)
+ }
+ for (const agent of agents) {
+ const now = (await row(client, agent.id))!
+ expect(await client.request('agent_close', { agentId: agent.id, sessionId: now.sessionId, createdAt: now.createdAt, mode: 'now' }, 60_000)).toMatchObject({ closed: true })
+ }
+ client.close()
+ })
+
+ it('a folder deleted under a running agent costs that agent at most: the daemon and the others go on', async () => {
+ const d = await fresh()
+ const client = await LocalClient.connect(d)
+ const doomed = await create(client, 'claude', join(d.projectsDir, 'swept-worktree'))
+ const other = await create(client, 'codex', join(d.projectsDir, 'kept-project'))
+ rmSync(join(d.projectsDir, 'swept-worktree'), { recursive: true, force: true })
+ // The engine is still running in its pane; what the apps read about its folder must not throw.
+ const listed = await rows(client)
+ expect(listed.some((agent) => agent.id === doomed.id)).toBe(true)
+ for (const type of ['git_project_info', 'project_preview', 'fs_list_dir'] as const) {
+ const answer = await client.request(type, { path: join(d.projectsDir, 'swept-worktree') }, 30_000)
+ expect(answer, type).toBeTruthy()
+ }
+ await turn(client, other.id, 'the other agent is fine')
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ })
+
+ it('a folder deleted while the daemon is down: the restart restores the rest, and says why it could not restore that one', async () => {
+ const d = await fresh()
+ let client = await LocalClient.connect(d)
+ const doomed = await create(client, 'claude', join(d.projectsDir, 'gone-by-morning'))
+ const kept = await create(client, 'codex', join(d.projectsDir, 'still-here'))
+ await turn(client, doomed.id, 'before its folder went')
+ client.close()
+ await d.stop()
+ // The machine is down; its agents' panes go with it, and one folder is deleted meanwhile.
+ await d.tmux.run('kill-server')
+ rmSync(join(d.projectsDir, 'gone-by-morning'), { recursive: true, force: true })
+ await d.start()
+ client = await LocalClient.connect(d)
+ await until('the agent whose folder is still there to come back', async () => {
+ const now = await row(client, kept.id)
+ return now?.status === 'active' && now.sessionId === kept.sessionId ? now : null
+ }, 60_000, 500)
+ await turn(client, kept.id, 'restored beside one that could not be')
+ // The one whose folder is gone is not shown as active, and the daemon did not restart over it.
+ await until('the agent whose folder is gone not to read as active', async () => {
+ const now = await row(client, doomed.id)
+ return !now || now.status !== 'active' || null
+ }, 60_000, 500)
+ expect(d.coresStarted()).toBe(2)
+ client.close()
+ })
+})
diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.ts
index 6742a5dca..426f3d06c 100644
--- a/cli/e2e/harness/daemon.ts
+++ b/cli/e2e/harness/daemon.ts
@@ -29,6 +29,9 @@ export interface DaemonOptions {
noMaster?: boolean
/** Run this bundle (an installed `cli.js`) instead of the checkout's source. */
scriptPath?: string
+ /** Keep the daemon's data folder here instead of under the throwaway root (a test volume). The fake
+ * engines read their hook credential from it, so it is set for them too. */
+ dataDir?: string
}
async function freePort(): Promise<number> {
@@ -70,7 +73,7 @@ export class IsolatedDaemon {
const root = tmux.root
const port = await freePort()
const dirs = {
- home: join(root, 'home'), data: join(root, 'data'), runtime: join(root, 'runtime'), auth: join(root, 'auth'),
+ home: join(root, 'home'), data: options.dataDir ?? join(root, 'data'), runtime: join(root, 'runtime'), auth: join(root, 'auth'),
bin: join(root, 'bin'), dsh: join(root, 'dsh'), claudeProjects: join(root, 'claude', 'projects'), codexHome: join(root, 'codex'),
projects: join(root, 'projects'),
}
diff --git a/cli/src/core/agents/bind.spec.ts b/cli/src/core/agents/bind.spec.ts
index 45d86423b..ffd7a3edc 100644
--- a/cli/src/core/agents/bind.spec.ts
+++ b/cli/src/core/agents/bind.spec.ts
@@ -96,6 +96,22 @@ describe('binding a registered session to its agent', () => {
})
})
+ it('still tells the app of a binding when the record it resumes from cannot be saved, as on a full disk', async () => {
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const run = setup()
+ const entry = agent({ boundAt: Date.parse('2026-10-04T10:00:00Z'), resumeOnly: true })
+ run.byAgent.set('a1', entry)
+ vi.mocked(run.deps.stoppedAgents.save).mockImplementationOnce(() => { throw Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' }) })
+ await run.binding.handleRegistered(entry, meta({ isNew: true }))
+ expect(error).toHaveBeenCalledWith('[agent] a1 could not save the record it resumes from: ENOSPC: no space left on device')
+ expect(run.deps.stoppedAgents.finishResume).toHaveBeenCalledWith('a1')
+ expect(run.deps.announceSession).toHaveBeenCalledWith(entry)
+ expect(run.deps.clients.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'session_synced' }))
+ vi.mocked(run.deps.stoppedAgents.save).mockImplementationOnce(() => { throw 'disk gone' })
+ await run.binding.handleRegistered(entry, meta({ isNew: true }))
+ expect(error).toHaveBeenLastCalledWith('[agent] a1 could not save the record it resumes from: disk gone')
+ })
+
it('stops after the attach for a session it already had, and finishes a resume it was waiting for', async () => {
const run = setup()
const entry = agent({ resumeOnly: true } as Partial<RegisteredSession>)
diff --git a/cli/src/core/agents/bind.ts b/cli/src/core/agents/bind.ts
index 332de9043..ecff132f4 100644
--- a/cli/src/core/agents/bind.ts
+++ b/cli/src/core/agents/bind.ts
@@ -163,7 +163,13 @@ export function createBinding({
}
const confirmed = registry.byAgent(entry.agentId)
if (confirmed?.sessionId === entry.sessionId) {
- stoppedAgents.save(confirmed)
+ // The record a stop is resumed from, kept current. Best effort: the binding has happened, and a
+ // full disk must not stop the windows hearing of it below (found end to end, e2e/diskfull.e2e.ts).
+ try {
+ stoppedAgents.save(confirmed)
+ } catch (error) {
+ console.error(`[agent] ${sid(entry.agentId)} could not save the record it resumes from: ${error instanceof Error ? error.message : error}`)
+ }
if (confirmed.resumeOnly) stoppedAgents.finishResume(confirmed.agentId)
}
syncRecapPool()
diff --git a/cli/src/lib/registry.spec.ts b/cli/src/lib/registry.spec.ts
index c3c096daf..916c19ef2 100644
--- a/cli/src/lib/registry.spec.ts
+++ b/cli/src/lib/registry.spec.ts
@@ -138,6 +138,24 @@ describe('registry remote display names', () => {
expect(statSync(join(dataDir, 'registry.json')).mode & 0o777).toBe(0o600)
})
+ it('tries a names write that failed again on the next save, so a rename survives a full disk', async () => {
+ const transcriptPath = join(dataDir, 'session-1.jsonl')
+ writeFileSync(transcriptPath, '{}\n')
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const { registry } = await loadRegistryModule()
+ registry.load()
+ registerProcess(registry, { launcherId: 'h1', sessionId: 'session-1', transcriptPath, tmuxPane: '%1', cwd: '/tmp/demo' })
+ // The write cannot land: a directory where the names file goes stands in for a full disk.
+ mkdirSync(join(dataDir, 'agent-names.json'))
+ expect(registry.rename('session-1', 'Named while full')).toBeTruthy()
+ expect(error.mock.calls.some(([line]) => String(line).includes('save names failed'))).toBe(true)
+ // Room again: the next save of anything writes the names too, with no second rename.
+ rmSync(join(dataDir, 'agent-names.json'), { recursive: true, force: true })
+ registry.remove('session-1')
+ expect(JSON.parse(readFileSync(join(dataDir, 'agent-names.json'), 'utf-8'))).toEqual({ 'session-1': 'Named while full' })
+ error.mockRestore()
+ })
+
it('auto-follows the tmux pane title until renamed, then the manual name stays fixed', async () => {
const transcriptPath = join(dataDir, 'session-title.jsonl')
writeFileSync(transcriptPath, '{}\n')
diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.ts
index 85bdacc78..86422ef72 100644
--- a/cli/src/lib/registry.ts
+++ b/cli/src/lib/registry.ts
@@ -2115,6 +2115,7 @@ class Registry {
console.error('[registry] save skipped because the loaded registry requires operator repair')
return
}
+ if (this.namesUnsaved) this.saveNames()
try {
secureStateDirectory(env.ADAPTER_DATA_DIR)
const currentRows = new Map(this.list().map((entry) => {
@@ -2228,6 +2229,14 @@ class Registry {
}
}
+ /**
+ * A names write that failed, tried again on every save until it lands. The rows heal on their own,
+ * because the next save sees them differ from what is on disk; names are written only on a rename,
+ * so a rename the daemon answered while the disk was full was lost to a restart that came before the
+ * next rename (found end to end, e2e/diskfull.e2e.ts).
+ */
+ private namesUnsaved = false
+
private saveNames(): void {
try {
secureStateDirectory(env.ADAPTER_DATA_DIR)
@@ -2239,7 +2248,9 @@ class Registry {
// no file yet / unreadable — write the in-memory overrides
}
atomicWriteJson(NAMES_FILE, { ...existing, ...Object.fromEntries(NAME_OVERRIDES) })
+ this.namesUnsaved = false
} catch (err) {
+ this.namesUnsaved = true
console.error('[registry] save names failed:', err)
}
}