The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(swarm): a worktree git cannot fully remove keeps its record (#2417)
harness-for-autonomous[bot]75b946bbd69d
diff --git a/cli/src/lib/worktreeSweep.spec.ts b/cli/src/lib/worktreeSweep.spec.tsindex 31e262350..207463f9d 100644--- a/cli/src/lib/worktreeSweep.spec.ts+++ b/cli/src/lib/worktreeSweep.spec.ts@@ -1,12 +1,18 @@import { execFile } from 'node:child_process'-import { mkdir, mkdtemp, readFile, rm, stat, symlink, utimes, writeFile } from 'node:fs/promises'+import { chmod, mkdir, mkdtemp, readFile, rm, stat, symlink, utimes, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import { promisify } from 'node:util'-import { afterEach, beforeEach, describe, expect, it } from 'vitest'-import { sweepWorktrees } from './worktreeSweep.js'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { sweepOne, sweepWorktrees, WorktreeKept } from './worktreeSweep.js'const exec = promisify(execFile)+/** Folders the sweep's own check reads as writable whatever their mode. */+const writable = vi.hoisted(() => ({ folders: new Set<string>() }))+vi.mock('node:fs/promises', async importOriginal => {+ const fs = await importOriginal<typeof import('node:fs/promises')>()+ return { ...fs, access: (file: string, mode?: number) => writable.folders.has(file) ? Promise.resolve() : fs.access(file, mode) }+})// Real Git, several worktrees a test: slower than the default 5s under a full, parallel run.describe('sweeping unused worktrees', { timeout: 30_000 }, () => {let root: string, repo: string, home: string@@ -140,6 +146,109 @@ describe('sweeping unused worktrees', { timeout: 30_000 }, () => {expect(await readFile(join(outside, '.gitignore'), 'utf8')).toBe(OLDER)})+ // One Mac, 10-28: a stopped worker's worktree held a skill folder its agent had copied in read-only. git deleted a third of+ // the folder, stopped there, and deleted the worktree's record anyway, so git and gh failed in what was left.+ // Folder permissions do not stop root or Windows from deleting.+ const permissions = process.platform !== 'win32' && process.getuid?.() !== 0+ const withReadOnlyFolder = async (path: string) => {+ await withHarnessFolder(path, '/*\n')+ const folder = join(path, '.harness', 'tools', 'skills', 'copied')+ await mkdir(folder, { recursive: true })+ await writeFile(join(folder, 'SKILL.md'), 'copied in read-only')+ await chmod(folder, 0o555)+ return folder+ }++ it.runIf(permissions)('keeps, untouched and still a worktree, one holding a folder git could not empty', async () => {+ const path = await add('read-only', 'harness/read-only')+ const folder = await withReadOnlyFolder(path)+ try {+ expect(await sweepWorktrees({ root: join(root, 'harnesses'), inUse: [], now: later() })).toEqual([])+ expect(await exists(join(path, 'file')), 'nothing in it was deleted').toBe(true)+ expect(await git(path, 'status', '--porcelain')).toBe('')+ expect(await git(path, 'rev-parse', '--git-dir')).toContain('/.git/worktrees/')+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_UNWRITABLE'))+ } finally { await chmod(folder, 0o755) }+ })++ it.runIf(permissions)('puts the record back when git began removing the worktree and could not finish', async () => {+ const path = await add('half-removed', 'harness/half-removed')+ // A folder that turns read-only while the worktree is removed: the check above reads it as writable.+ await withHarnessFolder(path, '/*\n')+ const folder = join(path, '.harness', 'late')+ await mkdir(folder)+ await writeFile(join(folder, 'held'), 'held')+ await chmod(folder, 0o555)+ writable.folders.add(folder)+ try {+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD'), 'git still works in what is left').toBe('harness/half-removed')+ expect(await git(path, 'rev-parse', '--git-dir')).toContain('/.git/worktrees/')+ expect(await branches()).toContain('harness/half-removed')+ } finally { writable.folders.clear(); await chmod(folder, 0o755) }+ })++ // Security review 1 of #2417: git empties the folder in directory order, so its .git file often went before git reached+ // what it could not delete (8 of 30 trials), and the record alone written back left nothing pointing to it.+ it.runIf(permissions)('puts the .git file and the record back whichever folder git stops at', async () => {+ const held: string[] = []+ // Beside .git at the top, where the order of a folder's entries decides which goes first (on this Mac's APFS, stop-long+ // came before .git; on ext4 the order is the filesystem's own); ignored, so nothing reads as work.+ await writeFile(join(repo, '.git', 'info', 'exclude'), '/stop-*/\n')+ try {+ for (const name of ['stop-a', 'stop-g', 'stop-m', 'stop-s', 'stop-z', 'stop-0', 'stop-9', 'stop-_', 'stop-~', 'stop-long']) {+ const path = await add(`stops-${held.length}`, `harness/stops-${held.length}`)+ const folder = join(path, name)+ await mkdir(folder, { recursive: true })+ await writeFile(join(folder, 'held'), 'held')+ await chmod(folder, 0o555)+ held.push(folder); writable.folders.add(folder)+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD'), `git still works when it stopped at ${name}`).toBe(`harness/stops-${held.length - 1}`)+ }+ } finally { writable.folders.clear(); for (const folder of held) await chmod(folder, 0o755) }+ })++ // The same review: a locked file passes a check of folder permissions, and git stops at it.+ it.runIf(process.platform === 'darwin' && process.getuid?.() !== 0)('keeps a worktree, still one, when git stops at a locked file', async () => {+ const path = await add('locked', 'harness/locked')+ await mkdir(join(path, 'src'))+ await writeFile(join(path, 'src', 'kept'), 'locked')+ await git(path, 'add', '.')+ await git(path, 'commit', '--quiet', '-m', 'locked file')+ await exec('chflags', ['uchg', join(path, 'src', 'kept')])+ try {+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD')).toBe('harness/locked')+ expect(await branches()).toContain('harness/locked')+ } finally { await exec('chflags', ['nouchg', join(path, 'src', 'kept')]) }+ })++ // Security review 2 of #2417: the record also holds the refs only this worktree has.+ it.runIf(permissions)('writes back the refs only the worktree held with its record', async () => {+ const path = await add('own-refs', 'harness/own-refs')+ await git(path, 'update-ref', 'refs/worktree/mark', 'HEAD')+ await git(path, 'update-ref', 'refs/bisect/bad', 'HEAD')+ const before = await git(path, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/worktree', 'refs/bisect')+ await withHarnessFolder(path, '/*\n')+ const folder = join(path, '.harness', 'late')+ await mkdir(folder)+ await writeFile(join(folder, 'held'), 'held')+ await chmod(folder, 0o555)+ writable.folders.add(folder)+ try {+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))+ expect(await git(path, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/worktree', 'refs/bisect')).toBe(before)+ } finally { writable.folders.clear(); await chmod(folder, 0o755) }+ })++ it('leaves a worktree alone that came into use while it was being checked', async () => {+ const path = await add('opened', 'harness/opened')+ expect(await sweepOne(path, later(), 0, () => false)).toBe(false)+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD')).toBe('harness/opened')+ expect(await exists(join(path, 'file'))).toBe(true)+ })+it('ignores a missing worktrees folder and checkouts that are not worktrees', async () => {expect(await sweepWorktrees({ root: join(root, 'nothing'), inUse: [] })).toEqual([])await mkdir(join(home, 'app', 'plain'), { recursive: true })diff --git a/cli/src/lib/worktreeSweep.ts b/cli/src/lib/worktreeSweep.tsindex 742a026e0..958975798 100644--- a/cli/src/lib/worktreeSweep.ts+++ b/cli/src/lib/worktreeSweep.ts@@ -1,6 +1,7 @@import { execFile } from 'node:child_process'-import { lstat, readdir, rmdir, stat } from 'node:fs/promises'-import { basename, dirname, join, normalize, resolve, sep } from 'node:path'+import { constants, type Dirent } from 'node:fs'+import { access, lstat, mkdir, readdir, readFile, rmdir, stat, writeFile } from 'node:fs/promises'+import { basename, dirname, join, normalize, relative, resolve, sep } from 'node:path'import { promisify } from 'node:util'import { upgradeOlderRuntimeIgnore } from './harnessRuntimeIgnore.js'@@ -68,9 +69,64 @@ export async function sweepWorktrees(input: {return removed}+/** Why a worktree that passed every check above was kept anyway, by a fixed code: an error's own text can carry its path.+ * UNWRITABLE: a folder in it git could not empty, so nothing was touched. NOT_REMOVED: git began and failed; whatever is+ * left keeps its record, so git still works there. */+export class WorktreeKept extends Error {+ constructor(readonly code: 'WORKTREE_UNWRITABLE' | 'WORKTREE_NOT_REMOVED') { super(code) }+}++/** Whether a folder in the worktree cannot be emptied: git would delete part of it and stop (below). One Mac, 10-28: a+ * skill folder an agent had copied in read-only stopped the removal of a stopped worker's worktree after a third of it. */+async function unwritableFolder(dir: string): Promise<boolean> {+ try { await access(dir, constants.W_OK | constants.X_OK) } catch { return true }+ let entries: Dirent[]+ try { entries = await readdir(dir, { withFileTypes: true }) } catch { return true }+ for (const entry of entries) if (entry.isDirectory() && await unwritableFolder(join(dir, entry.name))) return true+ return false+}++/**+ * `git worktree remove`, without losing the worktree when it fails: git deletes the worktree's record in the repository even+ * when it could not delete its folder ("there's no going back from here", builtin/worktree.c), so the folder's .git then+ * points to nothing and every git and gh command there fails. A stopped worker's folder on one Mac (10-28) was left like that,+ * and a Buddy's on another with the same symptom (10-30). git empties the folder in directory order, so its .git file is often+ * gone too before git reaches what it cannot delete (security review 1 of #2417: 8 of 30 trials). Both are read first and,+ * if the folder is still there, each is written back where it is missing: the record only where git keeps records, the+ * .git file never over one that is there.+ */+async function removeKeepingRecord(main: string, common: string, gitDir: string, path: string): Promise<void> {+ // The whole record: its HEAD, index and reflog, and the refs only this worktree holds (refs/bisect, refs/worktree; security+ // review 2 of #2417).+ const saved = new Map<string, Buffer>()+ const save = async (dir: string): Promise<void> => {+ for (const entry of await readdir(dir, { withFileTypes: true })) {+ const file = join(dir, entry.name)+ if (entry.isDirectory()) await save(file)+ else if (entry.isFile()) saved.set(relative(gitDir, file), await readFile(file))+ }+ }+ await save(gitDir)+ const link = await readFile(join(path, '.git'))+ try {+ await git(main, ['worktree', 'remove', '--', path])+ } catch (error) {+ if (!await lstat(path).then(info => info.isDirectory(), () => false)) throw error+ await writeFile(join(path, '.git'), link, { flag: 'wx' }).catch((failed: NodeJS.ErrnoException) => { if (failed.code !== 'EEXIST') throw failed })+ if (dirname(gitDir) === join(common, 'worktrees') && !await lstat(gitDir).then(() => true, () => false)) {+ for (const [name, data] of saved) {+ await mkdir(dirname(join(gitDir, name)), { recursive: true })+ await writeFile(join(gitDir, name), data)+ }+ }+ throw new WorktreeKept('WORKTREE_NOT_REMOVED')+ }+}+/** Removes one worktree under the rule above: true when it went. The Swarm's host calls it with no idle wait for a worker- * it just stopped (superTabs/machines.ts `removeStoppedWorktree`). */-export async function sweepOne(path: string, now: number, idleMs: number): Promise<boolean> {+ * it just stopped (superTabs/machines.ts `removeStoppedWorktree`), and asks [unused] again just before git removes it: the+ * reads and the folder walk take a while, and a session may open there meanwhile (security review 1 of #2417). */+export async function sweepOne(path: string, now: number, idleMs: number, unused: () => boolean = () => true): Promise<boolean> {const gitDir = resolve(path, await git(path, ['rev-parse', '--git-dir']))const common = resolve(path, await git(path, ['rev-parse', '--git-common-dir']))if (gitDir === common || basename(common) !== '.git') return false@@ -85,7 +141,9 @@ export async function sweepOne(path: string, now: number, idleMs: number): Promiif (await git(path, ['status', '--porcelain']) !== '') return falseconst branch = await git(path, ['symbolic-ref', '--quiet', 'HEAD']).catch(() => null)if (!branch && await git(path, ['rev-list', '--count', 'HEAD', '--not', '--branches', '--remotes']) !== '0') return false- await git(main, ['worktree', 'remove', '--', path])+ if (await unwritableFolder(path)) throw new WorktreeKept('WORKTREE_UNWRITABLE')+ if (!unused()) return false+ await removeKeepingRecord(main, common, gitDir, path)const name = branch?.slice('refs/heads/'.length)// Only a branch Harness made: marked in the repository's config, or named `harness/…` by older builds.const made = name !== undefined && (name.startsWith('harness/')diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.tsindex ad0779f5a..d424ae9e7 100644--- a/cli/src/superTabs/machines.spec.ts+++ b/cli/src/superTabs/machines.spec.ts@@ -1,5 +1,5 @@import { execFileSync } from 'node:child_process'-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'@@ -1844,6 +1844,29 @@ describe('removed machine cleanup', () => {expect(m.log.mock.calls.flat().join('\n')).not.toContain(m.recorded)})+ // One Mac, 10-28: git removed a third of a stopped worker's worktree, stopped at a skill folder copied in read-only, and+ // deleted the worktree's record anyway, so git and gh failed in the rest. Folder permissions do not stop root or Windows.+ it.runIf(process.platform !== 'win32' && process.getuid?.() !== 0)('keeps a stopped worker\'s worktree that git could not empty, still a worktree, and says why', async () => {+ const m = machine()+ rmSync(join(m.recorded, 'unpushed.txt'))+ mkdirSync(join(m.recorded, '.harness', 'skills', 'copied'), { recursive: true })+ writeFileSync(join(m.recorded, '.harness', '.gitignore'), '/*\n')+ writeFileSync(join(m.recorded, '.harness', 'skills', 'copied', 'SKILL.md'), 'copied in read-only')+ chmodSync(join(m.recorded, '.harness', 'skills', 'copied'), 0o555)+ try {+ // A Buddy's or an agent's live session in it: nothing is read or touched.+ m.setLive([session('a-buddy', join(m.recorded, '.harness'))])+ await m.open().removeStoppedWorktree('tab', 'h-recorded')+ expect(m.log).not.toHaveBeenCalled()+ m.setLive([])+ await m.open().removeStoppedWorktree('tab', 'h-recorded')+ expect(m.log.mock.calls).toEqual([['Kept a Swarm worktree: WORKTREE_UNWRITABLE']])+ expect(startIds(m)).toEqual(['recorded', 'kept'])+ expect(git(m.recorded, 'status', '--porcelain')).toBe('')+ expect(git(m.clone, 'worktree', 'list', '--porcelain')).toContain('/recorded\n')+ } finally { chmodSync(join(m.recorded, '.harness', 'skills', 'copied'), 0o755) }+ })+it('keeps the record when its folder cannot be read for any reason but being gone', async () => {const m = machine()m.setLive([])@@ -1878,8 +1901,9 @@ describe('removed machine cleanup', () => {await Promise.all([host.removeStoppedWorktree('tab', 'h-recorded'), host.removeStoppedWorktree('tab', 'h-recorded')])expect(existsSync(m.recorded)).toBe(false)expect(startIds(m)).toEqual(['kept'])- // Only the first looked for sessions in the folder; the second came once it was gone.- expect(folderAtCheck).toEqual([true])+ // Only the first looked for sessions in the folder, before its checks and again just before git removed it; the second+ // came once it was gone.+ expect(folderAtCheck).toEqual([true, true])expect(m.log).not.toHaveBeenCalled()})diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.tsindex 7cf88bb9e..63747c070 100644--- a/cli/src/superTabs/machines.ts+++ b/cli/src/superTabs/machines.ts@@ -15,7 +15,7 @@ import { startAgent, type ScopedRequest } from './start.js'import { PrepareRetryError, prepareSwarmStart } from './projectFolder.js'import { readAuthSession } from '../lib/authSession.js'import { createSwarmCloneTrustRepair, swarmCloneRoots } from './cloneTrust.js'-import { git, sweepOne } from '../lib/worktreeSweep.js'+import { git, sweepOne, WorktreeKept } from '../lib/worktreeSweep.js'import { projectsRoot } from '../lib/projectFolder.js'import { worktreeFolderName } from '../lib/agentNames.js'import { teamPrivacyObserver } from '../team/observer.js'@@ -911,13 +911,18 @@ export class ContributedMachines {this.log('Kept a Swarm worktree: WORKTREE_UNPROVEN')return}- if (this.core.agents.live().some(a => !!a.cwd && within(record.path, a.cwd))) return// Review of #2266: like the weekly sweep (services/workspaces.ts), a paused session of another agent can be resumed- // there, so its folder stays. This agent's own paused session is not kept: it was stopped or removed.- if (this.core.agents.all().some(a => a.agentId !== harnessId && !!a.cwd && within(record.path, a.cwd))) return+ // there, so its folder stays. This agent's own paused session is not kept: it was stopped or removed. Asked again just+ // before git removes it (lib/worktreeSweep.ts sweepOne).+ const unused = () => !this.core.agents.live().some(a => !!a.cwd && within(record.path, a.cwd))+ && !this.core.agents.all().some(a => a.agentId !== harnessId && !!a.cwd && within(record.path, a.cwd))+ if (!unused()) return// File times are wall-clock, so the sweep's idle check reads the real clock; no idle wait is asked.- // A failed git read (a locked worktree, a handle still open on Windows) keeps the folder; the log names no path.- if (!await sweepOne(record.path, Date.now(), 0).catch(() => { this.log('Kept a Swarm worktree: WORKTREE_CHECK_FAILED'); return false })) return+ // A failed git read (a locked worktree, a handle still open on Windows) keeps the folder; the log names no path. A folder git+ // could not empty keeps its record and this one, and says which kind it was (lib/worktreeSweep.ts WorktreeKept).+ if (!await sweepOne(record.path, Date.now(), 0, unused).catch(error => {+ this.log(`Kept a Swarm worktree: ${error instanceof WorktreeKept ? error.code : 'WORKTREE_CHECK_FAILED'}`); return false+ })) returnwriteWorktrees(file, readWorktrees(file).filter(w => !mine(w)))}