Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(swarm): a worktree git cannot fully remove keeps its record (#2417)

harness-for-autonomous[bot]75b946bbd69d
diff --git a/cli/src/lib/worktreeSweep.spec.ts b/cli/src/lib/worktreeSweep.spec.ts
index 31e262350..207463f9d 100644
--- a/cli/src/lib/worktreeSweep.spec.ts
+++ b/cli/src/lib/worktreeSweep.spec.ts
@@ -1,12 +1,18 @@
import { execFile } from 'node:child_process'
-import { mkdir, mkdtemp, readFile, rm, stat, symlink, utimes, writeFile } from 'node:fs/promises'
+import { chmod, mkdir, mkdtemp, readFile, rm, stat, symlink, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { promisify } from 'node:util'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { sweepWorktrees } from './worktreeSweep.js'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { sweepOne, sweepWorktrees, WorktreeKept } from './worktreeSweep.js'
const exec = promisify(execFile)
+/** Folders the sweep's own check reads as writable whatever their mode. */
+const writable = vi.hoisted(() => ({ folders: new Set<string>() }))
+vi.mock('node:fs/promises', async importOriginal => {
+ const fs = await importOriginal<typeof import('node:fs/promises')>()
+ return { ...fs, access: (file: string, mode?: number) => writable.folders.has(file) ? Promise.resolve() : fs.access(file, mode) }
+})
// Real Git, several worktrees a test: slower than the default 5s under a full, parallel run.
describe('sweeping unused worktrees', { timeout: 30_000 }, () => {
let root: string, repo: string, home: string
@@ -140,6 +146,109 @@ describe('sweeping unused worktrees', { timeout: 30_000 }, () => {
expect(await readFile(join(outside, '.gitignore'), 'utf8')).toBe(OLDER)
})
+ // One Mac, 10-28: a stopped worker's worktree held a skill folder its agent had copied in read-only. git deleted a third of
+ // the folder, stopped there, and deleted the worktree's record anyway, so git and gh failed in what was left.
+ // Folder permissions do not stop root or Windows from deleting.
+ const permissions = process.platform !== 'win32' && process.getuid?.() !== 0
+ const withReadOnlyFolder = async (path: string) => {
+ await withHarnessFolder(path, '/*\n')
+ const folder = join(path, '.harness', 'tools', 'skills', 'copied')
+ await mkdir(folder, { recursive: true })
+ await writeFile(join(folder, 'SKILL.md'), 'copied in read-only')
+ await chmod(folder, 0o555)
+ return folder
+ }
+
+ it.runIf(permissions)('keeps, untouched and still a worktree, one holding a folder git could not empty', async () => {
+ const path = await add('read-only', 'harness/read-only')
+ const folder = await withReadOnlyFolder(path)
+ try {
+ expect(await sweepWorktrees({ root: join(root, 'harnesses'), inUse: [], now: later() })).toEqual([])
+ expect(await exists(join(path, 'file')), 'nothing in it was deleted').toBe(true)
+ expect(await git(path, 'status', '--porcelain')).toBe('')
+ expect(await git(path, 'rev-parse', '--git-dir')).toContain('/.git/worktrees/')
+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_UNWRITABLE'))
+ } finally { await chmod(folder, 0o755) }
+ })
+
+ it.runIf(permissions)('puts the record back when git began removing the worktree and could not finish', async () => {
+ const path = await add('half-removed', 'harness/half-removed')
+ // A folder that turns read-only while the worktree is removed: the check above reads it as writable.
+ await withHarnessFolder(path, '/*\n')
+ const folder = join(path, '.harness', 'late')
+ await mkdir(folder)
+ await writeFile(join(folder, 'held'), 'held')
+ await chmod(folder, 0o555)
+ writable.folders.add(folder)
+ try {
+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))
+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD'), 'git still works in what is left').toBe('harness/half-removed')
+ expect(await git(path, 'rev-parse', '--git-dir')).toContain('/.git/worktrees/')
+ expect(await branches()).toContain('harness/half-removed')
+ } finally { writable.folders.clear(); await chmod(folder, 0o755) }
+ })
+
+ // Security review 1 of #2417: git empties the folder in directory order, so its .git file often went before git reached
+ // what it could not delete (8 of 30 trials), and the record alone written back left nothing pointing to it.
+ it.runIf(permissions)('puts the .git file and the record back whichever folder git stops at', async () => {
+ const held: string[] = []
+ // Beside .git at the top, where the order of a folder's entries decides which goes first (on this Mac's APFS, stop-long
+ // came before .git; on ext4 the order is the filesystem's own); ignored, so nothing reads as work.
+ await writeFile(join(repo, '.git', 'info', 'exclude'), '/stop-*/\n')
+ try {
+ for (const name of ['stop-a', 'stop-g', 'stop-m', 'stop-s', 'stop-z', 'stop-0', 'stop-9', 'stop-_', 'stop-~', 'stop-long']) {
+ const path = await add(`stops-${held.length}`, `harness/stops-${held.length}`)
+ const folder = join(path, name)
+ await mkdir(folder, { recursive: true })
+ await writeFile(join(folder, 'held'), 'held')
+ await chmod(folder, 0o555)
+ held.push(folder); writable.folders.add(folder)
+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))
+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD'), `git still works when it stopped at ${name}`).toBe(`harness/stops-${held.length - 1}`)
+ }
+ } finally { writable.folders.clear(); for (const folder of held) await chmod(folder, 0o755) }
+ })
+
+ // The same review: a locked file passes a check of folder permissions, and git stops at it.
+ it.runIf(process.platform === 'darwin' && process.getuid?.() !== 0)('keeps a worktree, still one, when git stops at a locked file', async () => {
+ const path = await add('locked', 'harness/locked')
+ await mkdir(join(path, 'src'))
+ await writeFile(join(path, 'src', 'kept'), 'locked')
+ await git(path, 'add', '.')
+ await git(path, 'commit', '--quiet', '-m', 'locked file')
+ await exec('chflags', ['uchg', join(path, 'src', 'kept')])
+ try {
+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))
+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD')).toBe('harness/locked')
+ expect(await branches()).toContain('harness/locked')
+ } finally { await exec('chflags', ['nouchg', join(path, 'src', 'kept')]) }
+ })
+
+ // Security review 2 of #2417: the record also holds the refs only this worktree has.
+ it.runIf(permissions)('writes back the refs only the worktree held with its record', async () => {
+ const path = await add('own-refs', 'harness/own-refs')
+ await git(path, 'update-ref', 'refs/worktree/mark', 'HEAD')
+ await git(path, 'update-ref', 'refs/bisect/bad', 'HEAD')
+ const before = await git(path, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/worktree', 'refs/bisect')
+ await withHarnessFolder(path, '/*\n')
+ const folder = join(path, '.harness', 'late')
+ await mkdir(folder)
+ await writeFile(join(folder, 'held'), 'held')
+ await chmod(folder, 0o555)
+ writable.folders.add(folder)
+ try {
+ await expect(sweepOne(path, later(), 0)).rejects.toEqual(new WorktreeKept('WORKTREE_NOT_REMOVED'))
+ expect(await git(path, 'for-each-ref', '--format=%(refname) %(objectname)', 'refs/worktree', 'refs/bisect')).toBe(before)
+ } finally { writable.folders.clear(); await chmod(folder, 0o755) }
+ })
+
+ it('leaves a worktree alone that came into use while it was being checked', async () => {
+ const path = await add('opened', 'harness/opened')
+ expect(await sweepOne(path, later(), 0, () => false)).toBe(false)
+ expect(await git(path, 'rev-parse', '--abbrev-ref', 'HEAD')).toBe('harness/opened')
+ expect(await exists(join(path, 'file'))).toBe(true)
+ })
+
it('ignores a missing worktrees folder and checkouts that are not worktrees', async () => {
expect(await sweepWorktrees({ root: join(root, 'nothing'), inUse: [] })).toEqual([])
await mkdir(join(home, 'app', 'plain'), { recursive: true })
diff --git a/cli/src/lib/worktreeSweep.ts b/cli/src/lib/worktreeSweep.ts
index 742a026e0..958975798 100644
--- a/cli/src/lib/worktreeSweep.ts
+++ b/cli/src/lib/worktreeSweep.ts
@@ -1,6 +1,7 @@
import { execFile } from 'node:child_process'
-import { lstat, readdir, rmdir, stat } from 'node:fs/promises'
-import { basename, dirname, join, normalize, resolve, sep } from 'node:path'
+import { constants, type Dirent } from 'node:fs'
+import { access, lstat, mkdir, readdir, readFile, rmdir, stat, writeFile } from 'node:fs/promises'
+import { basename, dirname, join, normalize, relative, resolve, sep } from 'node:path'
import { promisify } from 'node:util'
import { upgradeOlderRuntimeIgnore } from './harnessRuntimeIgnore.js'
@@ -68,9 +69,64 @@ export async function sweepWorktrees(input: {
return removed
}
+/** Why a worktree that passed every check above was kept anyway, by a fixed code: an error's own text can carry its path.
+ * UNWRITABLE: a folder in it git could not empty, so nothing was touched. NOT_REMOVED: git began and failed; whatever is
+ * left keeps its record, so git still works there. */
+export class WorktreeKept extends Error {
+ constructor(readonly code: 'WORKTREE_UNWRITABLE' | 'WORKTREE_NOT_REMOVED') { super(code) }
+}
+
+/** Whether a folder in the worktree cannot be emptied: git would delete part of it and stop (below). One Mac, 10-28: a
+ * skill folder an agent had copied in read-only stopped the removal of a stopped worker's worktree after a third of it. */
+async function unwritableFolder(dir: string): Promise<boolean> {
+ try { await access(dir, constants.W_OK | constants.X_OK) } catch { return true }
+ let entries: Dirent[]
+ try { entries = await readdir(dir, { withFileTypes: true }) } catch { return true }
+ for (const entry of entries) if (entry.isDirectory() && await unwritableFolder(join(dir, entry.name))) return true
+ return false
+}
+
+/**
+ * `git worktree remove`, without losing the worktree when it fails: git deletes the worktree's record in the repository even
+ * when it could not delete its folder ("there's no going back from here", builtin/worktree.c), so the folder's .git then
+ * points to nothing and every git and gh command there fails. A stopped worker's folder on one Mac (10-28) was left like that,
+ * and a Buddy's on another with the same symptom (10-30). git empties the folder in directory order, so its .git file is often
+ * gone too before git reaches what it cannot delete (security review 1 of #2417: 8 of 30 trials). Both are read first and,
+ * if the folder is still there, each is written back where it is missing: the record only where git keeps records, the
+ * .git file never over one that is there.
+ */
+async function removeKeepingRecord(main: string, common: string, gitDir: string, path: string): Promise<void> {
+ // The whole record: its HEAD, index and reflog, and the refs only this worktree holds (refs/bisect, refs/worktree; security
+ // review 2 of #2417).
+ const saved = new Map<string, Buffer>()
+ const save = async (dir: string): Promise<void> => {
+ for (const entry of await readdir(dir, { withFileTypes: true })) {
+ const file = join(dir, entry.name)
+ if (entry.isDirectory()) await save(file)
+ else if (entry.isFile()) saved.set(relative(gitDir, file), await readFile(file))
+ }
+ }
+ await save(gitDir)
+ const link = await readFile(join(path, '.git'))
+ try {
+ await git(main, ['worktree', 'remove', '--', path])
+ } catch (error) {
+ if (!await lstat(path).then(info => info.isDirectory(), () => false)) throw error
+ await writeFile(join(path, '.git'), link, { flag: 'wx' }).catch((failed: NodeJS.ErrnoException) => { if (failed.code !== 'EEXIST') throw failed })
+ if (dirname(gitDir) === join(common, 'worktrees') && !await lstat(gitDir).then(() => true, () => false)) {
+ for (const [name, data] of saved) {
+ await mkdir(dirname(join(gitDir, name)), { recursive: true })
+ await writeFile(join(gitDir, name), data)
+ }
+ }
+ throw new WorktreeKept('WORKTREE_NOT_REMOVED')
+ }
+}
+
/** Removes one worktree under the rule above: true when it went. The Swarm's host calls it with no idle wait for a worker
- * it just stopped (superTabs/machines.ts `removeStoppedWorktree`). */
-export async function sweepOne(path: string, now: number, idleMs: number): Promise<boolean> {
+ * it just stopped (superTabs/machines.ts `removeStoppedWorktree`), and asks [unused] again just before git removes it: the
+ * reads and the folder walk take a while, and a session may open there meanwhile (security review 1 of #2417). */
+export async function sweepOne(path: string, now: number, idleMs: number, unused: () => boolean = () => true): Promise<boolean> {
const gitDir = resolve(path, await git(path, ['rev-parse', '--git-dir']))
const common = resolve(path, await git(path, ['rev-parse', '--git-common-dir']))
if (gitDir === common || basename(common) !== '.git') return false
@@ -85,7 +141,9 @@ export async function sweepOne(path: string, now: number, idleMs: number): Promi
if (await git(path, ['status', '--porcelain']) !== '') return false
const branch = await git(path, ['symbolic-ref', '--quiet', 'HEAD']).catch(() => null)
if (!branch && await git(path, ['rev-list', '--count', 'HEAD', '--not', '--branches', '--remotes']) !== '0') return false
- await git(main, ['worktree', 'remove', '--', path])
+ if (await unwritableFolder(path)) throw new WorktreeKept('WORKTREE_UNWRITABLE')
+ if (!unused()) return false
+ await removeKeepingRecord(main, common, gitDir, path)
const name = branch?.slice('refs/heads/'.length)
// Only a branch Harness made: marked in the repository's config, or named `harness/…` by older builds.
const made = name !== undefined && (name.startsWith('harness/')
diff --git a/cli/src/superTabs/machines.spec.ts b/cli/src/superTabs/machines.spec.ts
index ad0779f5a..d424ae9e7 100644
--- a/cli/src/superTabs/machines.spec.ts
+++ b/cli/src/superTabs/machines.spec.ts
@@ -1,5 +1,5 @@
import { execFileSync } from 'node:child_process'
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -1844,6 +1844,29 @@ describe('removed machine cleanup', () => {
expect(m.log.mock.calls.flat().join('\n')).not.toContain(m.recorded)
})
+ // One Mac, 10-28: git removed a third of a stopped worker's worktree, stopped at a skill folder copied in read-only, and
+ // deleted the worktree's record anyway, so git and gh failed in the rest. Folder permissions do not stop root or Windows.
+ it.runIf(process.platform !== 'win32' && process.getuid?.() !== 0)('keeps a stopped worker\'s worktree that git could not empty, still a worktree, and says why', async () => {
+ const m = machine()
+ rmSync(join(m.recorded, 'unpushed.txt'))
+ mkdirSync(join(m.recorded, '.harness', 'skills', 'copied'), { recursive: true })
+ writeFileSync(join(m.recorded, '.harness', '.gitignore'), '/*\n')
+ writeFileSync(join(m.recorded, '.harness', 'skills', 'copied', 'SKILL.md'), 'copied in read-only')
+ chmodSync(join(m.recorded, '.harness', 'skills', 'copied'), 0o555)
+ try {
+ // A Buddy's or an agent's live session in it: nothing is read or touched.
+ m.setLive([session('a-buddy', join(m.recorded, '.harness'))])
+ await m.open().removeStoppedWorktree('tab', 'h-recorded')
+ expect(m.log).not.toHaveBeenCalled()
+ m.setLive([])
+ await m.open().removeStoppedWorktree('tab', 'h-recorded')
+ expect(m.log.mock.calls).toEqual([['Kept a Swarm worktree: WORKTREE_UNWRITABLE']])
+ expect(startIds(m)).toEqual(['recorded', 'kept'])
+ expect(git(m.recorded, 'status', '--porcelain')).toBe('')
+ expect(git(m.clone, 'worktree', 'list', '--porcelain')).toContain('/recorded\n')
+ } finally { chmodSync(join(m.recorded, '.harness', 'skills', 'copied'), 0o755) }
+ })
+
it('keeps the record when its folder cannot be read for any reason but being gone', async () => {
const m = machine()
m.setLive([])
@@ -1878,8 +1901,9 @@ describe('removed machine cleanup', () => {
await Promise.all([host.removeStoppedWorktree('tab', 'h-recorded'), host.removeStoppedWorktree('tab', 'h-recorded')])
expect(existsSync(m.recorded)).toBe(false)
expect(startIds(m)).toEqual(['kept'])
- // Only the first looked for sessions in the folder; the second came once it was gone.
- expect(folderAtCheck).toEqual([true])
+ // Only the first looked for sessions in the folder, before its checks and again just before git removed it; the second
+ // came once it was gone.
+ expect(folderAtCheck).toEqual([true, true])
expect(m.log).not.toHaveBeenCalled()
})
diff --git a/cli/src/superTabs/machines.ts b/cli/src/superTabs/machines.ts
index 7cf88bb9e..63747c070 100644
--- a/cli/src/superTabs/machines.ts
+++ b/cli/src/superTabs/machines.ts
@@ -15,7 +15,7 @@ import { startAgent, type ScopedRequest } from './start.js'
import { PrepareRetryError, prepareSwarmStart } from './projectFolder.js'
import { readAuthSession } from '../lib/authSession.js'
import { createSwarmCloneTrustRepair, swarmCloneRoots } from './cloneTrust.js'
-import { git, sweepOne } from '../lib/worktreeSweep.js'
+import { git, sweepOne, WorktreeKept } from '../lib/worktreeSweep.js'
import { projectsRoot } from '../lib/projectFolder.js'
import { worktreeFolderName } from '../lib/agentNames.js'
import { teamPrivacyObserver } from '../team/observer.js'
@@ -911,13 +911,18 @@ export class ContributedMachines {
this.log('Kept a Swarm worktree: WORKTREE_UNPROVEN')
return
}
- if (this.core.agents.live().some(a => !!a.cwd && within(record.path, a.cwd))) return
// Review of #2266: like the weekly sweep (services/workspaces.ts), a paused session of another agent can be resumed
- // there, so its folder stays. This agent's own paused session is not kept: it was stopped or removed.
- if (this.core.agents.all().some(a => a.agentId !== harnessId && !!a.cwd && within(record.path, a.cwd))) return
+ // there, so its folder stays. This agent's own paused session is not kept: it was stopped or removed. Asked again just
+ // before git removes it (lib/worktreeSweep.ts sweepOne).
+ const unused = () => !this.core.agents.live().some(a => !!a.cwd && within(record.path, a.cwd))
+ && !this.core.agents.all().some(a => a.agentId !== harnessId && !!a.cwd && within(record.path, a.cwd))
+ if (!unused()) return
// File times are wall-clock, so the sweep's idle check reads the real clock; no idle wait is asked.
- // A failed git read (a locked worktree, a handle still open on Windows) keeps the folder; the log names no path.
- if (!await sweepOne(record.path, Date.now(), 0).catch(() => { this.log('Kept a Swarm worktree: WORKTREE_CHECK_FAILED'); return false })) return
+ // A failed git read (a locked worktree, a handle still open on Windows) keeps the folder; the log names no path. A folder git
+ // could not empty keeps its record and this one, and says which kind it was (lib/worktreeSweep.ts WorktreeKept).
+ if (!await sweepOne(record.path, Date.now(), 0, unused).catch(error => {
+ this.log(`Kept a Swarm worktree: ${error instanceof WorktreeKept ? error.code : 'WORKTREE_CHECK_FAILED'}`); return false
+ })) return
writeWorktrees(file, readWorktrees(file).filter(w => !mine(w)))
}