Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12748]): archive agent-cleanup and orphan checkouts outside the claim-ownership lock

[._.]/ Adam Eivy26b60dc05fae
diff --git a/server/services/branchReconcile.js b/server/services/branchReconcile.js
index f897fd6a44..215757801a 100644
--- a/server/services/branchReconcile.js
+++ b/server/services/branchReconcile.js
@@ -1272,7 +1272,9 @@ export async function reapSupersededBranches(repoPath, defaultBranch, superseded
* `prepare` runs after every gate has passed and before the first irreversible
* step, and aborts the retirement by resolving to `{ error }`. That is the only
* place work like "write a recoverable backup" belongs: earlier it is paid for
- * branches that are then refused, later it is too late to matter.
+ * branches that are then refused, later it is too late to matter. It runs
+ * with the claim-ownership lock released and the branch fenced, and removal
+ * then refuses a worktree whose dirty paths no longer match `b.dirtyPaths`.
*
* `requireCleanWorktree` (default true) is the caller's answer to "could this
* tree hold work that is not accounted for?". For `cleanupMerged` it can, so a
@@ -1328,15 +1330,19 @@ async function retireBranch(repoPath, b, opts) {
allowLiveClaim: true,
});
if (gated.result) return gated;
- if (!gated.archiveTree) return { result: await finishBranchRetirement(repoPath, gated, null) };
- return { fence: { repoPath, path: gated.archiveTree.path, branch: gated.b.branch }, gated };
+ if (!gated.archiveTree && !gated.prepare) return { result: await finishBranchRetirement(repoPath, gated) };
+ return { fence: { repoPath, path: gated.archiveTree?.path ?? gated.b.worktreePath, branch: gated.b.branch }, gated };
},
+ // The superseded backup copies dirty files, so it takes this window too (#12748).
archive: async ({ gated }) => {
+ const prepared = gated.prepare ? await gated.prepare() : null;
+ if (prepared?.error) return { result: { ok: false, reason: prepared.error } };
+ if (!gated.archiveTree) return { archive: { prepared } };
const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');
- return preserveMergedCheckout(repoPath, gated.archiveTree, gated.mergeTarget).then(archive => ({ archive }),
+ return preserveMergedCheckout(repoPath, gated.archiveTree, gated.mergeTarget).then(preservation => ({ archive: { prepared, preservation } }),
error => ({ result: { ok: false, reason: 'archive-failed', detail: error.message } }));
},
- finish: ({ gated }, preservation) => finishBranchRetirement(repoPath, gated, preservation),
+ finish: ({ gated }, { prepared, preservation }) => finishBranchRetirement(repoPath, gated, { prepared, preservation }),
});
}
@@ -1411,10 +1417,11 @@ async function admitBranchRetirement(repoPath, b, { activeAgentIds = new Set(),
return { b, expectedHead, readPolicyHold, archiveTree, mergeTarget, label, requireCleanWorktree, prepare, claimOwners, mergedRetirement };
}
-/** Removal and branch deletion; `preservation` is the archive taken after admission, if any. */
-async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHold, mergeTarget, label, requireCleanWorktree, prepare, claimOwners, mergedRetirement }, preservation) {
- const prepared = prepare ? await prepare() : null;
- if (prepared?.error) return { ok: false, reason: prepared.error };
+/**
+ * Removal and branch deletion. `preservation` is the archive and `prepared` the
+ * `prepare` backup, each taken after admission with the ownership lock released.
+ */
+async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHold, mergeTarget, label, requireCleanWorktree, claimOwners, mergedRetirement }, { prepared = null, preservation = null } = {}) {
if (b.worktreePath) {
const verifyRemoval = async (holder) => {
if (!holder || holder.branch !== `refs/heads/${b.branch}` || holder.head !== expectedHead) return 'worktree-holder-changed';
@@ -1433,9 +1440,11 @@ async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHol
repoPath, branch: b.branch, path: b.worktreePath, head: expectedHead,
})) return 'claim-completion-unverified';
}
- const dirty = await isWorktreeDirty(b.worktreePath);
- if (dirty === null) return 'worktree-status-unavailable';
- if (requireCleanWorktree && dirty) return 'worktree-dirty';
+ const dirtyPaths = await worktreeDirtyPaths(b.worktreePath);
+ if (dirtyPaths === null) return 'worktree-status-unavailable';
+ if (requireCleanWorktree && dirtyPaths.length) return 'worktree-dirty';
+ // The backup ran unlocked; a tree whose change set moved since is not what it holds.
+ if (prepared && !sameDirtyPaths(b.dirtyPaths, dirtyPaths)) return 'worktree-changed-since-backup';
if (mergedRetirement && !await hasBranchMergeEvidence(repoPath, expectedHead, mergeTarget).catch(() => false)) return 'not-merged-on-recheck';
return preservation ? preservation.verify(holder) : null;
};
diff --git a/server/services/branchReconcile.test.js b/server/services/branchReconcile.test.js
index 6b24e6de46..eb067f4cd8 100644
--- a/server/services/branchReconcile.test.js
+++ b/server/services/branchReconcile.test.js
@@ -1613,6 +1613,51 @@ describe('reconcile — cached SUPERSEDED verdicts (#3842)', () => {
expect(wt.deleteCompletedLocalBranch).not.toHaveBeenCalled();
});
+ // #12748: the backup copies the dirty tree, and it ran inside the shared
+ // claim-ownership lock that every bind/release/reserve queues behind.
+ it('backs up an opted-in superseded checkout with the ownership queue free and only it fenced', async () => {
+ setupAbandoned();
+ tryReadFileMock.mockResolvedValue(ledger());
+ let started;
+ let finish;
+ const backingUp = new Promise(resolve => { started = resolve; });
+ backupSupersededBranchMock.mockImplementationOnce(() => new Promise(resolve => {
+ finish = () => resolve({ dir: '/backups/x', manifest: 'manifest.json', untracked: [] });
+ started();
+ }));
+ const pending = reconcile('/repo', { reapSuperseded: true, activeAgentIds: new Set() });
+ await backingUp;
+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', path: WORKTREE })).toBe('checkout-retirement-reserved');
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: BRANCH })).toBe('checkout-retirement-reserved');
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'feature/unrelated', path: '/repo/data/cos/worktrees/other' })).toBe(null);
+ finish();
+ expect((await pending).reapedSuperseded).toEqual([BRANCH]);
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: BRANCH })).toBe(null);
+ });
+
+ it('retains an opted-in superseded checkout whose change set moves during its unlocked backup', async () => {
+ setupAbandoned();
+ tryReadFileMock.mockResolvedValue(ledger());
+ const scripted = execGit.getMockImplementation();
+ let backedUp = false;
+ execGit.mockImplementation(async (args, ...rest) => (backedUp && args[0] === 'status'
+ ? { stdout: `${DIRTY}?? notes.md\n`, exitCode: 0 }
+ : scripted(args, ...rest)));
+ backupSupersededBranchMock.mockImplementation(async () => {
+ backedUp = true;
+ return { dir: '/backups/x', manifest: 'manifest.json', untracked: [] };
+ });
+ wt.forceRemoveWorktreeDir.mockImplementation(async (_repo, _path, options) => {
+ const reason = await options.verifyRemoval({ head: RETIRE_HEAD, path: WORKTREE, branch: `refs/heads/${BRANCH}` });
+ return { removed: !reason, reason };
+ });
+ const res = await reconcile('/repo', { reapSuperseded: true, activeAgentIds: new Set() });
+ expect(res.reapedSuperseded).toEqual([]);
+ expect(res.skipped).toContainEqual({ branch: BRANCH, reason: 'worktree-changed-since-backup' });
+ expect(wt.deleteCompletedLocalBranch).not.toHaveBeenCalled();
+ });
+
it('retains an explicitly opted-in superseded ref when its deletion lease fails', async () => {
setupAbandoned();
tryReadFileMock.mockResolvedValue(ledger());
diff --git a/server/services/cosClaimOwnershipState.js b/server/services/cosClaimOwnershipState.js
index 8dfe16c3f3..fd8b9a224f 100644
--- a/server/services/cosClaimOwnershipState.js
+++ b/server/services/cosClaimOwnershipState.js
@@ -33,14 +33,15 @@ export function checkoutRetirementFenceReason({ repoPath, path = null, branch =
}
/**
- * Fence one checkout while it is archived outside the lock. Call while holding
+ * Fence one checkout (or, with no `path`, one checkout-less branch) while it is
+ * archived outside the lock. Call while holding
* `withClaimOwnershipLock`, after the ownership checks pass, and call the returned
* release under the lock once removal settles (it is idempotent, so a `finally`
* may repeat it). Returns null when the checkout is already fenced.
*/
export function reserveCheckoutRetirementFence({ repoPath, path, branch = null }) {
if (checkoutRetirementFenceReason({ repoPath, path, branch })) return null;
- const fence = { repo: fencePath(repoPath), path: fencePath(path), branch: fenceBranch(branch) };
+ const fence = { repo: fencePath(repoPath), path: path ? fencePath(path) : null, branch: fenceBranch(branch) };
retirementFences.add(fence);
return () => { retirementFences.delete(fence); };
}
diff --git a/server/services/worktreeExternalCleanup.test.js b/server/services/worktreeExternalCleanup.test.js
index e8922b5b88..6b40c901b8 100644
--- a/server/services/worktreeExternalCleanup.test.js
+++ b/server/services/worktreeExternalCleanup.test.js
@@ -10,6 +10,16 @@ import { materializeGitRepo, resetGitWorktreeSandbox, SKIP_HEAVY_INTEGRATION } f
import { PATHS } from '../lib/fileUtils.js';
import { execGit } from '../lib/execGit.js';
import { cleanupOrphanedWorktrees } from './worktreeManager.js';
+import { withClaimOwnershipLock, checkoutRetirementFenceReason } from './cosClaimOwnershipState.js';
+
+const archiveGate = vi.hoisted(() => ({ beforeArchive: null }));
+vi.mock('./heldCheckoutRecovery.js', async importOriginal => {
+ const actual = await importOriginal();
+ return { ...actual, preserveMergedCheckout: async (...args) => {
+ await archiveGate.beforeArchive?.(...args);
+ return actual.preserveMergedCheckout(...args);
+ } };
+});
vi.mock('fs', async (importOriginal) => {
const actual = await importOriginal();
@@ -104,6 +114,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('external orphan cleanup recovery policy
fault.aliasTarget = null;
fault.registrationPath = null;
fault.onGit = null;
+ archiveGate.beforeArchive = null;
await resetGitWorktreeSandbox(external, initialHead);
await rm(PATHS.worktrees, { recursive: true, force: true });
await mkdir(PATHS.worktrees, { recursive: true });
@@ -137,6 +148,26 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('external orphan cleanup recovery policy
expect(await branchExists('agent-merged')).toBe(false);
});
+ // #12748: the archive ran inside the shared claim-ownership lock.
+ it('archives with the ownership queue free while fencing only the reclaimed checkout', async () => {
+ const path = await addTree('agent-fenced', { committed: true });
+ await execGit(['merge', '--no-ff', 'agent-fenced', '--no-edit'], external);
+ let started;
+ let resume;
+ const archiving = new Promise(resolve => { started = resolve; });
+ // Git spells the parent repository its own way on Windows; fence what it archives.
+ archiveGate.beforeArchive = repoPath => new Promise(resolve => { resume = resolve; started(repoPath); });
+ const cleaning = cleanupOrphanedWorktrees(primary, new Set(), NO_OWNERS);
+ const repoPath = await archiving;
+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');
+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-fenced' })).toBe('checkout-retirement-reserved');
+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-unrelated' })).toBe(null);
+ resume();
+ expect(await cleaning).toBe(1);
+ expect(existsSync(path)).toBe(false);
+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-fenced' })).toBe(null);
+ });
+
it('matches Git registration spelling when Node retains another alias for the same checkout', async () => {
const path = await addTree('agent-path-alias');
const aliasRoot = join(root(), 'git-spelling');
diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.js
index e21c6cf2cd..8b006d72e8 100644
--- a/server/services/worktreeManager.js
+++ b/server/services/worktreeManager.js
@@ -1385,31 +1385,35 @@ async function cleanupWorktreeBranch(sourceWorkspace, branchName, {
* run, forever. ONLY the discard path may pass this.
*/
export async function removeWorktree(agentId, sourceWorkspace, branchName, options = {}) {
- const { withClaimOwnershipLock, readClaimOwnershipAgents, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');
- // A retirement archiving this checkout with the lock released owns it until removal.
- const ownershipHold = async () => checkoutRetirementFenceReason({ repoPath: sourceWorkspace, path: join(WORKTREES_DIR, agentId), branch: branchName })
- || claimCheckoutOwnerReason({
- branchName, holderPath: join(WORKTREES_DIR, agentId), sourceWorkspace,
- agents: await readClaimOwnershipAgents().catch(() => null), ignoreIds: [agentId],
- });
+ const { readClaimOwnershipAgents } = await import('./cosClaimOwnershipState.js');
+ const ownershipHold = async () => claimCheckoutOwnerReason({
+ branchName, holderPath: join(WORKTREES_DIR, agentId), sourceWorkspace,
+ agents: await readClaimOwnershipAgents().catch(() => null), ignoreIds: [agentId],
+ });
const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');
- return withClaimOwnershipLock(() => withCheckoutPublication(sourceWorkspace,
- () => removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold)));
-
+ return withCheckoutPublication(sourceWorkspace,
+ () => removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold));
}
+/**
+ * Runs cleanup as claim-ownership critical sections with every archive between two
+ * of them (#12748): each archive hashes and copies the whole checkout, and every
+ * bind/release/reserve queues behind that lock on a 15 s budget. While an archive
+ * runs this checkout is fenced; each section re-reads the owners before it acts.
+ * `ownershipHold` reads registered owners only — this function owns the fence check.
+ */
async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold) {
+ const { withClaimOwnershipLock, checkoutRetirementFenceReason, retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');
const worktreePath = join(WORKTREES_DIR, agentId);
+ const fence = { repoPath: sourceWorkspace, path: worktreePath, branch: branchName };
const warnings = [];
- const { getDefaultBranch } = await import('./git.js');
- const defaultBranch = await getDefaultBranch(sourceWorkspace, { strict: true }).catch(() => null);
- const protectedReason = PROTECTED_BRANCHES.includes(branchName) || branchName === defaultBranch
- ? 'protected-branch' : !defaultBranch ? 'default-branch-unavailable' : null;
- if (protectedReason) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false,
- warnings: [`Worktree preserved — ${protectedReason}`] };
- const owner = await ownershipHold();
- if (owner) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false,
- warnings: [`Worktree preserved — ${owner}`] };
+ let merged = false;
+ let commitsAhead = 0;
+ let mergeRefused = false;
+ const held = reason => {
+ warnings.push(`Worktree preserved — ${reason}`);
+ return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
+ };
const completionHold = async (expectedHead) => {
const owner = await ownershipHold();
if (owner) return owner;
@@ -1434,95 +1438,102 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,
const archive = await preserveMergedCheckout(sourceWorkspace, tree, null);
return { ...archive, head: tree.head,
verify: async holder => await completionHold(tree.head) || await archive.verify(holder) };
-
};
- const preserveOrHold = () => preserve().catch(error => {
- warnings.push(`Worktree preserved — archive failed: ${error.message}`);
- return null;
+ // One archive between two critical sections. `admit` (under the lock, after the
+ // owner re-read) returns a result to stop before fencing; `finish` runs under the
+ // lock again with the fence still held.
+ const retire = ({ admit = async () => null, onArchiveFailure, finish }) => retireWithUnlockedArchive({
+ busy: () => held('checkout-retirement-reserved'),
+ admit: async () => {
+ const owner = await ownershipHold();
+ if (owner) return { result: held(owner) };
+ const stop = await admit();
+ return stop ? { result: stop } : { fence };
+ },
+ archive: () => preserve().then(archive => ({ archive }), error => {
+ warnings.push(`Worktree preserved — archive failed: ${error.message}`);
+ return { result: onArchiveFailure() };
+ }),
+ finish: (_admitted, preservation) => finish(preservation),
});
-
- if (!existsSync(worktreePath)) {
- // The directory is gone, but git may still register it (a stale entry pins the
- // branch). Verify rather than assume; prune when the registration survives.
- let gone = removalOutcome(worktreePath, await probeWorktreeRemoval(sourceWorkspace, worktreePath));
- let preservation = null;
- if (!gone.removed) {
- preservation = await preserveOrHold();
- if (!preservation) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
- gone = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, { verifyRemoval: preservation.verify });
- }
- if (!gone.removed) {
- warnings.push(gone.warning);
- return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
- }
- console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);
+ const cleanupBranch = async (expectedHead) => {
const finalOwner = await ownershipHold();
- if (finalOwner) return { merged: false, removed: true, uncommittedSaved: false,
- warnings: [...warnings, `Branch ${branchName} preserved — ${finalOwner}`] };
+ if (finalOwner) {
+ warnings.push(`Branch ${branchName} preserved — ${finalOwner}`);
+ return false;
+ }
await cleanupWorktreeBranch(sourceWorkspace, branchName, {
- warnings, expectedHead: preservation?.head,
+ hasUnmergedCommits: options.merge && !merged && (commitsAhead > 0 || mergeRefused), warnings, expectedHead,
});
- return { merged: false, removed: true, uncommittedSaved: false, warnings };
- }
+ return true;
+ };
- // Verify the worktree still points to the correct repo before trusting git status.
- // If the .git file is missing (e.g., worktree was partially cleaned up), git walks up
- // the directory tree and may find a parent repo (e.g., PortOS) instead of the app repo.
- // In that case, even a successful git status describes the wrong checkout and
- // cannot prove this directory has no unique work.
- const detectedToplevel = await execGit(['rev-parse', '--show-toplevel'], worktreePath)
- .then(r => r.stdout.trim())
- .catch(() => null);
- // Compare realpath-resolved forms so symlinks (e.g. macOS /var → /private/var)
- // or normalization differences don't false-positive as a broken worktree.
- if (detectedToplevel && !pathsEqual(detectedToplevel, worktreePath)) {
- warnings.push('Worktree preserved — broken checkout identity prevents verifying its contents');
- console.warn(`⚠️ Worktree ${agentId} has a broken checkout identity; preserving its directory and branch`);
- return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
- }
+ const inspected = await withClaimOwnershipLock(async () => {
+ const { getDefaultBranch } = await import('./git.js');
+ const defaultBranch = await getDefaultBranch(sourceWorkspace, { strict: true }).catch(() => null);
+ const protectedReason = PROTECTED_BRANCHES.includes(branchName) || branchName === defaultBranch
+ ? 'protected-branch' : !defaultBranch ? 'default-branch-unavailable' : null;
+ if (protectedReason) return { result: held(protectedReason) };
+ // A retirement archiving this checkout with the lock released owns it until removal.
+ const owner = checkoutRetirementFenceReason(fence) || await ownershipHold();
+ if (owner) return { result: held(owner) };
+
+ if (!existsSync(worktreePath)) {
+ // The directory is gone, but git may still register it (a stale entry pins the
+ // branch). Verify rather than assume; prune when the registration survives.
+ const gone = removalOutcome(worktreePath, await probeWorktreeRemoval(sourceWorkspace, worktreePath));
+ if (!gone.removed) return { missing: true };
+ console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);
+ await cleanupBranch(undefined);
+ return { result: { merged: false, removed: true, uncommittedSaved: false, warnings } };
+ }
- // Safety check: abort removal when uncommitted changes are detected.
- // Also fail closed if git status itself fails — treat unknown state as dirty.
- let dirtyFiles;
- try {
- dirtyFiles = (await execGit(['status', '--porcelain'], worktreePath)).stdout.trim();
- } catch (err) {
- console.log(`⚠️ git status failed for worktree ${agentId}, preserving to avoid data loss: ${err.message}`);
- warnings.push(`Worktree preserved — git status failed: ${err.message}`);
- return { merged: false, removed: false, uncommittedSaved: false, warnings };
- }
- // The completion sentinel is not authored work: it has already been consumed
- // by finalizeAgent before this cleanup runs. Ignore it for the preservation
- // decision, while still preserving the tree if any real change remains; the
- // eventual forced worktree removal discards it with the rest of the completed
- // checkout. The run's canonical name and its bounded truncation-recovery
- // name, plus the legacy shared one a pre-upgrade run may have left — and no
- // other agent's.
- const dirt = classifyWorktreeDirt(dirtyFiles, {
- ignoredPaths: [DONE_SENTINEL_NAME, ...doneSentinelCandidateNames(agentId)]
- });
- if (!dirt.clean && options.discardDirt) {
- // Throwaway posture: the caller has already established that nothing in this
- // tree is wanted. Log what is being dropped so a surprising loss is at least
- // traceable, then fall through to removal.
- const shown = dirt.realChangePaths.slice(0, DIRT_PATHS_IN_WARNING);
- if (shown.length) console.log(`🌳 Discarding uncommitted changes in throwaway worktree ${agentId}: ${shown.join(', ')}`);
- } else if (!dirt.clean) {
- // Discard auto-generated lockfile changes that agents don't intend to commit
- // (e.g., npm install resolving ^version to exact version in package-lock.json)
- if (dirt.lockfileOnly) {
- // Preserve the original bytes before the intentional lockfile cleanup.
- // A second snapshot below guards the post-cleanup removal boundary.
- if (!await preserveOrHold()) return { merged: false, removed: false, uncommittedSaved: false, warnings };
- console.log(`🧹 Discarding ${dirt.lockfilePaths.length} auto-generated lockfile change(s) in worktree ${agentId}`);
- // Never throws past removeWorktree: an undiscardable lockfile keeps the tree.
- const discarded = await discardLockfileChanges(worktreePath, dirt).then(() => true, err => {
- console.log(`⚠️ Lockfile discard failed in worktree ${agentId}, preserving to avoid data loss: ${err.message}`);
- warnings.push(`Worktree preserved — lockfile discard failed in ${worktreePath}: ${err.message}`);
- return false;
- });
- if (!discarded) return { merged: false, removed: false, uncommittedSaved: false, warnings };
- } else {
+ // Verify the worktree still points to the correct repo before trusting git status.
+ // If the .git file is missing (e.g., worktree was partially cleaned up), git walks up
+ // the directory tree and may find a parent repo (e.g., PortOS) instead of the app repo.
+ // In that case, even a successful git status describes the wrong checkout and
+ // cannot prove this directory has no unique work.
+ const detectedToplevel = await execGit(['rev-parse', '--show-toplevel'], worktreePath)
+ .then(r => r.stdout.trim())
+ .catch(() => null);
+ // Compare realpath-resolved forms so symlinks (e.g. macOS /var → /private/var)
+ // or normalization differences don't false-positive as a broken worktree.
+ if (detectedToplevel && !pathsEqual(detectedToplevel, worktreePath)) {
+ warnings.push('Worktree preserved — broken checkout identity prevents verifying its contents');
+ console.warn(`⚠️ Worktree ${agentId} has a broken checkout identity; preserving its directory and branch`);
+ return { result: { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings } };
+ }
+
+ // Safety check: abort removal when uncommitted changes are detected.
+ // Also fail closed if git status itself fails — treat unknown state as dirty.
+ let dirtyFiles;
+ try {
+ dirtyFiles = (await execGit(['status', '--porcelain'], worktreePath)).stdout.trim();
+ } catch (err) {
+ console.log(`⚠️ git status failed for worktree ${agentId}, preserving to avoid data loss: ${err.message}`);
+ warnings.push(`Worktree preserved — git status failed: ${err.message}`);
+ return { result: { merged: false, removed: false, uncommittedSaved: false, warnings } };
+ }
+ // The completion sentinel is not authored work: it has already been consumed
+ // by finalizeAgent before this cleanup runs. Ignore it for the preservation
+ // decision, while still preserving the tree if any real change remains; the
+ // eventual forced worktree removal discards it with the rest of the completed
+ // checkout. The run's canonical name and its bounded truncation-recovery
+ // name, plus the legacy shared one a pre-upgrade run may have left — and no
+ // other agent's.
+ const dirt = classifyWorktreeDirt(dirtyFiles, {
+ ignoredPaths: [DONE_SENTINEL_NAME, ...doneSentinelCandidateNames(agentId)]
+ });
+ if (!dirt.clean && options.discardDirt) {
+ // Throwaway posture: the caller has already established that nothing in this
+ // tree is wanted. Log what is being dropped so a surprising loss is at least
+ // traceable, then fall through to removal.
+ const shown = dirt.realChangePaths.slice(0, DIRT_PATHS_IN_WARNING);
+ if (shown.length) console.log(`🌳 Discarding uncommitted changes in throwaway worktree ${agentId}: ${shown.join(', ')}`);
+ } else if (!dirt.clean) {
+ // Auto-generated lockfile changes agents don't intend to commit (e.g. npm
+ // install resolving ^version to an exact version) are discarded below.
+ if (dirt.lockfileOnly) return { lockfile: { dirt, dirtyFiles } };
// NAME the dirty paths. The bare message reads as "your work was left
// behind" with no way to tell real uncommitted work from a transient
// observation, and the worktree is usually gone by the time anyone looks —
@@ -1533,29 +1544,71 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,
const detail = `${shown.join(', ')}${more > 0 ? ` (+${more} more)` : ''}`;
console.log(`⚠️ Preserving worktree for ${agentId} — uncommitted changes detected (${detail}), aborting cleanup to avoid data loss`);
warnings.push(`Worktree preserved — uncommitted changes detected in ${worktreePath}: ${detail}`);
- return { merged: false, removed: false, uncommittedSaved: false, warnings };
+ return { result: { merged: false, removed: false, uncommittedSaved: false, warnings } };
}
+ return {};
+ });
+ if (inspected.result) return inspected.result;
+
+ if (inspected.missing) {
+ return retire({
+ onArchiveFailure: () => ({ merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings }),
+ finish: async (preservation) => {
+ const gone = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, { verifyRemoval: preservation.verify });
+ if (!gone.removed) {
+ warnings.push(gone.warning);
+ return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
+ }
+ console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);
+ await cleanupBranch(preservation.head);
+ return { merged: false, removed: true, uncommittedSaved: false, warnings };
+ },
+ });
}
- let merged = false;
- let commitsAhead = 0;
- let mergeRefused = false;
+ if (inspected.lockfile) {
+ const { dirt, dirtyFiles } = inspected.lockfile;
+ // Preserve the original bytes before the intentional lockfile cleanup.
+ // The second snapshot below guards the post-cleanup removal boundary.
+ const discardFailure = await retire({
+ onArchiveFailure: () => ({ merged: false, removed: false, uncommittedSaved: false, warnings }),
+ finish: async () => {
+ // The snapshot ran with the lock released; discard only the tree it captured.
+ const current = await execGit(['status', '--porcelain'], worktreePath).then(r => r.stdout.trim(), () => null);
+ if (current !== dirtyFiles) {
+ warnings.push(`Worktree preserved — ${worktreePath} changed while its pre-cleanup snapshot was archived`);
+ return { merged: false, removed: false, uncommittedSaved: false, warnings };
+ }
+ console.log(`🧹 Discarding ${dirt.lockfilePaths.length} auto-generated lockfile change(s) in worktree ${agentId}`);
+ // Never throws past removeWorktree: an undiscardable lockfile keeps the tree.
+ return discardLockfileChanges(worktreePath, dirt).then(() => null, err => {
+ console.log(`⚠️ Lockfile discard failed in worktree ${agentId}, preserving to avoid data loss: ${err.message}`);
+ warnings.push(`Worktree preserved — lockfile discard failed in ${worktreePath}: ${err.message}`);
+ return { merged: false, removed: false, uncommittedSaved: false, warnings };
+ });
+ },
+ });
+ if (discardFailure) return discardFailure;
+ }
- if (options.merge) {
- const currentBranch = (await execGit(
- ['rev-parse', '--abbrev-ref', 'HEAD'],
- sourceWorkspace
- )).stdout.trim();
+ return retire({
+ admit: async () => {
+ if (!options.merge) return null;
+ const currentBranch = (await execGit(
+ ['rev-parse', '--abbrev-ref', 'HEAD'],
+ sourceWorkspace
+ )).stdout.trim();
- // Defense-in-depth: NEVER merge the agent branch into a non-default branch.
- // See shouldRefuseDefaultBranchMerge for the rationale.
- const { getDefaultBranch } = await import('./git.js');
- const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null);
- if (shouldRefuseDefaultBranchMerge(currentBranch, defaultBranch)) {
- console.log(`🌳 Refusing auto-merge of ${branchName} into '${currentBranch}' (default branch is '${defaultBranch || 'unknown'}'). Use \`gh pr merge\` for non-default targets.`);
- warnings.push(`Auto-merge skipped — source repo HEAD is on '${currentBranch}', not default '${defaultBranch || 'unknown'}'. Branch ${branchName} preserved for manual review.`);
- mergeRefused = true;
- } else {
+ // Defense-in-depth: NEVER merge the agent branch into a non-default branch.
+ // See shouldRefuseDefaultBranchMerge for the rationale.
+ const { getDefaultBranch } = await import('./git.js');
+ const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null);
+ if (shouldRefuseDefaultBranchMerge(currentBranch, defaultBranch)) {
+ console.log(`🌳 Refusing auto-merge of ${branchName} into '${currentBranch}' (default branch is '${defaultBranch || 'unknown'}'). Use \`gh pr merge\` for non-default targets.`);
+ warnings.push(`Auto-merge skipped — source repo HEAD is on '${currentBranch}', not default '${defaultBranch || 'unknown'}'. Branch ${branchName} preserved for manual review.`);
+ mergeRefused = true;
+ return null;
+ }
const countResult = await execGit(
['rev-list', '--count', `${currentBranch}..${branchName}`],
sourceWorkspace,
@@ -1585,42 +1638,32 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,
warnings.push(`Auto-merge failed for branch ${branchName} — branch preserved for manual recovery`);
});
}
- }
- }
-
- const preservation = await preserveOrHold();
- if (!preservation) return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
- const removal = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, {
- verifyRemoval: preservation.verify,
- label: `Worktree remove failed for ${agentId}, falling back to manual cleanup`,
- log: 'all',
- subject: agentId,
- });
- if (!removal.removed) {
- // The checkout (or its registration) survived: the branch is still held, so
- // deleting it would fail, and `removed: true` would tell the caller it may
- // queue work onto a branch the old worktree still owns. Report the truth with
- // a retryable warning; `merged` stays accurate for the publication outcome.
- console.warn(`⚠️ Worktree for ${agentId} was not removed — branch ${branchName} kept`);
- warnings.push(removal.warning);
- return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
- }
+ return null;
+ },
+ onArchiveFailure: () => ({ merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings }),
+ finish: async (preservation) => {
+ const removal = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, {
+ verifyRemoval: preservation.verify,
+ label: `Worktree remove failed for ${agentId}, falling back to manual cleanup`,
+ log: 'all',
+ subject: agentId,
+ });
+ if (!removal.removed) {
+ // The checkout (or its registration) survived: the branch is still held, so
+ // deleting it would fail, and `removed: true` would tell the caller it may
+ // queue work onto a branch the old worktree still owns. Report the truth with
+ // a retryable warning; `merged` stays accurate for the publication outcome.
+ console.warn(`⚠️ Worktree for ${agentId} was not removed — branch ${branchName} kept`);
+ warnings.push(removal.warning);
+ return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
+ }
- // Merge evidence and completion policy were checked before removal; the
- // branch helper checks them again before its leased local deletion.
- const hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);
- const finalOwner = await ownershipHold();
- if (finalOwner) {
- warnings.push(`Branch ${branchName} preserved — ${finalOwner}`);
- return { merged, removed: true, uncommittedSaved: false, warnings };
- }
- await cleanupWorktreeBranch(sourceWorkspace, branchName, {
- hasUnmergedCommits, warnings, expectedHead: preservation.head,
+ // Merge evidence and completion policy were checked before removal; the
+ // branch helper checks them again before its leased local deletion.
+ if (await cleanupBranch(preservation.head)) console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);
+ return { merged, removed: true, uncommittedSaved: false, warnings };
+ },
});
-
- console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);
-
- return { merged, removed: true, uncommittedSaved: false, warnings };
}
/**
@@ -1839,23 +1882,6 @@ function activeIdsReader(activeIds) {
return typeof activeIds === 'function' ? activeIds : () => activeIds;
}
-/**
- * Remove an orphan only if it is STILL unowned: liveness re-read and the agent
- * registry checked inside the lock a claim bind takes, so neither an agent
- * dispatched since the caller's last check nor a registered claim/agent owner
- * can be removed out from under it. Resolves `remove()`'s result, or null when held.
- */
-async function removeIfStillOrphaned({ isOwned, branchName, holderPath, sourceWorkspace, readOwners }, remove) {
- const { withClaimOwnershipLock, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');
- return withClaimOwnershipLock(async () => {
- if (isOwned() || checkoutRetirementFenceReason({ repoPath: sourceWorkspace, path: holderPath, branch: branchName })) return null;
- const agents = await readOwners();
- if (claimCheckoutOwnerReason({ branchName, holderPath, sourceWorkspace, agents })) return null;
- const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');
- return withCheckoutPublication(sourceWorkspace, remove);
- });
-}
-
/**
* Clean up any orphaned worktrees (worktrees whose agent no longer exists)
*
@@ -1871,6 +1897,7 @@ export async function cleanupOrphanedWorktrees(sourceWorkspace, getActiveIds, {
const readActiveIds = activeIdsReader(getActiveIds);
const { readClaimOwnershipAgents } = await import('./cosClaimOwnershipState.js');
+ const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');
const readOwners = claimOwnerReader(claimOwners, readClaimOwnershipAgents);
const worktrees = await listWorktrees(sourceWorkspace).catch(() => []);
let cleaned = 0;
@@ -1896,9 +1923,10 @@ export async function cleanupOrphanedWorktrees(sourceWorkspace, getActiveIds, {
const branchName = wt.branch.replace('refs/heads/', '');
// Attempt merge so committed work from preserved worktrees (e.g., PR/push failures) isn't lost.
- // If merge fails, the branch is preserved for manual recovery.
- const result = await removeIfStillOrphaned(
- { isOwned: ownershipReason, branchName, holderPath: wt.path, sourceWorkspace, readOwners },
+ // If merge fails, the branch is preserved for manual recovery. Every critical
+ // section re-reads liveness and the registry, so neither an agent dispatched
+ // since the check above nor a registered claim owner loses its checkout.
+ const result = await withCheckoutPublication(sourceWorkspace,
() => removeWorktreeNow(agentId, sourceWorkspace, branchName, { merge: true }, async () =>
ownershipReason() || claimCheckoutOwnerReason({ branchName, holderPath: wt.path, sourceWorkspace, agents: await readOwners() })),
).catch(err => {
@@ -2095,6 +2123,7 @@ export async function reapMergedWorktrees(sourceWorkspace, {
*/
async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readOwners) {
const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');
+ const { retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');
const entries = await readdir(WORKTREES_DIR, { withFileTypes: true }).catch(() => []);
let cleaned = 0;
@@ -2146,12 +2175,21 @@ async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readO
// Non-force removal is a final Git guard against newly dirty or locked
// trees. A refusal must never fall through to recursive filesystem removal.
+ // The archive runs between two ownership critical sections (#12748) with
+ // this checkout fenced; both re-read liveness and the registry.
const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');
- const removed = await withCheckoutPublication(parentRepo, () => removeIfStillOrphaned(
- { isOwned: () => ownership({ locked: wt.locked }), branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, readOwners },
- async () => {
+ const removed = await withCheckoutPublication(parentRepo, () => retireWithUnlockedArchive({
+ busy: () => false,
+ admit: async () => {
+ if (ownership({ locked: wt.locked })
+ || claimCheckoutOwnerReason({ branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, agents: await readOwners() })) return { result: false };
+ return { fence: { repoPath: parentRepo, path: worktreePath, branch: branchName } };
+ },
+ archive: async () => {
const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');
- const preservation = await preserveMergedCheckout(parentRepo, wt, target);
+ return { archive: await preserveMergedCheckout(parentRepo, wt, target) };
+ },
+ finish: async (_admitted, preservation) => {
const latest = (await listWorktrees(parentRepo)).find(holder => pathsEqual(holder.path, worktreePath));
if (ownership({ locked: latest?.locked }) || await preservation.verify(latest)
|| claimCheckoutOwnerReason({ branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, agents: await readOwners() })
@@ -2167,7 +2205,7 @@ async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readO
}
return ok;
},
- )).catch(err => {
+ })).catch(err => {
console.warn(`⚠️ Preserved external worktree ${entry.name}: ${err.message}`);
return false;
});
diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.js
index cd19dd861b..91afc12457 100644
--- a/server/services/worktreeManager.test.js
+++ b/server/services/worktreeManager.test.js
@@ -1184,6 +1184,49 @@ describe('removeWorktree identity, dirt and branch preservation', () => {
expect(rm).not.toHaveBeenCalled();
});
+ // #12748: every agent completion archived its checkout inside the shared
+ // claim-ownership lock, starving each bind/release/reserve queued behind it.
+ it.each(['present', 'missing', 'lockfile'])('archives a %s checkout with the ownership queue free and only it fenced', async state => {
+ if (state === 'missing') existsSync.mockReturnValue(false);
+ if (state === 'lockfile') scriptGit({ porcelain: ' M package-lock.json' });
+ const { withClaimOwnershipLock, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');
+ const fenced = { repoPath: '/repo', path: join(PATHS.worktrees, 'agent-x') };
+ let started;
+ let finish;
+ const archiving = new Promise(resolve => { started = resolve; });
+ preserveCheckoutMock.mockImplementationOnce(() => new Promise(resolve => {
+ finish = () => resolve({ archiveDir: '/archive', verify: async () => null });
+ started();
+ }));
+ const pending = removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');
+ await archiving;
+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');
+ expect(checkoutRetirementFenceReason(fenced)).toBe('checkout-retirement-reserved');
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'cos/task-1/agent-x' })).toBe('checkout-retirement-reserved');
+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'cos/task-2/agent-y', path: join(PATHS.worktrees, 'agent-y') })).toBe(null);
+ finish();
+ expect((await pending).removed).toBe(true);
+ expect(checkoutRetirementFenceReason(fenced)).toBe(null);
+ });
+
+ it('keeps a lockfile-only checkout that changed while its pre-cleanup snapshot was archived', async () => {
+ scriptGit({ porcelain: ' M package-lock.json' });
+ const scripted = execGitMock.getMockImplementation();
+ let archived = false;
+ execGitMock.mockImplementation((args, ...rest) => (archived && args[0] === 'status'
+ ? Promise.resolve({ stdout: ' M package-lock.json\n?? notes.md', stderr: '', exitCode: 0 })
+ : scripted(args, ...rest)));
+ preserveCheckoutMock.mockImplementationOnce(async () => {
+ archived = true;
+ return { archiveDir: '/archive', verify: async () => null };
+ });
+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');
+ expect(result.removed).toBe(false);
+ expect(result.warnings.join(' ')).toContain('changed while its pre-cleanup snapshot was archived');
+ expect(execGitMock.mock.calls.some(([args]) => args[0] === 'checkout' || args[0] === 'push'
+ || (args[0] === 'worktree' && args[1] === 'remove'))).toBe(false);
+ });
+
it('retains checkout and ref when preserved bytes change before removal', async () => {
preserveCheckoutMock.mockResolvedValue({ archiveDir: '/archive', verify: async () => 'checkout-content-changed' });
const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');