Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12748]): archive agent-cleanup and orphan checkouts outside the claim-ownership lock
[._.]/ Adam Eivy26b60dc05fae
diff --git a/server/services/branchReconcile.js b/server/services/branchReconcile.jsindex f897fd6a44..215757801a 100644--- a/server/services/branchReconcile.js+++ b/server/services/branchReconcile.js@@ -1272,7 +1272,9 @@ export async function reapSupersededBranches(repoPath, defaultBranch, superseded* `prepare` runs after every gate has passed and before the first irreversible* step, and aborts the retirement by resolving to `{ error }`. That is the only* place work like "write a recoverable backup" belongs: earlier it is paid for- * branches that are then refused, later it is too late to matter.+ * branches that are then refused, later it is too late to matter. It runs+ * with the claim-ownership lock released and the branch fenced, and removal+ * then refuses a worktree whose dirty paths no longer match `b.dirtyPaths`.** `requireCleanWorktree` (default true) is the caller's answer to "could this* tree hold work that is not accounted for?". For `cleanupMerged` it can, so a@@ -1328,15 +1330,19 @@ async function retireBranch(repoPath, b, opts) {allowLiveClaim: true,});if (gated.result) return gated;- if (!gated.archiveTree) return { result: await finishBranchRetirement(repoPath, gated, null) };- return { fence: { repoPath, path: gated.archiveTree.path, branch: gated.b.branch }, gated };+ if (!gated.archiveTree && !gated.prepare) return { result: await finishBranchRetirement(repoPath, gated) };+ return { fence: { repoPath, path: gated.archiveTree?.path ?? gated.b.worktreePath, branch: gated.b.branch }, gated };},+ // The superseded backup copies dirty files, so it takes this window too (#12748).archive: async ({ gated }) => {+ const prepared = gated.prepare ? await gated.prepare() : null;+ if (prepared?.error) return { result: { ok: false, reason: prepared.error } };+ if (!gated.archiveTree) return { archive: { prepared } };const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');- return preserveMergedCheckout(repoPath, gated.archiveTree, gated.mergeTarget).then(archive => ({ archive }),+ return preserveMergedCheckout(repoPath, gated.archiveTree, gated.mergeTarget).then(preservation => ({ archive: { prepared, preservation } }),error => ({ result: { ok: false, reason: 'archive-failed', detail: error.message } }));},- finish: ({ gated }, preservation) => finishBranchRetirement(repoPath, gated, preservation),+ finish: ({ gated }, { prepared, preservation }) => finishBranchRetirement(repoPath, gated, { prepared, preservation }),});}@@ -1411,10 +1417,11 @@ async function admitBranchRetirement(repoPath, b, { activeAgentIds = new Set(),return { b, expectedHead, readPolicyHold, archiveTree, mergeTarget, label, requireCleanWorktree, prepare, claimOwners, mergedRetirement };}-/** Removal and branch deletion; `preservation` is the archive taken after admission, if any. */-async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHold, mergeTarget, label, requireCleanWorktree, prepare, claimOwners, mergedRetirement }, preservation) {- const prepared = prepare ? await prepare() : null;- if (prepared?.error) return { ok: false, reason: prepared.error };+/**+ * Removal and branch deletion. `preservation` is the archive and `prepared` the+ * `prepare` backup, each taken after admission with the ownership lock released.+ */+async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHold, mergeTarget, label, requireCleanWorktree, claimOwners, mergedRetirement }, { prepared = null, preservation = null } = {}) {if (b.worktreePath) {const verifyRemoval = async (holder) => {if (!holder || holder.branch !== `refs/heads/${b.branch}` || holder.head !== expectedHead) return 'worktree-holder-changed';@@ -1433,9 +1440,11 @@ async function finishBranchRetirement(repoPath, { b, expectedHead, readPolicyHolrepoPath, branch: b.branch, path: b.worktreePath, head: expectedHead,})) return 'claim-completion-unverified';}- const dirty = await isWorktreeDirty(b.worktreePath);- if (dirty === null) return 'worktree-status-unavailable';- if (requireCleanWorktree && dirty) return 'worktree-dirty';+ const dirtyPaths = await worktreeDirtyPaths(b.worktreePath);+ if (dirtyPaths === null) return 'worktree-status-unavailable';+ if (requireCleanWorktree && dirtyPaths.length) return 'worktree-dirty';+ // The backup ran unlocked; a tree whose change set moved since is not what it holds.+ if (prepared && !sameDirtyPaths(b.dirtyPaths, dirtyPaths)) return 'worktree-changed-since-backup';if (mergedRetirement && !await hasBranchMergeEvidence(repoPath, expectedHead, mergeTarget).catch(() => false)) return 'not-merged-on-recheck';return preservation ? preservation.verify(holder) : null;};diff --git a/server/services/branchReconcile.test.js b/server/services/branchReconcile.test.jsindex 6b24e6de46..eb067f4cd8 100644--- a/server/services/branchReconcile.test.js+++ b/server/services/branchReconcile.test.js@@ -1613,6 +1613,51 @@ describe('reconcile — cached SUPERSEDED verdicts (#3842)', () => {expect(wt.deleteCompletedLocalBranch).not.toHaveBeenCalled();});+ // #12748: the backup copies the dirty tree, and it ran inside the shared+ // claim-ownership lock that every bind/release/reserve queues behind.+ it('backs up an opted-in superseded checkout with the ownership queue free and only it fenced', async () => {+ setupAbandoned();+ tryReadFileMock.mockResolvedValue(ledger());+ let started;+ let finish;+ const backingUp = new Promise(resolve => { started = resolve; });+ backupSupersededBranchMock.mockImplementationOnce(() => new Promise(resolve => {+ finish = () => resolve({ dir: '/backups/x', manifest: 'manifest.json', untracked: [] });+ started();+ }));+ const pending = reconcile('/repo', { reapSuperseded: true, activeAgentIds: new Set() });+ await backingUp;+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', path: WORKTREE })).toBe('checkout-retirement-reserved');+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: BRANCH })).toBe('checkout-retirement-reserved');+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'feature/unrelated', path: '/repo/data/cos/worktrees/other' })).toBe(null);+ finish();+ expect((await pending).reapedSuperseded).toEqual([BRANCH]);+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: BRANCH })).toBe(null);+ });++ it('retains an opted-in superseded checkout whose change set moves during its unlocked backup', async () => {+ setupAbandoned();+ tryReadFileMock.mockResolvedValue(ledger());+ const scripted = execGit.getMockImplementation();+ let backedUp = false;+ execGit.mockImplementation(async (args, ...rest) => (backedUp && args[0] === 'status'+ ? { stdout: `${DIRTY}?? notes.md\n`, exitCode: 0 }+ : scripted(args, ...rest)));+ backupSupersededBranchMock.mockImplementation(async () => {+ backedUp = true;+ return { dir: '/backups/x', manifest: 'manifest.json', untracked: [] };+ });+ wt.forceRemoveWorktreeDir.mockImplementation(async (_repo, _path, options) => {+ const reason = await options.verifyRemoval({ head: RETIRE_HEAD, path: WORKTREE, branch: `refs/heads/${BRANCH}` });+ return { removed: !reason, reason };+ });+ const res = await reconcile('/repo', { reapSuperseded: true, activeAgentIds: new Set() });+ expect(res.reapedSuperseded).toEqual([]);+ expect(res.skipped).toContainEqual({ branch: BRANCH, reason: 'worktree-changed-since-backup' });+ expect(wt.deleteCompletedLocalBranch).not.toHaveBeenCalled();+ });+it('retains an explicitly opted-in superseded ref when its deletion lease fails', async () => {setupAbandoned();tryReadFileMock.mockResolvedValue(ledger());diff --git a/server/services/cosClaimOwnershipState.js b/server/services/cosClaimOwnershipState.jsindex 8dfe16c3f3..fd8b9a224f 100644--- a/server/services/cosClaimOwnershipState.js+++ b/server/services/cosClaimOwnershipState.js@@ -33,14 +33,15 @@ export function checkoutRetirementFenceReason({ repoPath, path = null, branch =}/**- * Fence one checkout while it is archived outside the lock. Call while holding+ * Fence one checkout (or, with no `path`, one checkout-less branch) while it is+ * archived outside the lock. Call while holding* `withClaimOwnershipLock`, after the ownership checks pass, and call the returned* release under the lock once removal settles (it is idempotent, so a `finally`* may repeat it). Returns null when the checkout is already fenced.*/export function reserveCheckoutRetirementFence({ repoPath, path, branch = null }) {if (checkoutRetirementFenceReason({ repoPath, path, branch })) return null;- const fence = { repo: fencePath(repoPath), path: fencePath(path), branch: fenceBranch(branch) };+ const fence = { repo: fencePath(repoPath), path: path ? fencePath(path) : null, branch: fenceBranch(branch) };retirementFences.add(fence);return () => { retirementFences.delete(fence); };}diff --git a/server/services/worktreeExternalCleanup.test.js b/server/services/worktreeExternalCleanup.test.jsindex e8922b5b88..6b40c901b8 100644--- a/server/services/worktreeExternalCleanup.test.js+++ b/server/services/worktreeExternalCleanup.test.js@@ -10,6 +10,16 @@ import { materializeGitRepo, resetGitWorktreeSandbox, SKIP_HEAVY_INTEGRATION } fimport { PATHS } from '../lib/fileUtils.js';import { execGit } from '../lib/execGit.js';import { cleanupOrphanedWorktrees } from './worktreeManager.js';+import { withClaimOwnershipLock, checkoutRetirementFenceReason } from './cosClaimOwnershipState.js';++const archiveGate = vi.hoisted(() => ({ beforeArchive: null }));+vi.mock('./heldCheckoutRecovery.js', async importOriginal => {+ const actual = await importOriginal();+ return { ...actual, preserveMergedCheckout: async (...args) => {+ await archiveGate.beforeArchive?.(...args);+ return actual.preserveMergedCheckout(...args);+ } };+});vi.mock('fs', async (importOriginal) => {const actual = await importOriginal();@@ -104,6 +114,7 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('external orphan cleanup recovery policyfault.aliasTarget = null;fault.registrationPath = null;fault.onGit = null;+ archiveGate.beforeArchive = null;await resetGitWorktreeSandbox(external, initialHead);await rm(PATHS.worktrees, { recursive: true, force: true });await mkdir(PATHS.worktrees, { recursive: true });@@ -137,6 +148,26 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('external orphan cleanup recovery policyexpect(await branchExists('agent-merged')).toBe(false);});+ // #12748: the archive ran inside the shared claim-ownership lock.+ it('archives with the ownership queue free while fencing only the reclaimed checkout', async () => {+ const path = await addTree('agent-fenced', { committed: true });+ await execGit(['merge', '--no-ff', 'agent-fenced', '--no-edit'], external);+ let started;+ let resume;+ const archiving = new Promise(resolve => { started = resolve; });+ // Git spells the parent repository its own way on Windows; fence what it archives.+ archiveGate.beforeArchive = repoPath => new Promise(resolve => { resume = resolve; started(repoPath); });+ const cleaning = cleanupOrphanedWorktrees(primary, new Set(), NO_OWNERS);+ const repoPath = await archiving;+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-fenced' })).toBe('checkout-retirement-reserved');+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-unrelated' })).toBe(null);+ resume();+ expect(await cleaning).toBe(1);+ expect(existsSync(path)).toBe(false);+ expect(checkoutRetirementFenceReason({ repoPath, branch: 'agent-fenced' })).toBe(null);+ });+it('matches Git registration spelling when Node retains another alias for the same checkout', async () => {const path = await addTree('agent-path-alias');const aliasRoot = join(root(), 'git-spelling');diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.jsindex e21c6cf2cd..8b006d72e8 100644--- a/server/services/worktreeManager.js+++ b/server/services/worktreeManager.js@@ -1385,31 +1385,35 @@ async function cleanupWorktreeBranch(sourceWorkspace, branchName, {* run, forever. ONLY the discard path may pass this.*/export async function removeWorktree(agentId, sourceWorkspace, branchName, options = {}) {- const { withClaimOwnershipLock, readClaimOwnershipAgents, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');- // A retirement archiving this checkout with the lock released owns it until removal.- const ownershipHold = async () => checkoutRetirementFenceReason({ repoPath: sourceWorkspace, path: join(WORKTREES_DIR, agentId), branch: branchName })- || claimCheckoutOwnerReason({- branchName, holderPath: join(WORKTREES_DIR, agentId), sourceWorkspace,- agents: await readClaimOwnershipAgents().catch(() => null), ignoreIds: [agentId],- });+ const { readClaimOwnershipAgents } = await import('./cosClaimOwnershipState.js');+ const ownershipHold = async () => claimCheckoutOwnerReason({+ branchName, holderPath: join(WORKTREES_DIR, agentId), sourceWorkspace,+ agents: await readClaimOwnershipAgents().catch(() => null), ignoreIds: [agentId],+ });const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');- return withClaimOwnershipLock(() => withCheckoutPublication(sourceWorkspace,- () => removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold)));-+ return withCheckoutPublication(sourceWorkspace,+ () => removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold));}+/**+ * Runs cleanup as claim-ownership critical sections with every archive between two+ * of them (#12748): each archive hashes and copies the whole checkout, and every+ * bind/release/reserve queues behind that lock on a 15 s budget. While an archive+ * runs this checkout is fenced; each section re-reads the owners before it acts.+ * `ownershipHold` reads registered owners only — this function owns the fence check.+ */async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options, ownershipHold) {+ const { withClaimOwnershipLock, checkoutRetirementFenceReason, retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');const worktreePath = join(WORKTREES_DIR, agentId);+ const fence = { repoPath: sourceWorkspace, path: worktreePath, branch: branchName };const warnings = [];- const { getDefaultBranch } = await import('./git.js');- const defaultBranch = await getDefaultBranch(sourceWorkspace, { strict: true }).catch(() => null);- const protectedReason = PROTECTED_BRANCHES.includes(branchName) || branchName === defaultBranch- ? 'protected-branch' : !defaultBranch ? 'default-branch-unavailable' : null;- if (protectedReason) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false,- warnings: [`Worktree preserved — ${protectedReason}`] };- const owner = await ownershipHold();- if (owner) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false,- warnings: [`Worktree preserved — ${owner}`] };+ let merged = false;+ let commitsAhead = 0;+ let mergeRefused = false;+ const held = reason => {+ warnings.push(`Worktree preserved — ${reason}`);+ return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };+ };const completionHold = async (expectedHead) => {const owner = await ownershipHold();if (owner) return owner;@@ -1434,95 +1438,102 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,const archive = await preserveMergedCheckout(sourceWorkspace, tree, null);return { ...archive, head: tree.head,verify: async holder => await completionHold(tree.head) || await archive.verify(holder) };-};- const preserveOrHold = () => preserve().catch(error => {- warnings.push(`Worktree preserved — archive failed: ${error.message}`);- return null;+ // One archive between two critical sections. `admit` (under the lock, after the+ // owner re-read) returns a result to stop before fencing; `finish` runs under the+ // lock again with the fence still held.+ const retire = ({ admit = async () => null, onArchiveFailure, finish }) => retireWithUnlockedArchive({+ busy: () => held('checkout-retirement-reserved'),+ admit: async () => {+ const owner = await ownershipHold();+ if (owner) return { result: held(owner) };+ const stop = await admit();+ return stop ? { result: stop } : { fence };+ },+ archive: () => preserve().then(archive => ({ archive }), error => {+ warnings.push(`Worktree preserved — archive failed: ${error.message}`);+ return { result: onArchiveFailure() };+ }),+ finish: (_admitted, preservation) => finish(preservation),});-- if (!existsSync(worktreePath)) {- // The directory is gone, but git may still register it (a stale entry pins the- // branch). Verify rather than assume; prune when the registration survives.- let gone = removalOutcome(worktreePath, await probeWorktreeRemoval(sourceWorkspace, worktreePath));- let preservation = null;- if (!gone.removed) {- preservation = await preserveOrHold();- if (!preservation) return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };- gone = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, { verifyRemoval: preservation.verify });- }- if (!gone.removed) {- warnings.push(gone.warning);- return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };- }- console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);+ const cleanupBranch = async (expectedHead) => {const finalOwner = await ownershipHold();- if (finalOwner) return { merged: false, removed: true, uncommittedSaved: false,- warnings: [...warnings, `Branch ${branchName} preserved — ${finalOwner}`] };+ if (finalOwner) {+ warnings.push(`Branch ${branchName} preserved — ${finalOwner}`);+ return false;+ }await cleanupWorktreeBranch(sourceWorkspace, branchName, {- warnings, expectedHead: preservation?.head,+ hasUnmergedCommits: options.merge && !merged && (commitsAhead > 0 || mergeRefused), warnings, expectedHead,});- return { merged: false, removed: true, uncommittedSaved: false, warnings };- }+ return true;+ };- // Verify the worktree still points to the correct repo before trusting git status.- // If the .git file is missing (e.g., worktree was partially cleaned up), git walks up- // the directory tree and may find a parent repo (e.g., PortOS) instead of the app repo.- // In that case, even a successful git status describes the wrong checkout and- // cannot prove this directory has no unique work.- const detectedToplevel = await execGit(['rev-parse', '--show-toplevel'], worktreePath)- .then(r => r.stdout.trim())- .catch(() => null);- // Compare realpath-resolved forms so symlinks (e.g. macOS /var → /private/var)- // or normalization differences don't false-positive as a broken worktree.- if (detectedToplevel && !pathsEqual(detectedToplevel, worktreePath)) {- warnings.push('Worktree preserved — broken checkout identity prevents verifying its contents');- console.warn(`⚠️ Worktree ${agentId} has a broken checkout identity; preserving its directory and branch`);- return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };- }+ const inspected = await withClaimOwnershipLock(async () => {+ const { getDefaultBranch } = await import('./git.js');+ const defaultBranch = await getDefaultBranch(sourceWorkspace, { strict: true }).catch(() => null);+ const protectedReason = PROTECTED_BRANCHES.includes(branchName) || branchName === defaultBranch+ ? 'protected-branch' : !defaultBranch ? 'default-branch-unavailable' : null;+ if (protectedReason) return { result: held(protectedReason) };+ // A retirement archiving this checkout with the lock released owns it until removal.+ const owner = checkoutRetirementFenceReason(fence) || await ownershipHold();+ if (owner) return { result: held(owner) };++ if (!existsSync(worktreePath)) {+ // The directory is gone, but git may still register it (a stale entry pins the+ // branch). Verify rather than assume; prune when the registration survives.+ const gone = removalOutcome(worktreePath, await probeWorktreeRemoval(sourceWorkspace, worktreePath));+ if (!gone.removed) return { missing: true };+ console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);+ await cleanupBranch(undefined);+ return { result: { merged: false, removed: true, uncommittedSaved: false, warnings } };+ }- // Safety check: abort removal when uncommitted changes are detected.- // Also fail closed if git status itself fails — treat unknown state as dirty.- let dirtyFiles;- try {- dirtyFiles = (await execGit(['status', '--porcelain'], worktreePath)).stdout.trim();- } catch (err) {- console.log(`⚠️ git status failed for worktree ${agentId}, preserving to avoid data loss: ${err.message}`);- warnings.push(`Worktree preserved — git status failed: ${err.message}`);- return { merged: false, removed: false, uncommittedSaved: false, warnings };- }- // The completion sentinel is not authored work: it has already been consumed- // by finalizeAgent before this cleanup runs. Ignore it for the preservation- // decision, while still preserving the tree if any real change remains; the- // eventual forced worktree removal discards it with the rest of the completed- // checkout. The run's canonical name and its bounded truncation-recovery- // name, plus the legacy shared one a pre-upgrade run may have left — and no- // other agent's.- const dirt = classifyWorktreeDirt(dirtyFiles, {- ignoredPaths: [DONE_SENTINEL_NAME, ...doneSentinelCandidateNames(agentId)]- });- if (!dirt.clean && options.discardDirt) {- // Throwaway posture: the caller has already established that nothing in this- // tree is wanted. Log what is being dropped so a surprising loss is at least- // traceable, then fall through to removal.- const shown = dirt.realChangePaths.slice(0, DIRT_PATHS_IN_WARNING);- if (shown.length) console.log(`🌳 Discarding uncommitted changes in throwaway worktree ${agentId}: ${shown.join(', ')}`);- } else if (!dirt.clean) {- // Discard auto-generated lockfile changes that agents don't intend to commit- // (e.g., npm install resolving ^version to exact version in package-lock.json)- if (dirt.lockfileOnly) {- // Preserve the original bytes before the intentional lockfile cleanup.- // A second snapshot below guards the post-cleanup removal boundary.- if (!await preserveOrHold()) return { merged: false, removed: false, uncommittedSaved: false, warnings };- console.log(`🧹 Discarding ${dirt.lockfilePaths.length} auto-generated lockfile change(s) in worktree ${agentId}`);- // Never throws past removeWorktree: an undiscardable lockfile keeps the tree.- const discarded = await discardLockfileChanges(worktreePath, dirt).then(() => true, err => {- console.log(`⚠️ Lockfile discard failed in worktree ${agentId}, preserving to avoid data loss: ${err.message}`);- warnings.push(`Worktree preserved — lockfile discard failed in ${worktreePath}: ${err.message}`);- return false;- });- if (!discarded) return { merged: false, removed: false, uncommittedSaved: false, warnings };- } else {+ // Verify the worktree still points to the correct repo before trusting git status.+ // If the .git file is missing (e.g., worktree was partially cleaned up), git walks up+ // the directory tree and may find a parent repo (e.g., PortOS) instead of the app repo.+ // In that case, even a successful git status describes the wrong checkout and+ // cannot prove this directory has no unique work.+ const detectedToplevel = await execGit(['rev-parse', '--show-toplevel'], worktreePath)+ .then(r => r.stdout.trim())+ .catch(() => null);+ // Compare realpath-resolved forms so symlinks (e.g. macOS /var → /private/var)+ // or normalization differences don't false-positive as a broken worktree.+ if (detectedToplevel && !pathsEqual(detectedToplevel, worktreePath)) {+ warnings.push('Worktree preserved — broken checkout identity prevents verifying its contents');+ console.warn(`⚠️ Worktree ${agentId} has a broken checkout identity; preserving its directory and branch`);+ return { result: { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings } };+ }++ // Safety check: abort removal when uncommitted changes are detected.+ // Also fail closed if git status itself fails — treat unknown state as dirty.+ let dirtyFiles;+ try {+ dirtyFiles = (await execGit(['status', '--porcelain'], worktreePath)).stdout.trim();+ } catch (err) {+ console.log(`⚠️ git status failed for worktree ${agentId}, preserving to avoid data loss: ${err.message}`);+ warnings.push(`Worktree preserved — git status failed: ${err.message}`);+ return { result: { merged: false, removed: false, uncommittedSaved: false, warnings } };+ }+ // The completion sentinel is not authored work: it has already been consumed+ // by finalizeAgent before this cleanup runs. Ignore it for the preservation+ // decision, while still preserving the tree if any real change remains; the+ // eventual forced worktree removal discards it with the rest of the completed+ // checkout. The run's canonical name and its bounded truncation-recovery+ // name, plus the legacy shared one a pre-upgrade run may have left — and no+ // other agent's.+ const dirt = classifyWorktreeDirt(dirtyFiles, {+ ignoredPaths: [DONE_SENTINEL_NAME, ...doneSentinelCandidateNames(agentId)]+ });+ if (!dirt.clean && options.discardDirt) {+ // Throwaway posture: the caller has already established that nothing in this+ // tree is wanted. Log what is being dropped so a surprising loss is at least+ // traceable, then fall through to removal.+ const shown = dirt.realChangePaths.slice(0, DIRT_PATHS_IN_WARNING);+ if (shown.length) console.log(`🌳 Discarding uncommitted changes in throwaway worktree ${agentId}: ${shown.join(', ')}`);+ } else if (!dirt.clean) {+ // Auto-generated lockfile changes agents don't intend to commit (e.g. npm+ // install resolving ^version to an exact version) are discarded below.+ if (dirt.lockfileOnly) return { lockfile: { dirt, dirtyFiles } };// NAME the dirty paths. The bare message reads as "your work was left// behind" with no way to tell real uncommitted work from a transient// observation, and the worktree is usually gone by the time anyone looks —@@ -1533,29 +1544,71 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,const detail = `${shown.join(', ')}${more > 0 ? ` (+${more} more)` : ''}`;console.log(`⚠️ Preserving worktree for ${agentId} — uncommitted changes detected (${detail}), aborting cleanup to avoid data loss`);warnings.push(`Worktree preserved — uncommitted changes detected in ${worktreePath}: ${detail}`);- return { merged: false, removed: false, uncommittedSaved: false, warnings };+ return { result: { merged: false, removed: false, uncommittedSaved: false, warnings } };}+ return {};+ });+ if (inspected.result) return inspected.result;++ if (inspected.missing) {+ return retire({+ onArchiveFailure: () => ({ merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings }),+ finish: async (preservation) => {+ const gone = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, { verifyRemoval: preservation.verify });+ if (!gone.removed) {+ warnings.push(gone.warning);+ return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };+ }+ console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);+ await cleanupBranch(preservation.head);+ return { merged: false, removed: true, uncommittedSaved: false, warnings };+ },+ });}- let merged = false;- let commitsAhead = 0;- let mergeRefused = false;+ if (inspected.lockfile) {+ const { dirt, dirtyFiles } = inspected.lockfile;+ // Preserve the original bytes before the intentional lockfile cleanup.+ // The second snapshot below guards the post-cleanup removal boundary.+ const discardFailure = await retire({+ onArchiveFailure: () => ({ merged: false, removed: false, uncommittedSaved: false, warnings }),+ finish: async () => {+ // The snapshot ran with the lock released; discard only the tree it captured.+ const current = await execGit(['status', '--porcelain'], worktreePath).then(r => r.stdout.trim(), () => null);+ if (current !== dirtyFiles) {+ warnings.push(`Worktree preserved — ${worktreePath} changed while its pre-cleanup snapshot was archived`);+ return { merged: false, removed: false, uncommittedSaved: false, warnings };+ }+ console.log(`🧹 Discarding ${dirt.lockfilePaths.length} auto-generated lockfile change(s) in worktree ${agentId}`);+ // Never throws past removeWorktree: an undiscardable lockfile keeps the tree.+ return discardLockfileChanges(worktreePath, dirt).then(() => null, err => {+ console.log(`⚠️ Lockfile discard failed in worktree ${agentId}, preserving to avoid data loss: ${err.message}`);+ warnings.push(`Worktree preserved — lockfile discard failed in ${worktreePath}: ${err.message}`);+ return { merged: false, removed: false, uncommittedSaved: false, warnings };+ });+ },+ });+ if (discardFailure) return discardFailure;+ }- if (options.merge) {- const currentBranch = (await execGit(- ['rev-parse', '--abbrev-ref', 'HEAD'],- sourceWorkspace- )).stdout.trim();+ return retire({+ admit: async () => {+ if (!options.merge) return null;+ const currentBranch = (await execGit(+ ['rev-parse', '--abbrev-ref', 'HEAD'],+ sourceWorkspace+ )).stdout.trim();- // Defense-in-depth: NEVER merge the agent branch into a non-default branch.- // See shouldRefuseDefaultBranchMerge for the rationale.- const { getDefaultBranch } = await import('./git.js');- const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null);- if (shouldRefuseDefaultBranchMerge(currentBranch, defaultBranch)) {- console.log(`🌳 Refusing auto-merge of ${branchName} into '${currentBranch}' (default branch is '${defaultBranch || 'unknown'}'). Use \`gh pr merge\` for non-default targets.`);- warnings.push(`Auto-merge skipped — source repo HEAD is on '${currentBranch}', not default '${defaultBranch || 'unknown'}'. Branch ${branchName} preserved for manual review.`);- mergeRefused = true;- } else {+ // Defense-in-depth: NEVER merge the agent branch into a non-default branch.+ // See shouldRefuseDefaultBranchMerge for the rationale.+ const { getDefaultBranch } = await import('./git.js');+ const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null);+ if (shouldRefuseDefaultBranchMerge(currentBranch, defaultBranch)) {+ console.log(`🌳 Refusing auto-merge of ${branchName} into '${currentBranch}' (default branch is '${defaultBranch || 'unknown'}'). Use \`gh pr merge\` for non-default targets.`);+ warnings.push(`Auto-merge skipped — source repo HEAD is on '${currentBranch}', not default '${defaultBranch || 'unknown'}'. Branch ${branchName} preserved for manual review.`);+ mergeRefused = true;+ return null;+ }const countResult = await execGit(['rev-list', '--count', `${currentBranch}..${branchName}`],sourceWorkspace,@@ -1585,42 +1638,32 @@ async function removeWorktreeNow(agentId, sourceWorkspace, branchName, options,warnings.push(`Auto-merge failed for branch ${branchName} — branch preserved for manual recovery`);});}- }- }-- const preservation = await preserveOrHold();- if (!preservation) return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };- const removal = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, {- verifyRemoval: preservation.verify,- label: `Worktree remove failed for ${agentId}, falling back to manual cleanup`,- log: 'all',- subject: agentId,- });- if (!removal.removed) {- // The checkout (or its registration) survived: the branch is still held, so- // deleting it would fail, and `removed: true` would tell the caller it may- // queue work onto a branch the old worktree still owns. Report the truth with- // a retryable warning; `merged` stays accurate for the publication outcome.- console.warn(`⚠️ Worktree for ${agentId} was not removed — branch ${branchName} kept`);- warnings.push(removal.warning);- return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };- }+ return null;+ },+ onArchiveFailure: () => ({ merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings }),+ finish: async (preservation) => {+ const removal = await forceRemoveWorktreeDir(sourceWorkspace, worktreePath, {+ verifyRemoval: preservation.verify,+ label: `Worktree remove failed for ${agentId}, falling back to manual cleanup`,+ log: 'all',+ subject: agentId,+ });+ if (!removal.removed) {+ // The checkout (or its registration) survived: the branch is still held, so+ // deleting it would fail, and `removed: true` would tell the caller it may+ // queue work onto a branch the old worktree still owns. Report the truth with+ // a retryable warning; `merged` stays accurate for the publication outcome.+ console.warn(`⚠️ Worktree for ${agentId} was not removed — branch ${branchName} kept`);+ warnings.push(removal.warning);+ return { merged, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };+ }- // Merge evidence and completion policy were checked before removal; the- // branch helper checks them again before its leased local deletion.- const hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);- const finalOwner = await ownershipHold();- if (finalOwner) {- warnings.push(`Branch ${branchName} preserved — ${finalOwner}`);- return { merged, removed: true, uncommittedSaved: false, warnings };- }- await cleanupWorktreeBranch(sourceWorkspace, branchName, {- hasUnmergedCommits, warnings, expectedHead: preservation.head,+ // Merge evidence and completion policy were checked before removal; the+ // branch helper checks them again before its leased local deletion.+ if (await cleanupBranch(preservation.head)) console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);+ return { merged, removed: true, uncommittedSaved: false, warnings };+ },});-- console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);-- return { merged, removed: true, uncommittedSaved: false, warnings };}/**@@ -1839,23 +1882,6 @@ function activeIdsReader(activeIds) {return typeof activeIds === 'function' ? activeIds : () => activeIds;}-/**- * Remove an orphan only if it is STILL unowned: liveness re-read and the agent- * registry checked inside the lock a claim bind takes, so neither an agent- * dispatched since the caller's last check nor a registered claim/agent owner- * can be removed out from under it. Resolves `remove()`'s result, or null when held.- */-async function removeIfStillOrphaned({ isOwned, branchName, holderPath, sourceWorkspace, readOwners }, remove) {- const { withClaimOwnershipLock, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');- return withClaimOwnershipLock(async () => {- if (isOwned() || checkoutRetirementFenceReason({ repoPath: sourceWorkspace, path: holderPath, branch: branchName })) return null;- const agents = await readOwners();- if (claimCheckoutOwnerReason({ branchName, holderPath, sourceWorkspace, agents })) return null;- const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');- return withCheckoutPublication(sourceWorkspace, remove);- });-}-/*** Clean up any orphaned worktrees (worktrees whose agent no longer exists)*@@ -1871,6 +1897,7 @@ export async function cleanupOrphanedWorktrees(sourceWorkspace, getActiveIds, {const readActiveIds = activeIdsReader(getActiveIds);const { readClaimOwnershipAgents } = await import('./cosClaimOwnershipState.js');+ const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');const readOwners = claimOwnerReader(claimOwners, readClaimOwnershipAgents);const worktrees = await listWorktrees(sourceWorkspace).catch(() => []);let cleaned = 0;@@ -1896,9 +1923,10 @@ export async function cleanupOrphanedWorktrees(sourceWorkspace, getActiveIds, {const branchName = wt.branch.replace('refs/heads/', '');// Attempt merge so committed work from preserved worktrees (e.g., PR/push failures) isn't lost.- // If merge fails, the branch is preserved for manual recovery.- const result = await removeIfStillOrphaned(- { isOwned: ownershipReason, branchName, holderPath: wt.path, sourceWorkspace, readOwners },+ // If merge fails, the branch is preserved for manual recovery. Every critical+ // section re-reads liveness and the registry, so neither an agent dispatched+ // since the check above nor a registered claim owner loses its checkout.+ const result = await withCheckoutPublication(sourceWorkspace,() => removeWorktreeNow(agentId, sourceWorkspace, branchName, { merge: true }, async () =>ownershipReason() || claimCheckoutOwnerReason({ branchName, holderPath: wt.path, sourceWorkspace, agents: await readOwners() })),).catch(err => {@@ -2095,6 +2123,7 @@ export async function reapMergedWorktrees(sourceWorkspace, {*/async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readOwners) {const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');+ const { retireWithUnlockedArchive } = await import('./cosClaimOwnershipState.js');const entries = await readdir(WORKTREES_DIR, { withFileTypes: true }).catch(() => []);let cleaned = 0;@@ -2146,12 +2175,21 @@ async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readO// Non-force removal is a final Git guard against newly dirty or locked// trees. A refusal must never fall through to recursive filesystem removal.+ // The archive runs between two ownership critical sections (#12748) with+ // this checkout fenced; both re-read liveness and the registry.const { withCheckoutPublication } = await import('../lib/checkoutRetirementAdmission.js');- const removed = await withCheckoutPublication(parentRepo, () => removeIfStillOrphaned(- { isOwned: () => ownership({ locked: wt.locked }), branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, readOwners },- async () => {+ const removed = await withCheckoutPublication(parentRepo, () => retireWithUnlockedArchive({+ busy: () => false,+ admit: async () => {+ if (ownership({ locked: wt.locked })+ || claimCheckoutOwnerReason({ branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, agents: await readOwners() })) return { result: false };+ return { fence: { repoPath: parentRepo, path: worktreePath, branch: branchName } };+ },+ archive: async () => {const { preserveMergedCheckout } = await import('./heldCheckoutRecovery.js');- const preservation = await preserveMergedCheckout(parentRepo, wt, target);+ return { archive: await preserveMergedCheckout(parentRepo, wt, target) };+ },+ finish: async (_admitted, preservation) => {const latest = (await listWorktrees(parentRepo)).find(holder => pathsEqual(holder.path, worktreePath));if (ownership({ locked: latest?.locked }) || await preservation.verify(latest)|| claimCheckoutOwnerReason({ branchName, holderPath: worktreePath, sourceWorkspace: parentRepo, agents: await readOwners() })@@ -2167,7 +2205,7 @@ async function cleanupExternalRepoWorktrees(readActiveIds, alreadyHandled, readO}return ok;},- )).catch(err => {+ })).catch(err => {console.warn(`⚠️ Preserved external worktree ${entry.name}: ${err.message}`);return false;});diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.jsindex cd19dd861b..91afc12457 100644--- a/server/services/worktreeManager.test.js+++ b/server/services/worktreeManager.test.js@@ -1184,6 +1184,49 @@ describe('removeWorktree identity, dirt and branch preservation', () => {expect(rm).not.toHaveBeenCalled();});+ // #12748: every agent completion archived its checkout inside the shared+ // claim-ownership lock, starving each bind/release/reserve queued behind it.+ it.each(['present', 'missing', 'lockfile'])('archives a %s checkout with the ownership queue free and only it fenced', async state => {+ if (state === 'missing') existsSync.mockReturnValue(false);+ if (state === 'lockfile') scriptGit({ porcelain: ' M package-lock.json' });+ const { withClaimOwnershipLock, checkoutRetirementFenceReason } = await import('./cosClaimOwnershipState.js');+ const fenced = { repoPath: '/repo', path: join(PATHS.worktrees, 'agent-x') };+ let started;+ let finish;+ const archiving = new Promise(resolve => { started = resolve; });+ preserveCheckoutMock.mockImplementationOnce(() => new Promise(resolve => {+ finish = () => resolve({ archiveDir: '/archive', verify: async () => null });+ started();+ }));+ const pending = removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');+ await archiving;+ expect(await withClaimOwnershipLock(() => 'settled')).toBe('settled');+ expect(checkoutRetirementFenceReason(fenced)).toBe('checkout-retirement-reserved');+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'cos/task-1/agent-x' })).toBe('checkout-retirement-reserved');+ expect(checkoutRetirementFenceReason({ repoPath: '/repo', branch: 'cos/task-2/agent-y', path: join(PATHS.worktrees, 'agent-y') })).toBe(null);+ finish();+ expect((await pending).removed).toBe(true);+ expect(checkoutRetirementFenceReason(fenced)).toBe(null);+ });++ it('keeps a lockfile-only checkout that changed while its pre-cleanup snapshot was archived', async () => {+ scriptGit({ porcelain: ' M package-lock.json' });+ const scripted = execGitMock.getMockImplementation();+ let archived = false;+ execGitMock.mockImplementation((args, ...rest) => (archived && args[0] === 'status'+ ? Promise.resolve({ stdout: ' M package-lock.json\n?? notes.md', stderr: '', exitCode: 0 })+ : scripted(args, ...rest)));+ preserveCheckoutMock.mockImplementationOnce(async () => {+ archived = true;+ return { archiveDir: '/archive', verify: async () => null };+ });+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');+ expect(result.removed).toBe(false);+ expect(result.warnings.join(' ')).toContain('changed while its pre-cleanup snapshot was archived');+ expect(execGitMock.mock.calls.some(([args]) => args[0] === 'checkout' || args[0] === 'push'+ || (args[0] === 'worktree' && args[1] === 'remove'))).toBe(false);+ });+it('retains checkout and ref when preserved bytes change before removal', async () => {preserveCheckoutMock.mockResolvedValue({ archiveDir: '/archive', verify: async () => 'checkout-content-changed' });const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x');