Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(cos): preserve partially removed worktrees on resume (#12512)

[._.]/ Adam Eivyac079cb8c188
diff --git a/server/services/worktreeManager.forceRemove.test.js b/server/services/worktreeManager.forceRemove.test.js
index 921dd5336b..07ff0f23cc 100644
--- a/server/services/worktreeManager.forceRemove.test.js
+++ b/server/services/worktreeManager.forceRemove.test.js
@@ -21,7 +21,7 @@ vi.mock('../lib/execGit.js', async (importOriginal) => {
});
import { execGit } from '../lib/execGit.js';
-import { forceRemoveWorktreeDir } from './worktreeManager.js';
+import { forceRemoveWorktreeDir, WORKTREE_ADD_TIMEOUT_MS } from './worktreeManager.js';
describe.skipIf(SKIP_HEAVY_INTEGRATION)('forceRemoveWorktreeDir', () => {
let repo;
@@ -59,6 +59,9 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('forceRemoveWorktreeDir', () => {
const path = await addWorktree();
const outcome = await forceRemoveWorktreeDir(repo, path, { verifyRemoval: async () => null });
expect(outcome).toMatchObject({ removed: true, directory: 'absent', registration: 'absent', warning: null });
+ expect(execGit).toHaveBeenCalledWith(
+ ['worktree', 'remove', path, '--force'], repo, { timeout: WORKTREE_ADD_TIMEOUT_MS }
+ );
expect(existsSync(path)).toBe(false);
expect(await registered(path)).toBe(false);
});
diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.js
index 6e0571f425..5e2620349b 100644
--- a/server/services/worktreeManager.js
+++ b/server/services/worktreeManager.js
@@ -120,7 +120,9 @@ const WORKTREE_ADD_RETRY_DELAY_MS = 250;
// while the worktree and branch it just gave up on exist on disk, which then
// defeats the orphan-branch cleanup ("Cannot delete branch … checked out at …").
// Ten minutes is far above any healthy add and still bounded. Applies to
-// `worktree move` too — it routes through this same wrapper.
+// `worktree move` too — it routes through this same wrapper. Removal also
+// deletes the full checkout (including ignored output), so it uses this ceiling
+// to avoid leaving a partially deleted tree when the metadata timeout expires.
//
// EXPORTED because it is the ceiling `STALE_GIT_LOCK_MIN_AGE_MS` is sized
// against: the longest a git command PortOS launches can legitimately still be
@@ -390,7 +392,7 @@ export async function forceRemoveWorktreeDir(repo, worktreePath, { label, log =
return null;
};
heldReason = await check();
- if (!heldReason) await execGit(['worktree', 'remove', worktreePath, '--force'], repo).catch(async (err) => {
+ if (!heldReason) await execGit(['worktree', 'remove', worktreePath, '--force'], repo, { timeout: WORKTREE_ADD_TIMEOUT_MS }).catch(async (err) => {
lastFailure = errorExcerpt(err);
if (label) console.log(`⚠️ ${label}: ${err.message}`);
heldReason = await check();
@@ -965,8 +967,9 @@ export async function releaseIdleSiblingNextHolder(sourceWorkspace, branchName,
*
* Returns null (never throws) whenever adoption isn't possible — the source tree
* is gone, the target path is occupied, or git refuses the move (a locked worktree,
- * or one with initialized submodules). Callers fall back to a fresh worktree, which
- * is the pre-existing behavior.
+ * or one with initialized submodules), or its status suggests a partial removal.
+ * The refused tree and branch stay intact for recovery: a fresh attachment to
+ * that same branch will remain blocked while this checkout holds it.
*
* @param {string} agentId - the RETRYING agent (names the destination directory)
* @param {string} sourceWorkspace - the parent git repository
@@ -992,14 +995,33 @@ async function adoptWorktreeUnlocked(agentId, sourceWorkspace, existingWorktreeP
existingBranch: true, adopted: true, instanceId
};
- // Already at the destination (a retry that re-entered prep) — nothing to move.
- if (pathsEqual(existingWorktreePath, targetPath)) return adopted;
-
- if (existsSync(targetPath)) {
+ if (!pathsEqual(existingWorktreePath, targetPath) && existsSync(targetPath)) {
console.log(`🌳 Cannot adopt worktree for ${agentId} — ${targetPath} is already occupied`);
return null;
}
+ // A killed removal can leave thousands of unstaged deletions. The resume
+ // prompt attributes all surviving changes to the agent, so do not hand it a
+ // likely damaged checkout. Keep intentional small deletions and staged work;
+ // this conservative threshold requires operator recovery for ambiguous bulk
+ // deletions instead of silently restoring or detaching somebody's work.
+ const status = await execGit(['status', '--porcelain', '--untracked-files=no'], existingWorktreePath)
+ .then(r => r.stdout)
+ .catch(() => null);
+ if (status === null) {
+ console.warn(`⚠️ Cannot adopt worktree for ${agentId} — status unreadable; preserved for recovery`);
+ return null;
+ }
+ const trackedChanges = status.split(/\r?\n/).filter(Boolean);
+ const deletions = trackedChanges.filter(line => line.startsWith(' D ')).length;
+ if (deletions >= 100 && deletions > trackedChanges.length / 2) {
+ console.warn(`⚠️ Cannot adopt worktree for ${agentId} — ${deletions} unstaged tracked deletions suggest an interrupted removal; preserved for recovery`);
+ return null;
+ }
+
+ // Re-entered prep still needs the damage check before resuming in place.
+ if (pathsEqual(existingWorktreePath, targetPath)) return adopted;
+
await ensureDir(WORKTREES_DIR);
if (options.deepResume) {
diff --git a/server/services/worktreeManager.recovery.test.js b/server/services/worktreeManager.recovery.test.js
index 0e2b2e278c..4d7194c8e7 100644
--- a/server/services/worktreeManager.recovery.test.js
+++ b/server/services/worktreeManager.recovery.test.js
@@ -1,14 +1,16 @@
import { afterAll, describe, expect, it, vi } from 'vitest';
-import { mkdir, writeFile } from 'fs/promises';
+import { mkdir, writeFile, rm, readFile } from 'fs/promises';
+import { existsSync } from 'fs';
import { join } from 'path';
-import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';
+import { mockPathsDataRoot, mockTestIdentity } from '../lib/mockPathsDataRoot.js';
import { materializeGitRepo, SKIP_HEAVY_INTEGRATION } from '../lib/gitTestRepo.js';
import { execGit } from '../lib/execGit.js';
const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-worktree-recovery-' });
vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));
+vi.mock('./instanceIdentity.js', () => mockTestIdentity());
const { PATHS } = await import('../lib/fileUtils.js');
-const { removeWorktree } = await import('./worktreeManager.js');
+const { removeWorktree, adoptWorktree, createWorktree } = await import('./worktreeManager.js');
afterAll(cleanup);
describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () => {
@@ -40,3 +42,47 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () =
expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });
});
});
+
+describe.skipIf(SKIP_HEAVY_INTEGRATION)('partial removal adoption safety', () => {
+ it('preserves a damaged checkout and occupied branch, then permits recovery with ordinary edits', async () => {
+ const repo = join(tempRoot, 'partial-source');
+ await materializeGitRepo(repo);
+ const files = Array.from({ length: 120 }, (_, i) => `tracked-${i}.txt`);
+ await Promise.all(files.map(file => writeFile(join(repo, file), 'original\n')));
+ await execGit(['add', '.'], repo);
+ await execGit(['commit', '-m', 'tracked fixture'], repo);
+ await mkdir(PATHS.worktrees, { recursive: true });
+ const branch = 'cos/partial/agent-partial';
+ const worktree = join(PATHS.worktrees, 'agent-partial');
+ const target = join(PATHS.worktrees, 'agent-partial-retry');
+ await execGit(['worktree', 'add', '-b', branch, worktree, 'main'], repo);
+ await Promise.all(files.slice(0, 105).map(file => rm(join(worktree, file))));
+ await writeFile(join(worktree, 'tracked-119.txt'), 'authored edit\n');
+ await writeFile(join(worktree, 'draft.txt'), 'authored draft\n');
+ const status = (await execGit(['status', '--porcelain'], worktree)).stdout;
+ const head = (await execGit(['rev-parse', branch], repo)).stdout;
+ const registrations = (await execGit(['worktree', 'list', '--porcelain'], repo)).stdout;
+
+ expect(await adoptWorktree('agent-partial-retry', repo, worktree, branch)).toBeNull();
+ expect(existsSync(target)).toBe(false);
+ expect((await execGit(['status', '--porcelain'], worktree)).stdout).toBe(status);
+ expect((await execGit(['rev-parse', branch], repo)).stdout).toBe(head);
+ expect((await execGit(['worktree', 'list', '--porcelain'], repo)).stdout).toBe(registrations);
+ // The caller's fresh-tree fallback must fail closed on the still-held branch;
+ // it cannot detach the damaged checkout or reset its branch to get unstuck.
+ await expect(createWorktree('agent-partial-retry', repo, 'task-partial', { existingBranch: branch }))
+ .rejects.toThrow(/already (?:checked out|used by worktree)/);
+ expect(existsSync(target)).toBe(false);
+ expect((await execGit(['worktree', 'list', '--porcelain'], repo)).stdout).toBe(registrations);
+ expect(await readFile(join(worktree, 'draft.txt'), 'utf8')).toBe('authored draft\n');
+
+ // Explicit recovery restores only the simulated accidental deletions. One
+ // authored deletion plus the tracked edit and untracked draft still resume.
+ await execGit(['restore', '--', ...files.slice(1, 105)], worktree);
+ const recoveredStatus = (await execGit(['status', '--porcelain'], worktree)).stdout;
+ expect(await adoptWorktree('agent-partial-retry', repo, worktree, branch)).toMatchObject({ adopted: true, worktreePath: target });
+ expect((await execGit(['status', '--porcelain'], target)).stdout).toBe(recoveredStatus);
+ expect(await readFile(join(target, 'tracked-119.txt'), 'utf8')).toBe('authored edit\n');
+ expect(await readFile(join(target, 'draft.txt'), 'utf8')).toBe('authored draft\n');
+ });
+});
diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.js
index 093008d7d0..c4c6e34ae0 100644
--- a/server/services/worktreeManager.test.js
+++ b/server/services/worktreeManager.test.js
@@ -1008,7 +1008,7 @@ describe('removeWorktree identity, dirt and branch preservation', () => {
join(PATHS.worktrees, 'agent-x')
);
expect(execGitMock).toHaveBeenCalledWith(
- ['worktree', 'remove', join(PATHS.worktrees, 'agent-x'), '--force'], '/repo'
+ ['worktree', 'remove', join(PATHS.worktrees, 'agent-x'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS }
);
});
@@ -1278,9 +1278,11 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()
});
// git refuses to move a locked worktree or one with initialized submodules —
- // never throw at the spawn path, just decline so the caller builds a fresh tree.
+ // Decline at the spawn path; the caller preserves any surviving branch holder.
it('returns null rather than throwing when git refuses the move', async () => {
- execGitMock.mockRejectedValue(new Error('working trees containing submodules cannot be moved'));
+ execGitMock.mockImplementation(args => args[0] === 'worktree' && args[1] === 'move'
+ ? Promise.reject(new Error('working trees containing submodules cannot be moved'))
+ : Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }));
await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();
});
@@ -1301,7 +1303,7 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()
const result = await adoptWorktree('agent-new', '/repo', NEW_TREE, 'cos/task-1/agent-dead');
- expect(execGitMock).not.toHaveBeenCalled();
+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], NEW_TREE);
expect(result).toMatchObject({ worktreePath: NEW_TREE, adopted: true });
});
@@ -1333,6 +1335,21 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()
expect(execGitMock).not.toHaveBeenCalledWith(expect.arrayContaining(['checkout']), expect.anything());
});
+ it('refuses a damaged tree even when it already sits at the destination', async () => {
+ scriptPaths({ source: true, target: true });
+ execGitMock.mockResolvedValue({ stdout: Array.from({ length: 100 }, (_, i) => ` D file-${i}`).join('\n') });
+
+ await expect(adoptWorktree('agent-new', '/repo', NEW_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();
+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], NEW_TREE);
+ });
+
+ it('preserves the tree when its status cannot be inspected', async () => {
+ execGitMock.mockRejectedValue(new Error('index unreadable'));
+
+ await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();
+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], DEAD_TREE);
+ });
+
it('returns null on incomplete input instead of guessing', async () => {
await expect(adoptWorktree(null, '/repo', DEAD_TREE, 'b')).resolves.toBeNull();
await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, null)).resolves.toBeNull();
@@ -1639,7 +1656,7 @@ describe('cleanupOrphanedWorktrees ownership and removal', () => {
expect(await cleanupOrphanedWorktrees('/repo', new Set(['agent-active']), { claimOwners: { agents: [] } })).toBe(1);
expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'worktree' && args[1] === 'remove'))
- .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo']]);
+ .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS }]]);
expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'branch' && args[1] === '-D'))
.toEqual([[['branch', '-D', 'cos/task/agent-dead'], '/repo']]);
expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'status').map(([, cwd]) => cwd))
@@ -1672,7 +1689,7 @@ describe('cleanupOrphanedWorktrees ownership and removal', () => {
expect(await cleanupOrphanedWorktrees('/repo', () => new Set(live), { claimOwners })).toBe(1);
expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'worktree' && args[1] === 'remove'))
- .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo']]);
+ .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS }]]);
expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'branch' && args[1] === '-D'))
.toEqual([[['branch', '-D', 'cos/task/agent-dead'], '/repo']]);
});