Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): preserve partially removed worktrees on resume (#12512)
[._.]/ Adam Eivyac079cb8c188
diff --git a/server/services/worktreeManager.forceRemove.test.js b/server/services/worktreeManager.forceRemove.test.jsindex 921dd5336b..07ff0f23cc 100644--- a/server/services/worktreeManager.forceRemove.test.js+++ b/server/services/worktreeManager.forceRemove.test.js@@ -21,7 +21,7 @@ vi.mock('../lib/execGit.js', async (importOriginal) => {});import { execGit } from '../lib/execGit.js';-import { forceRemoveWorktreeDir } from './worktreeManager.js';+import { forceRemoveWorktreeDir, WORKTREE_ADD_TIMEOUT_MS } from './worktreeManager.js';describe.skipIf(SKIP_HEAVY_INTEGRATION)('forceRemoveWorktreeDir', () => {let repo;@@ -59,6 +59,9 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('forceRemoveWorktreeDir', () => {const path = await addWorktree();const outcome = await forceRemoveWorktreeDir(repo, path, { verifyRemoval: async () => null });expect(outcome).toMatchObject({ removed: true, directory: 'absent', registration: 'absent', warning: null });+ expect(execGit).toHaveBeenCalledWith(+ ['worktree', 'remove', path, '--force'], repo, { timeout: WORKTREE_ADD_TIMEOUT_MS }+ );expect(existsSync(path)).toBe(false);expect(await registered(path)).toBe(false);});diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.jsindex 6e0571f425..5e2620349b 100644--- a/server/services/worktreeManager.js+++ b/server/services/worktreeManager.js@@ -120,7 +120,9 @@ const WORKTREE_ADD_RETRY_DELAY_MS = 250;// while the worktree and branch it just gave up on exist on disk, which then// defeats the orphan-branch cleanup ("Cannot delete branch … checked out at …").// Ten minutes is far above any healthy add and still bounded. Applies to-// `worktree move` too — it routes through this same wrapper.+// `worktree move` too — it routes through this same wrapper. Removal also+// deletes the full checkout (including ignored output), so it uses this ceiling+// to avoid leaving a partially deleted tree when the metadata timeout expires.//// EXPORTED because it is the ceiling `STALE_GIT_LOCK_MIN_AGE_MS` is sized// against: the longest a git command PortOS launches can legitimately still be@@ -390,7 +392,7 @@ export async function forceRemoveWorktreeDir(repo, worktreePath, { label, log =return null;};heldReason = await check();- if (!heldReason) await execGit(['worktree', 'remove', worktreePath, '--force'], repo).catch(async (err) => {+ if (!heldReason) await execGit(['worktree', 'remove', worktreePath, '--force'], repo, { timeout: WORKTREE_ADD_TIMEOUT_MS }).catch(async (err) => {lastFailure = errorExcerpt(err);if (label) console.log(`⚠️ ${label}: ${err.message}`);heldReason = await check();@@ -965,8 +967,9 @@ export async function releaseIdleSiblingNextHolder(sourceWorkspace, branchName,** Returns null (never throws) whenever adoption isn't possible — the source tree* is gone, the target path is occupied, or git refuses the move (a locked worktree,- * or one with initialized submodules). Callers fall back to a fresh worktree, which- * is the pre-existing behavior.+ * or one with initialized submodules), or its status suggests a partial removal.+ * The refused tree and branch stay intact for recovery: a fresh attachment to+ * that same branch will remain blocked while this checkout holds it.** @param {string} agentId - the RETRYING agent (names the destination directory)* @param {string} sourceWorkspace - the parent git repository@@ -992,14 +995,33 @@ async function adoptWorktreeUnlocked(agentId, sourceWorkspace, existingWorktreePexistingBranch: true, adopted: true, instanceId};- // Already at the destination (a retry that re-entered prep) — nothing to move.- if (pathsEqual(existingWorktreePath, targetPath)) return adopted;-- if (existsSync(targetPath)) {+ if (!pathsEqual(existingWorktreePath, targetPath) && existsSync(targetPath)) {console.log(`🌳 Cannot adopt worktree for ${agentId} — ${targetPath} is already occupied`);return null;}+ // A killed removal can leave thousands of unstaged deletions. The resume+ // prompt attributes all surviving changes to the agent, so do not hand it a+ // likely damaged checkout. Keep intentional small deletions and staged work;+ // this conservative threshold requires operator recovery for ambiguous bulk+ // deletions instead of silently restoring or detaching somebody's work.+ const status = await execGit(['status', '--porcelain', '--untracked-files=no'], existingWorktreePath)+ .then(r => r.stdout)+ .catch(() => null);+ if (status === null) {+ console.warn(`⚠️ Cannot adopt worktree for ${agentId} — status unreadable; preserved for recovery`);+ return null;+ }+ const trackedChanges = status.split(/\r?\n/).filter(Boolean);+ const deletions = trackedChanges.filter(line => line.startsWith(' D ')).length;+ if (deletions >= 100 && deletions > trackedChanges.length / 2) {+ console.warn(`⚠️ Cannot adopt worktree for ${agentId} — ${deletions} unstaged tracked deletions suggest an interrupted removal; preserved for recovery`);+ return null;+ }++ // Re-entered prep still needs the damage check before resuming in place.+ if (pathsEqual(existingWorktreePath, targetPath)) return adopted;+await ensureDir(WORKTREES_DIR);if (options.deepResume) {diff --git a/server/services/worktreeManager.recovery.test.js b/server/services/worktreeManager.recovery.test.jsindex 0e2b2e278c..4d7194c8e7 100644--- a/server/services/worktreeManager.recovery.test.js+++ b/server/services/worktreeManager.recovery.test.js@@ -1,14 +1,16 @@import { afterAll, describe, expect, it, vi } from 'vitest';-import { mkdir, writeFile } from 'fs/promises';+import { mkdir, writeFile, rm, readFile } from 'fs/promises';+import { existsSync } from 'fs';import { join } from 'path';-import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';+import { mockPathsDataRoot, mockTestIdentity } from '../lib/mockPathsDataRoot.js';import { materializeGitRepo, SKIP_HEAVY_INTEGRATION } from '../lib/gitTestRepo.js';import { execGit } from '../lib/execGit.js';const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-worktree-recovery-' });vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));+vi.mock('./instanceIdentity.js', () => mockTestIdentity());const { PATHS } = await import('../lib/fileUtils.js');-const { removeWorktree } = await import('./worktreeManager.js');+const { removeWorktree, adoptWorktree, createWorktree } = await import('./worktreeManager.js');afterAll(cleanup);describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () => {@@ -40,3 +42,47 @@ describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () =expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });});});++describe.skipIf(SKIP_HEAVY_INTEGRATION)('partial removal adoption safety', () => {+ it('preserves a damaged checkout and occupied branch, then permits recovery with ordinary edits', async () => {+ const repo = join(tempRoot, 'partial-source');+ await materializeGitRepo(repo);+ const files = Array.from({ length: 120 }, (_, i) => `tracked-${i}.txt`);+ await Promise.all(files.map(file => writeFile(join(repo, file), 'original\n')));+ await execGit(['add', '.'], repo);+ await execGit(['commit', '-m', 'tracked fixture'], repo);+ await mkdir(PATHS.worktrees, { recursive: true });+ const branch = 'cos/partial/agent-partial';+ const worktree = join(PATHS.worktrees, 'agent-partial');+ const target = join(PATHS.worktrees, 'agent-partial-retry');+ await execGit(['worktree', 'add', '-b', branch, worktree, 'main'], repo);+ await Promise.all(files.slice(0, 105).map(file => rm(join(worktree, file))));+ await writeFile(join(worktree, 'tracked-119.txt'), 'authored edit\n');+ await writeFile(join(worktree, 'draft.txt'), 'authored draft\n');+ const status = (await execGit(['status', '--porcelain'], worktree)).stdout;+ const head = (await execGit(['rev-parse', branch], repo)).stdout;+ const registrations = (await execGit(['worktree', 'list', '--porcelain'], repo)).stdout;++ expect(await adoptWorktree('agent-partial-retry', repo, worktree, branch)).toBeNull();+ expect(existsSync(target)).toBe(false);+ expect((await execGit(['status', '--porcelain'], worktree)).stdout).toBe(status);+ expect((await execGit(['rev-parse', branch], repo)).stdout).toBe(head);+ expect((await execGit(['worktree', 'list', '--porcelain'], repo)).stdout).toBe(registrations);+ // The caller's fresh-tree fallback must fail closed on the still-held branch;+ // it cannot detach the damaged checkout or reset its branch to get unstuck.+ await expect(createWorktree('agent-partial-retry', repo, 'task-partial', { existingBranch: branch }))+ .rejects.toThrow(/already (?:checked out|used by worktree)/);+ expect(existsSync(target)).toBe(false);+ expect((await execGit(['worktree', 'list', '--porcelain'], repo)).stdout).toBe(registrations);+ expect(await readFile(join(worktree, 'draft.txt'), 'utf8')).toBe('authored draft\n');++ // Explicit recovery restores only the simulated accidental deletions. One+ // authored deletion plus the tracked edit and untracked draft still resume.+ await execGit(['restore', '--', ...files.slice(1, 105)], worktree);+ const recoveredStatus = (await execGit(['status', '--porcelain'], worktree)).stdout;+ expect(await adoptWorktree('agent-partial-retry', repo, worktree, branch)).toMatchObject({ adopted: true, worktreePath: target });+ expect((await execGit(['status', '--porcelain'], target)).stdout).toBe(recoveredStatus);+ expect(await readFile(join(target, 'tracked-119.txt'), 'utf8')).toBe('authored edit\n');+ expect(await readFile(join(target, 'draft.txt'), 'utf8')).toBe('authored draft\n');+ });+});diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.jsindex 093008d7d0..c4c6e34ae0 100644--- a/server/services/worktreeManager.test.js+++ b/server/services/worktreeManager.test.js@@ -1008,7 +1008,7 @@ describe('removeWorktree identity, dirt and branch preservation', () => {join(PATHS.worktrees, 'agent-x'));expect(execGitMock).toHaveBeenCalledWith(- ['worktree', 'remove', join(PATHS.worktrees, 'agent-x'), '--force'], '/repo'+ ['worktree', 'remove', join(PATHS.worktrees, 'agent-x'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS });});@@ -1278,9 +1278,11 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()});// git refuses to move a locked worktree or one with initialized submodules —- // never throw at the spawn path, just decline so the caller builds a fresh tree.+ // Decline at the spawn path; the caller preserves any surviving branch holder.it('returns null rather than throwing when git refuses the move', async () => {- execGitMock.mockRejectedValue(new Error('working trees containing submodules cannot be moved'));+ execGitMock.mockImplementation(args => args[0] === 'worktree' && args[1] === 'move'+ ? Promise.reject(new Error('working trees containing submodules cannot be moved'))+ : Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }));await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();});@@ -1301,7 +1303,7 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()const result = await adoptWorktree('agent-new', '/repo', NEW_TREE, 'cos/task-1/agent-dead');- expect(execGitMock).not.toHaveBeenCalled();+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], NEW_TREE);expect(result).toMatchObject({ worktreePath: NEW_TREE, adopted: true });});@@ -1333,6 +1335,21 @@ describe('adoptWorktree — resuming an interrupted run in its own worktree', ()expect(execGitMock).not.toHaveBeenCalledWith(expect.arrayContaining(['checkout']), expect.anything());});+ it('refuses a damaged tree even when it already sits at the destination', async () => {+ scriptPaths({ source: true, target: true });+ execGitMock.mockResolvedValue({ stdout: Array.from({ length: 100 }, (_, i) => ` D file-${i}`).join('\n') });++ await expect(adoptWorktree('agent-new', '/repo', NEW_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], NEW_TREE);+ });++ it('preserves the tree when its status cannot be inspected', async () => {+ execGitMock.mockRejectedValue(new Error('index unreadable'));++ await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, 'cos/task-1/agent-dead')).resolves.toBeNull();+ expect(execGitMock).toHaveBeenCalledExactlyOnceWith(['status', '--porcelain', '--untracked-files=no'], DEAD_TREE);+ });+it('returns null on incomplete input instead of guessing', async () => {await expect(adoptWorktree(null, '/repo', DEAD_TREE, 'b')).resolves.toBeNull();await expect(adoptWorktree('agent-new', '/repo', DEAD_TREE, null)).resolves.toBeNull();@@ -1639,7 +1656,7 @@ describe('cleanupOrphanedWorktrees ownership and removal', () => {expect(await cleanupOrphanedWorktrees('/repo', new Set(['agent-active']), { claimOwners: { agents: [] } })).toBe(1);expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'worktree' && args[1] === 'remove'))- .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo']]);+ .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS }]]);expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'branch' && args[1] === '-D')).toEqual([[['branch', '-D', 'cos/task/agent-dead'], '/repo']]);expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'status').map(([, cwd]) => cwd))@@ -1672,7 +1689,7 @@ describe('cleanupOrphanedWorktrees ownership and removal', () => {expect(await cleanupOrphanedWorktrees('/repo', () => new Set(live), { claimOwners })).toBe(1);expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'worktree' && args[1] === 'remove'))- .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo']]);+ .toEqual([[['worktree', 'remove', tree('agent-dead'), '--force'], '/repo', { timeout: WORKTREE_ADD_TIMEOUT_MS }]]);expect(execGitMock.mock.calls.filter(([args]) => args[0] === 'branch' && args[1] === '-D')).toEqual([[['branch', '-D', 'cos/task/agent-dead'], '/repo']]);});