Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve failed agent work through repeated cleanup

Adam Eivyc731380ce834
diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.js
index 6e36cbaedf..4edb8c1fab 100644
--- a/server/services/worktreeManager.js
+++ b/server/services/worktreeManager.js
@@ -1034,6 +1034,52 @@ async function resolveMergeTarget(sourceWorkspace, defaultBranch) {
return remote || defaultBranch;
}
+// Branch recovery is independent of checkout recovery. A second cleanup can
+// arrive after the first removed the directory but preserved unpublished work;
+// every successful removal path must apply the same branch-retention policy.
+async function cleanupWorktreeBranch(sourceWorkspace, branchName, {
+ preserveBranchWithCommits = false, hasUnmergedCommits = false, warnings,
+}) {
+ if (!hasUnmergedCommits && preserveBranchWithCommits) {
+ // A prior cleanup may have already deleted a landed branch. Confirmed
+ // absence is a clean no-op; an unreadable ref is still work to preserve.
+ const localRef = await execGit(
+ ['show-ref', '--verify', '--quiet', `refs/heads/${branchName}`],
+ sourceWorkspace, { ignoreExitCode: true },
+ ).catch(() => null);
+ if (localRef?.exitCode === 1) return;
+ const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');
+ const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null) || 'main';
+ const target = await resolveMergeTarget(sourceWorkspace, defaultBranch);
+ // Patch-equivalence recognizes squash/rebase merges; a failed probe keeps
+ // the branch. Ask the refreshed remote target so a landed PR is not resumed
+ // just because this install's local default branch has not advanced.
+ const hasMergeEvidence = localRef?.exitCode === 0
+ && await hasBranchMergeEvidence(sourceWorkspace, branchName, target).catch(() => false);
+ if (!hasMergeEvidence) {
+ hasUnmergedCommits = true;
+ console.log(`🌳 Preserving branch ${branchName} — merge evidence into ${target} was not established, kept so a retry can resume from it`);
+ warnings.push(`Branch ${branchName} preserved — merge evidence was not established; kept for a retry`);
+ } else {
+ const ahead = parseInt((await execGit(
+ ['rev-list', '--count', `${target}..${branchName}`],
+ sourceWorkspace,
+ ).catch(() => ({ stdout: '' }))).stdout.trim(), 10);
+ console.log(ahead === 0
+ ? `🌳 Branch ${branchName} has no commits beyond ${target} — deleting it`
+ : `🌳 Branch ${branchName} is already merged into ${target} — safe to delete`);
+ }
+ }
+ if (hasUnmergedCommits) {
+ console.warn(`⚠️ Preserving worktree branch ${branchName} for manual/automated recovery`);
+ } else {
+ await execGit(['branch', '-D', branchName], sourceWorkspace)
+ .catch(err => {
+ console.warn(`⚠️ Branch delete failed for ${branchName}: ${err.message}`);
+ });
+ }
+}
+
/**
* Remove a git worktree and its associated branch.
*
@@ -1070,7 +1116,9 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio
return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
}
console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);
- await execGit(['branch', '-D', branchName], sourceWorkspace).catch(() => {});
+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {
+ preserveBranchWithCommits: options.preserveBranchWithCommits || options.merge, warnings,
+ });
return { merged: false, removed: true, uncommittedSaved: false, warnings };
}
@@ -1094,7 +1142,9 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio
warnings.push(`Worktree cleanup incomplete for ${worktreePath} (broken checkout still present): ${errorExcerpt(rmFailure)} — retryable`);
return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };
}
- await execGit(['branch', '-D', branchName], sourceWorkspace).catch(() => {});
+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {
+ preserveBranchWithCommits: options.preserveBranchWithCommits || options.merge, warnings,
+ });
return { merged: false, removed: true, uncommittedSaved: false, warnings };
}
@@ -1214,7 +1264,7 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio
// Preserve branch when (a) merge was attempted, failed, and has unmerged commits,
// OR (b) merge was refused because HEAD is on a non-default branch — the commits
// are still there and the user / a follow-up task may want to integrate manually.
- let hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);
+ const hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);
// ...OR (c) the caller asked us to keep any branch that still holds commits
// (`preserveBranchWithCommits`). The no-merge path — which is what a FAILED
// agent's cleanup takes — otherwise deletes the branch unconditionally, taking
@@ -1224,45 +1274,10 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio
// re-dispatched to a fresh agent. Opt-in rather than automatic so the PR flow
// (which deletes the LOCAL branch after pushing — the remote branch is what the
// PR points at) keeps cleaning up after itself.
- if (!hasUnmergedCommits && options.preserveBranchWithCommits && !merged) {
- const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');
- const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null) || 'main';
- const target = await resolveMergeTarget(sourceWorkspace, defaultBranch);
- // `hasBranchMergeEvidence` — NOT a bare `rev-list --count target..branch`. A branch
- // whose PR was REBASE- or SQUASH-merged has new SHAs, so rev-list still reports
- // it ahead and we would preserve an already-landed branch and point a retry at
- // it. PortOS merges with `--rebase` by default, so that is the COMMON shape of
- // the very incident this preservation exists for ("PR merged, then reaped").
- // hasBranchMergeEvidence covers patch-equivalence (`git cherry`) and fails closed,
- // which is the polarity we want here too: unknown ⇒ keep the work.
- // Asked against `resolveMergeTarget`, not the local default branch — see there
- // for why a branch merged on the forge otherwise reads as unmerged here.
- const hasMergeEvidence = await hasBranchMergeEvidence(sourceWorkspace, branchName, target).catch(() => false);
- if (!hasMergeEvidence) {
- hasUnmergedCommits = true;
- console.log(`🌳 Preserving branch ${branchName} — merge evidence into ${target} was not established, kept so a retry can resume from it`);
- warnings.push(`Branch ${branchName} preserved — merge evidence was not established; kept for a retry`);
- } else {
- // An ancestor of the target includes a branch that never received a commit
- // of its own. Calling that "already merged" reads as if the agent's work
- // landed, which is how a startup failure's empty branch looks like churn.
- const ahead = parseInt((await execGit(
- ['rev-list', '--count', `${target}..${branchName}`],
- sourceWorkspace,
- ).catch(() => ({ stdout: '' }))).stdout.trim(), 10);
- console.log(ahead === 0
- ? `🌳 Branch ${branchName} has no commits beyond ${target} — deleting it`
- : `🌳 Branch ${branchName} is already merged into ${target} — safe to delete`);
- }
- }
- if (hasUnmergedCommits) {
- console.log(`⚠️ Preserving worktree branch ${branchName} for manual/automated recovery`);
- } else {
- await execGit(['branch', '-D', branchName], sourceWorkspace)
- .catch(err => {
- console.log(`⚠️ Branch delete failed for ${branchName}: ${err.message}`);
- });
- }
+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {
+ preserveBranchWithCommits: options.preserveBranchWithCommits && !merged,
+ hasUnmergedCommits, warnings,
+ });
console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);
diff --git a/server/services/worktreeManager.recovery.test.js b/server/services/worktreeManager.recovery.test.js
new file mode 100644
index 0000000000..0e2b2e278c
--- /dev/null
+++ b/server/services/worktreeManager.recovery.test.js
@@ -0,0 +1,42 @@
+import { afterAll, describe, expect, it, vi } from 'vitest';
+import { mkdir, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';
+import { materializeGitRepo, SKIP_HEAVY_INTEGRATION } from '../lib/gitTestRepo.js';
+import { execGit } from '../lib/execGit.js';
+
+const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-worktree-recovery-' });
+vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));
+const { PATHS } = await import('../lib/fileUtils.js');
+const { removeWorktree } = await import('./worktreeManager.js');
+afterAll(cleanup);
+
+describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () => {
+ it('keeps the only committed copy recoverable when cleanup runs again after removing its checkout', async () => {
+ const repo = join(tempRoot, 'source');
+ await materializeGitRepo(repo);
+ await mkdir(PATHS.worktrees, { recursive: true });
+ const branch = 'cos/recovery/agent-example';
+ const worktree = join(PATHS.worktrees, 'agent-example');
+ await execGit(['worktree', 'add', '-b', branch, worktree, 'main'], repo);
+ await writeFile(join(worktree, 'recoverable.txt'), 'unpublished agent work\n');
+ await execGit(['add', 'recoverable.txt'], worktree);
+ await execGit(['commit', '-m', 'unpublished work'], worktree);
+ const options = { merge: false, preserveBranchWithCommits: true };
+
+ expect((await removeWorktree('agent-example', repo, branch, options)).removed).toBe(true);
+ expect((await removeWorktree('agent-example', repo, branch, options)).removed).toBe(true);
+
+ // A retry can attach a new checkout and read the authored file, without
+ // reflog surgery or a remote branch that the failed run never pushed.
+ const resumed = join(PATHS.worktrees, 'agent-retry');
+ await expect(execGit(['worktree', 'add', resumed, branch], repo)).resolves.toMatchObject({ exitCode: 0 });
+ expect((await execGit(['show', 'HEAD:recoverable.txt'], resumed)).stdout).toBe('unpublished agent work\n');
+
+ // Once that retry lands, both its first cleanup and any later completion
+ // callback must report success without inventing a preserved branch.
+ await execGit(['merge', branch, '--no-edit'], repo);
+ expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });
+ expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });
+ });
+});
diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.js
index 5bfa8f9ecf..4bb8f0387c 100644
--- a/server/services/worktreeManager.test.js
+++ b/server/services/worktreeManager.test.js
@@ -948,6 +948,60 @@ describe('removeWorktree identity, dirt and branch preservation', () => {
expect(result.warnings.join(' ')).toMatch(/preserved/i);
});
+ describe.each(['missing', 'broken'])('when the checkout is %s', checkout => {
+ beforeEach(() => {
+ if (checkout === 'missing') existsSync.mockReturnValue(false);
+ else scriptGit({ detectedToplevel: '/repo' });
+ });
+
+ it.each([
+ ['unmerged', false],
+ ['unknown', null],
+ ['landed', true],
+ ])('honors the retry branch policy when merge evidence is %s', async (_label, evidence) => {
+ if (evidence === null) hasBranchMergeEvidenceMock.mockRejectedValue(new Error('ref probe failed'));
+ else hasBranchMergeEvidenceMock.mockResolvedValue(evidence);
+
+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', {
+ merge: false, preserveBranchWithCommits: true,
+ });
+
+ expect(result.removed).toBe(true);
+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(evidence === true);
+ if (evidence !== true) expect(result.warnings.join(' ')).toMatch(/preserved/);
+ });
+
+ it('preserves unlanded commits when the requested merge cannot use a checkout', async () => {
+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', { merge: true });
+
+ expect(result).toMatchObject({ merged: false, removed: true });
+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(false);
+ expect(result.warnings.join(' ')).toMatch(/preserved/);
+ });
+
+ it('keeps explicit no-preservation cleanup behavior', async () => {
+ await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', { merge: false });
+
+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(true);
+ expect(hasBranchMergeEvidenceMock).not.toHaveBeenCalled();
+ });
+
+ it('preserves the branch when its existence probe fails', async () => {
+ hasBranchMergeEvidenceMock.mockResolvedValue(true);
+ const scripted = execGitMock.getMockImplementation();
+ execGitMock.mockImplementation((args, ...rest) => args[0] === 'show-ref'
+ ? Promise.resolve({ stdout: '', stderr: 'cannot read refs', exitCode: 128 })
+ : scripted(args, ...rest));
+
+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', {
+ merge: false, preserveBranchWithCommits: true,
+ });
+
+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(false);
+ expect(result.warnings.join(' ')).toMatch(/preserved/);
+ });
+ });
+
// Patch-equivalence matters here: PortOS merges with `--rebase`, so a landed
// branch has new SHAs. `hasBranchMergeEvidence` is what sees through that — a bare
// `rev-list --count` would report it ahead and preserve a merged branch forever.