Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve failed agent work through repeated cleanup
Adam Eivyc731380ce834
diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.jsindex 6e36cbaedf..4edb8c1fab 100644--- a/server/services/worktreeManager.js+++ b/server/services/worktreeManager.js@@ -1034,6 +1034,52 @@ async function resolveMergeTarget(sourceWorkspace, defaultBranch) {return remote || defaultBranch;}+// Branch recovery is independent of checkout recovery. A second cleanup can+// arrive after the first removed the directory but preserved unpublished work;+// every successful removal path must apply the same branch-retention policy.+async function cleanupWorktreeBranch(sourceWorkspace, branchName, {+ preserveBranchWithCommits = false, hasUnmergedCommits = false, warnings,+}) {+ if (!hasUnmergedCommits && preserveBranchWithCommits) {+ // A prior cleanup may have already deleted a landed branch. Confirmed+ // absence is a clean no-op; an unreadable ref is still work to preserve.+ const localRef = await execGit(+ ['show-ref', '--verify', '--quiet', `refs/heads/${branchName}`],+ sourceWorkspace, { ignoreExitCode: true },+ ).catch(() => null);+ if (localRef?.exitCode === 1) return;+ const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');+ const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null) || 'main';+ const target = await resolveMergeTarget(sourceWorkspace, defaultBranch);+ // Patch-equivalence recognizes squash/rebase merges; a failed probe keeps+ // the branch. Ask the refreshed remote target so a landed PR is not resumed+ // just because this install's local default branch has not advanced.+ const hasMergeEvidence = localRef?.exitCode === 0+ && await hasBranchMergeEvidence(sourceWorkspace, branchName, target).catch(() => false);+ if (!hasMergeEvidence) {+ hasUnmergedCommits = true;+ console.log(`🌳 Preserving branch ${branchName} — merge evidence into ${target} was not established, kept so a retry can resume from it`);+ warnings.push(`Branch ${branchName} preserved — merge evidence was not established; kept for a retry`);+ } else {+ const ahead = parseInt((await execGit(+ ['rev-list', '--count', `${target}..${branchName}`],+ sourceWorkspace,+ ).catch(() => ({ stdout: '' }))).stdout.trim(), 10);+ console.log(ahead === 0+ ? `🌳 Branch ${branchName} has no commits beyond ${target} — deleting it`+ : `🌳 Branch ${branchName} is already merged into ${target} — safe to delete`);+ }+ }+ if (hasUnmergedCommits) {+ console.warn(`⚠️ Preserving worktree branch ${branchName} for manual/automated recovery`);+ } else {+ await execGit(['branch', '-D', branchName], sourceWorkspace)+ .catch(err => {+ console.warn(`⚠️ Branch delete failed for ${branchName}: ${err.message}`);+ });+ }+}+/*** Remove a git worktree and its associated branch.*@@ -1070,7 +1116,9 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optioreturn { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };}console.log(`🌳 Worktree already removed for ${agentId}, cleaning up branch`);- await execGit(['branch', '-D', branchName], sourceWorkspace).catch(() => {});+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {+ preserveBranchWithCommits: options.preserveBranchWithCommits || options.merge, warnings,+ });return { merged: false, removed: true, uncommittedSaved: false, warnings };}@@ -1094,7 +1142,9 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optiowarnings.push(`Worktree cleanup incomplete for ${worktreePath} (broken checkout still present): ${errorExcerpt(rmFailure)} — retryable`);return { merged: false, removed: false, cleanupIncomplete: true, uncommittedSaved: false, warnings };}- await execGit(['branch', '-D', branchName], sourceWorkspace).catch(() => {});+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {+ preserveBranchWithCommits: options.preserveBranchWithCommits || options.merge, warnings,+ });return { merged: false, removed: true, uncommittedSaved: false, warnings };}@@ -1214,7 +1264,7 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio// Preserve branch when (a) merge was attempted, failed, and has unmerged commits,// OR (b) merge was refused because HEAD is on a non-default branch — the commits// are still there and the user / a follow-up task may want to integrate manually.- let hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);+ const hasUnmergedCommits = options.merge && !merged && (commitsAhead > 0 || mergeRefused);// ...OR (c) the caller asked us to keep any branch that still holds commits// (`preserveBranchWithCommits`). The no-merge path — which is what a FAILED// agent's cleanup takes — otherwise deletes the branch unconditionally, taking@@ -1224,45 +1274,10 @@ export async function removeWorktree(agentId, sourceWorkspace, branchName, optio// re-dispatched to a fresh agent. Opt-in rather than automatic so the PR flow// (which deletes the LOCAL branch after pushing — the remote branch is what the// PR points at) keeps cleaning up after itself.- if (!hasUnmergedCommits && options.preserveBranchWithCommits && !merged) {- const { getDefaultBranch, hasBranchMergeEvidence } = await import('./git.js');- const defaultBranch = await getDefaultBranch(sourceWorkspace).catch(() => null) || 'main';- const target = await resolveMergeTarget(sourceWorkspace, defaultBranch);- // `hasBranchMergeEvidence` — NOT a bare `rev-list --count target..branch`. A branch- // whose PR was REBASE- or SQUASH-merged has new SHAs, so rev-list still reports- // it ahead and we would preserve an already-landed branch and point a retry at- // it. PortOS merges with `--rebase` by default, so that is the COMMON shape of- // the very incident this preservation exists for ("PR merged, then reaped").- // hasBranchMergeEvidence covers patch-equivalence (`git cherry`) and fails closed,- // which is the polarity we want here too: unknown ⇒ keep the work.- // Asked against `resolveMergeTarget`, not the local default branch — see there- // for why a branch merged on the forge otherwise reads as unmerged here.- const hasMergeEvidence = await hasBranchMergeEvidence(sourceWorkspace, branchName, target).catch(() => false);- if (!hasMergeEvidence) {- hasUnmergedCommits = true;- console.log(`🌳 Preserving branch ${branchName} — merge evidence into ${target} was not established, kept so a retry can resume from it`);- warnings.push(`Branch ${branchName} preserved — merge evidence was not established; kept for a retry`);- } else {- // An ancestor of the target includes a branch that never received a commit- // of its own. Calling that "already merged" reads as if the agent's work- // landed, which is how a startup failure's empty branch looks like churn.- const ahead = parseInt((await execGit(- ['rev-list', '--count', `${target}..${branchName}`],- sourceWorkspace,- ).catch(() => ({ stdout: '' }))).stdout.trim(), 10);- console.log(ahead === 0- ? `🌳 Branch ${branchName} has no commits beyond ${target} — deleting it`- : `🌳 Branch ${branchName} is already merged into ${target} — safe to delete`);- }- }- if (hasUnmergedCommits) {- console.log(`⚠️ Preserving worktree branch ${branchName} for manual/automated recovery`);- } else {- await execGit(['branch', '-D', branchName], sourceWorkspace)- .catch(err => {- console.log(`⚠️ Branch delete failed for ${branchName}: ${err.message}`);- });- }+ await cleanupWorktreeBranch(sourceWorkspace, branchName, {+ preserveBranchWithCommits: options.preserveBranchWithCommits && !merged,+ hasUnmergedCommits, warnings,+ });console.log(`🌳 Removed worktree for ${agentId}${merged ? ' (merged)' : ''}`);diff --git a/server/services/worktreeManager.recovery.test.js b/server/services/worktreeManager.recovery.test.jsnew file mode 100644index 0000000000..0e2b2e278c--- /dev/null+++ b/server/services/worktreeManager.recovery.test.js@@ -0,0 +1,42 @@+import { afterAll, describe, expect, it, vi } from 'vitest';+import { mkdir, writeFile } from 'fs/promises';+import { join } from 'path';+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';+import { materializeGitRepo, SKIP_HEAVY_INTEGRATION } from '../lib/gitTestRepo.js';+import { execGit } from '../lib/execGit.js';++const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-worktree-recovery-' });+vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));+const { PATHS } = await import('../lib/fileUtils.js');+const { removeWorktree } = await import('./worktreeManager.js');+afterAll(cleanup);++describe.skipIf(SKIP_HEAVY_INTEGRATION)('failed worktree cleanup recovery', () => {+ it('keeps the only committed copy recoverable when cleanup runs again after removing its checkout', async () => {+ const repo = join(tempRoot, 'source');+ await materializeGitRepo(repo);+ await mkdir(PATHS.worktrees, { recursive: true });+ const branch = 'cos/recovery/agent-example';+ const worktree = join(PATHS.worktrees, 'agent-example');+ await execGit(['worktree', 'add', '-b', branch, worktree, 'main'], repo);+ await writeFile(join(worktree, 'recoverable.txt'), 'unpublished agent work\n');+ await execGit(['add', 'recoverable.txt'], worktree);+ await execGit(['commit', '-m', 'unpublished work'], worktree);+ const options = { merge: false, preserveBranchWithCommits: true };++ expect((await removeWorktree('agent-example', repo, branch, options)).removed).toBe(true);+ expect((await removeWorktree('agent-example', repo, branch, options)).removed).toBe(true);++ // A retry can attach a new checkout and read the authored file, without+ // reflog surgery or a remote branch that the failed run never pushed.+ const resumed = join(PATHS.worktrees, 'agent-retry');+ await expect(execGit(['worktree', 'add', resumed, branch], repo)).resolves.toMatchObject({ exitCode: 0 });+ expect((await execGit(['show', 'HEAD:recoverable.txt'], resumed)).stdout).toBe('unpublished agent work\n');++ // Once that retry lands, both its first cleanup and any later completion+ // callback must report success without inventing a preserved branch.+ await execGit(['merge', branch, '--no-edit'], repo);+ expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });+ expect(await removeWorktree('agent-retry', repo, branch, options)).toMatchObject({ removed: true, warnings: [] });+ });+});diff --git a/server/services/worktreeManager.test.js b/server/services/worktreeManager.test.jsindex 5bfa8f9ecf..4bb8f0387c 100644--- a/server/services/worktreeManager.test.js+++ b/server/services/worktreeManager.test.js@@ -948,6 +948,60 @@ describe('removeWorktree identity, dirt and branch preservation', () => {expect(result.warnings.join(' ')).toMatch(/preserved/i);});+ describe.each(['missing', 'broken'])('when the checkout is %s', checkout => {+ beforeEach(() => {+ if (checkout === 'missing') existsSync.mockReturnValue(false);+ else scriptGit({ detectedToplevel: '/repo' });+ });++ it.each([+ ['unmerged', false],+ ['unknown', null],+ ['landed', true],+ ])('honors the retry branch policy when merge evidence is %s', async (_label, evidence) => {+ if (evidence === null) hasBranchMergeEvidenceMock.mockRejectedValue(new Error('ref probe failed'));+ else hasBranchMergeEvidenceMock.mockResolvedValue(evidence);++ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', {+ merge: false, preserveBranchWithCommits: true,+ });++ expect(result.removed).toBe(true);+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(evidence === true);+ if (evidence !== true) expect(result.warnings.join(' ')).toMatch(/preserved/);+ });++ it('preserves unlanded commits when the requested merge cannot use a checkout', async () => {+ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', { merge: true });++ expect(result).toMatchObject({ merged: false, removed: true });+ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(false);+ expect(result.warnings.join(' ')).toMatch(/preserved/);+ });++ it('keeps explicit no-preservation cleanup behavior', async () => {+ await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', { merge: false });++ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(true);+ expect(hasBranchMergeEvidenceMock).not.toHaveBeenCalled();+ });++ it('preserves the branch when its existence probe fails', async () => {+ hasBranchMergeEvidenceMock.mockResolvedValue(true);+ const scripted = execGitMock.getMockImplementation();+ execGitMock.mockImplementation((args, ...rest) => args[0] === 'show-ref'+ ? Promise.resolve({ stdout: '', stderr: 'cannot read refs', exitCode: 128 })+ : scripted(args, ...rest));++ const result = await removeWorktree('agent-x', '/repo', 'cos/task-1/agent-x', {+ merge: false, preserveBranchWithCommits: true,+ });++ expect(calledWith(['branch', '-D', 'cos/task-1/agent-x'])).toBe(false);+ expect(result.warnings.join(' ')).toMatch(/preserved/);+ });+ });+// Patch-equivalence matters here: PortOS merges with `--rebase`, so a landed// branch has new SHAs. `hasBranchMergeEvidence` is what sees through that — a bare// `rev-list --count` would report it ahead and preserve a merged branch forever.