Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep recoverable cleanup out of personal Brain obligations (#13479)

[._.]/d4cc19979679
diff --git a/server/lib/repoStateExpectations.js b/server/lib/repoStateExpectations.js
index 2233cd0876..d6a4fc31ce 100644
--- a/server/lib/repoStateExpectations.js
+++ b/server/lib/repoStateExpectations.js
@@ -85,7 +85,7 @@ export const REPO_STATE_SKIPS = Object.freeze({
* @param {boolean} [params.isPersistentWorktree] - long-lived feature worktree
* @param {boolean} [params.discardWorktree] - reasoning agent whose tree is thrown away
* @param {boolean} [params.followUpPending] - something is already queued to land this branch
- * @param {number} [params.cleanupWarningCount] - cleanup spawns its own recovery when it warned
+ * @param {number} [params.cleanupWarningCount] - warning persistence precedes the durable completion retry
* @param {string} [params.prCompletion] - resolved `PR_COMPLETIONS` policy
* @param {boolean} [params.leaveOpen] - the PR is a human's to land (JIRA hand-off)
* @param {boolean} [params.prExpected] - the task asked for a PR
diff --git a/server/services/agentCompletionCleanup.js b/server/services/agentCompletionCleanup.js
index cafe7a6b14..c9c68b57be 100644
--- a/server/services/agentCompletionCleanup.js
+++ b/server/services/agentCompletionCleanup.js
@@ -48,7 +48,6 @@ import { cleanupAgentWorktree, spawnMergeRecoveryTask, releaseRetryHold } from '
import { PR_COMPLETIONS, PR_CREATION, resolvePrCompletion, resolvePrCreation } from '../lib/prDisposition.js';
import { resolvePrOpenedBy, PR_OPENED_BY } from '../lib/slashdoInvocation.js';
import { isPublicReviewRestrictedProfile, publicReviewPostureForProfile } from '../lib/agentExecutionProfiles.js';
-import { ensureTaskThread } from './brainTaskThreads.js';
import { acknowledgeCompletionSentinel, completionSentinelPresent } from './agentCompletionSentinel.js';
const ROOT_DIR = PATHS.root;
@@ -693,18 +692,7 @@ async function reportWorktreeCleanupWarnings({ agentId, task, cleanupWarnings })
if (currentAgent?.result?.deliveryRecoveryOwned === true) return;
const appName = task?.metadata?.appName || task?.metadata?.app || 'PortOS';
- void spawnMergeRecoveryTask(cleanupWarnings, agentId, task, appName, currentAgent?.metadata?.sourceWorkspace).then(recoveryTask => {
- const recoveryTaskId = recoveryTask?.id || recoveryTask;
- return ensureTaskThread({
- taskId: recoveryTaskId,
- title: `Agent cleanup issue: ${appName}`,
- nextAction: 'Let the recovery task resolve the cleanup warning, then verify the worktree.',
- notes: cleanupWarnings.join('\n'),
- priority: 'high',
- });
- }).catch(err => {
- emitLog('warn', `Failed to spawn merge recovery task: ${err.message}`, { agentId, taskId: task?.id });
- });
+ await spawnMergeRecoveryTask(cleanupWarnings, agentId, task, appName, currentAgent?.metadata?.sourceWorkspace);
}
}
diff --git a/server/services/agentCompletionCleanup.test.js b/server/services/agentCompletionCleanup.test.js
index b603184543..a437b013b7 100644
--- a/server/services/agentCompletionCleanup.test.js
+++ b/server/services/agentCompletionCleanup.test.js
@@ -820,9 +820,7 @@ describe.each(['runner', 'spawner'])('%s completion side effects', (path) => {
await complete({ appName: 'Example App' });
expect(updateAgent).toHaveBeenCalledWith('a1', { result: { success: true, warnings } });
expect(addNotification).not.toHaveBeenCalled();
- expect(ensureTaskThread).toHaveBeenCalledWith(expect.objectContaining({
- title: 'Agent cleanup issue: Example App', notes: warnings[0], priority: 'high',
- }));
+ expect(ensureTaskThread).not.toHaveBeenCalled();
expect(spawnMergeRecoveryTask).toHaveBeenCalledWith(warnings, 'a1', expect.objectContaining({ id: 't' }), 'Example App', '/example/repo');
});
diff --git a/server/services/agentErrorAnalysis.js b/server/services/agentErrorAnalysis.js
index 22120591dd..badbe30ef5 100644
--- a/server/services/agentErrorAnalysis.js
+++ b/server/services/agentErrorAnalysis.js
@@ -1442,6 +1442,12 @@ export const API_ACCESS_ERROR_CATEGORIES = new Set([
]);
export async function maybeCreateInvestigationTask(agentId, task, analysis) {
+ // The original cleanup receipt and bounded review task already own this
+ // failure. Investigation would allocate another task and fresh spawn budget.
+ if (task?.metadata?.recoveryOrigin?.subsystem === 'repository-cleanup') {
+ emitLog('info', 'Cleanup recovery failure retained by its existing owner', { agentId, taskId: task.id, category: analysis?.category });
+ return;
+ }
if (resolvePrCompletion(task?.metadata) === PR_COMPLETIONS.LEAVE_OPEN) {
emitLog('warn', `Review-only task ${task.id} needs attention; automatic investigation suppressed`, { agentId, taskId: task.id });
return;
diff --git a/server/services/agentErrorAnalysis.test.js b/server/services/agentErrorAnalysis.test.js
index ef989627df..a9aef44f56 100644
--- a/server/services/agentErrorAnalysis.test.js
+++ b/server/services/agentErrorAnalysis.test.js
@@ -717,6 +717,13 @@ describe('maybeCreateInvestigationTask', () => {
expect(addTask).toHaveBeenCalledTimes(1);
});
+ it('does not turn a failed repository cleanup recovery into a new investigation', async () => {
+ await maybeCreateInvestigationTask('agent-1', { id: 'cleanup-review', metadata: {
+ recoveryOrigin: { subsystem: 'repository-cleanup', parentAgentId: 'original' },
+ } }, { category: 'unknown', message: 'synthetic recoverable failure', actionable: true });
+ expect(addTask).not.toHaveBeenCalled();
+ });
+
it('never spawns an investigation for a failed investigation task (meta-cascade guard)', async () => {
await maybeCreateInvestigationTask('agent-1', { id: 'inv-1', description: '[Auto] Investigate agent failure: x', metadata: { isInvestigation: true } }, { category: 'unknown', message: 'boom' });
expect(addTask).not.toHaveBeenCalled();
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.js
index a3de1bfe03..61b57329c2 100644
--- a/server/services/agentRepoStateVerification.js
+++ b/server/services/agentRepoStateVerification.js
@@ -52,7 +52,6 @@ import { resolveTaskTargetBranch } from '../lib/taskTargetBranch.js';
import { leavesPrForHuman, prStaysOpen, resolvePrCompletion, PR_COMPLETIONS } from '../lib/prDisposition.js';
import { probePrForBranch, probeChangeRequestState } from './prProbe.js';
import { parsePullRequestUrl } from '../lib/gitForge.js';
-import { ensureTaskThread } from './brainTaskThreads.js';
import {
REPO_STATE_ISSUES,
REPO_STATE_SKIPS,
@@ -323,7 +322,8 @@ export async function verifyAgentRepoState(input) {
const terminalSkips = [REPO_STATE_SKIPS.DISABLED, REPO_STATE_SKIPS.NOT_WORKTREE,
REPO_STATE_SKIPS.PERSISTENT_WORKTREE, REPO_STATE_SKIPS.DISCARDED_WORKTREE, REPO_STATE_SKIPS.FAILED_RUN];
const settled = cleanupPending ? result.verified
- : outcomes.every(item => item.verified || terminalSkips.includes(item.skipReason)) || leavesPrForHuman(input.task);
+ : outcomes.every(item => item.verified || (terminalSkips.includes(item.skipReason)
+ && !(item.skipReason === REPO_STATE_SKIPS.FAILED_RUN && input.agentState?.metadata?.cleanupOutcome?.status === 'pending'))) || leavesPrForHuman(input.task);
const { recordRepoStateVerification } = await import('./cosAgentLifecycle.js');
await recordRepoStateVerification(input.agentId, { status: settled ? 'completed' : 'pending',
checkedAt: new Date().toISOString(), reasons: [...new Set(outcomes.map(item => item.skipReason).filter(Boolean))] });
@@ -365,17 +365,22 @@ async function resumeCompletionAudits() {
const { readBranchCompletionPolicies } = await import('./branchCompletionPolicy.js');
const { getTaskById } = await import('./cosTaskStore.js');
const agents = await readBranchCompletionPolicies({ repairIndex: false });
+ const { reconcileCleanupThreads } = await import('./brainTaskThreads.js');
+ await reconcileCleanupThreads({ dryRun: false }).catch(error => emitLog('warn', `Cleanup thread reconciliation deferred: ${error.message}`));
const { recoverLegacyDeliveryBlocks } = await import('./deliveryRecovery.js');
await recoverLegacyDeliveryBlocks(agents);
const lastAttempt = agent => [agent.metadata.repoStateVerification?.checkedAt, agent.metadata.completionCleanup?.updatedAt,
agent.metadata.completionCleanup?.lastAttemptAt].filter(value => typeof value === 'string').sort().at(-1) || '';
const pending = agents.filter(agent => agent.status === 'completed' && (agent.metadata?.repoStateVerification?.status === 'pending'
|| agent.metadata?.completionCleanup?.status === 'pending'))
+ .filter(agent => !agent.metadata.completionCleanup?.retryAt || Date.parse(agent.metadata.completionCleanup.retryAt) <= Date.now())
.sort((a, b) => lastAttempt(a).localeCompare(lastAttempt(b)));
let checked = 0;
for (const original of pending.slice(0, COMPLETION_RECOVERY_BATCH_SIZE)) {
const { recordCompletionCleanup } = await import('./cosAgentLifecycle.js');
- await recordCompletionCleanup(original.id || original.agentId, original.startedAt, { lastAttemptAt: new Date().toISOString() });
+ const attempt = (original.metadata.completionCleanup?.replayAttempts || 0) + 1;
+ await recordCompletionCleanup(original.id || original.agentId, original.startedAt, { lastAttemptAt: new Date().toISOString(),
+ replayAttempts: attempt, retryAt: new Date(Date.now() + Math.min(30 * 60_000, 60_000 * 2 ** Math.min(attempt - 1, 5))).toISOString() });
const { replayAgentCompletionCleanup } = await import('./agentCompletionCleanup.js');
const { getAgentRecord } = await import('./cosAgentLifecycle.js');
await replayAgentCompletionCleanup(original).catch(error => emitLog('warn', `Completion cleanup deferred for ${original.id}: ${error.message}`, { agentId: original.id }));
@@ -652,6 +657,10 @@ async function settleDivergedBranch(ctx) {
return held;
}
if (prepared.integrated) return retireIntegratedCheckout(ctx, prepared.retirement);
+ // A recovery successor cannot start a new lineage with a fresh budget.
+ if (ctx.task?.metadata?.recoveryOrigin?.subsystem === 'repository-cleanup') {
+ return ctx.blockWith('cleanup-successor-review-unresolved', { issues: ctx.issues, observed: ctx.observed });
+ }
return fileDivergenceRecovery(ctx, prepared.retirement);
}
@@ -706,16 +715,6 @@ async function fileDivergenceRecovery({ agentId, task, appId, metadata, sourceWo
return null;
});
- if (recoveryTaskId) {
- await ensureTaskThread({
- taskId: recoveryTaskId,
- title: `${RECOVERY_TASK_PREFIX} Finish incomplete cleanup for branch ${branchName}`,
- nextAction: 'Let the recovery task finish, then verify the branch and worktree are clean.',
- notes: issues.map(issue => issue.message).join('\n'),
- priority: 'high',
- }).catch(err => emitLog('warn', `Failed to create repo-state Brain thread for ${branchName}: ${err.message}`, { agentId, branchName }));
- }
-
return { verified: false, skipReason: null, issues, observed, recoveryTaskId };
}
@@ -869,7 +868,7 @@ async function fileRepoStateRecoveryTask({ agentId, task, branchName, sourceWork
...(retirement ? { retirementReview: { appId, release: retirement.release, createdAt: retirement.createdAt } } : {}) },
context,
useWorktree: false,
- }, 'user');
+ }, 'internal');
await recordRepoCleanupRecovery(recoveryOwner, branchName, observation, { phase: created?.id ? 'dispatched' : 'blocked', taskId: created?.id || null });
await recordCleanupOutcome(agentId, { status: 'pending', reason: 'cognitive-review-pending', branch: branchName, path: worktreePath, taskId: created?.id || null });
diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.js
index b31735e7b3..81db0e80a4 100644
--- a/server/services/agentRepoStateVerification.test.js
+++ b/server/services/agentRepoStateVerification.test.js
@@ -71,7 +71,7 @@ vi.mock('./notifications.js', () => ({
NOTIFICATION_TYPES: { AGENT_WARNING: 'agent_warning' },
PRIORITY_LEVELS: { HIGH: 'high' },
}));
-vi.mock('./brainTaskThreads.js', () => ({ ensureTaskThread: vi.fn().mockResolvedValue({}) }));
+vi.mock('./brainTaskThreads.js', () => ({ ensureTaskThread: vi.fn().mockResolvedValue({}), reconcileCleanupThreads: vi.fn(async () => ({ candidates: 0, archived: 0 })) }));
vi.mock('../lib/execGit.js', () => ({ execGit: vi.fn() }));
vi.mock('fs', () => ({ existsSync: vi.fn().mockReturnValue(false) }));
@@ -366,7 +366,7 @@ describe('verifyAgentRepoState — divergent runs', () => {
]);
expect(addTask).toHaveBeenCalledTimes(1);
const [payload, kind] = addTask.mock.calls[0];
- expect(kind).toBe('user');
+ expect(kind).toBe('internal');
expect(payload.description).toContain(BRANCH);
expect(payload.isRecovery).toBe(true);
expect(payload.metadata.recoveryOrigin).toMatchObject({ subsystem: 'repository-cleanup', attempt: 1, noProgress: false });
@@ -391,9 +391,18 @@ describe('verifyAgentRepoState — divergent runs', () => {
expect(recordBranchReconcileOutcome.mock.invocationCallOrder[0]).toBeLessThan(prepareInventoriedCheckoutRetirement.mock.invocationCallOrder[0]);
expect(prepareInventoriedCheckoutRetirement.mock.invocationCallOrder[0]).toBeLessThan(addTask.mock.invocationCallOrder[0]);
expect(payload.context).toContain(`touch ONLY ${BRANCH}`);
- expect(ensureTaskThread).toHaveBeenCalledWith(expect.objectContaining({
- taskId: 'task-recovery-1', priority: 'high',
- }));
+ expect(ensureTaskThread).not.toHaveBeenCalled();
+ });
+
+ it('retains a recovery successor divergence without starting another lineage', async () => {
+ existsSync.mockReturnValue(true);
+ localBranch(true);
+ listWorktrees.mockResolvedValue([{ ...RELEASE, branch: `refs/heads/${BRANCH}` }]);
+ findPullRequestForBranch.mockResolvedValue({ status: 'found', url: 'https://github.com/example/repo/pull/1', detail: 'MERGED' });
+ const result = await run({ task: task({ recoveryOrigin: { subsystem: 'repository-cleanup', parentAgentId: 'original' } }) });
+ expect(result).toMatchObject({ verified: false, skipReason: 'cleanup-successor-review-unresolved' });
+ expect(addTask).not.toHaveBeenCalled();
+ expect(recordCleanupOutcome).toHaveBeenCalledWith('agent-1', expect.objectContaining({ status: 'blocked', reason: 'cleanup-successor-review-unresolved' }));
});
it('retains a refused preview obligation and retries fresh evidence without dispatching unsafe cleanup', async () => {
@@ -836,6 +845,19 @@ describe('durable completion replay', () => {
expect(recordRepoStateVerification).toHaveBeenCalledWith('agent-1', expect.objectContaining({ status: 'pending' }));
expect(addTask).not.toHaveBeenCalled();
});
+ it('backs off repeated machine recovery across restarts without creating tasks', async () => {
+ const prior = archived();
+ prior.metadata.completionCleanup = { replayAttempts: 5, retryAt: new Date(Date.now() + 60_000).toISOString() };
+ readBranchCompletionPolicies.mockResolvedValue([prior]);
+ expect(await sweepRepoStateVerifications()).toMatchObject({ checked: 0 });
+ prior.metadata.completionCleanup.retryAt = new Date(Date.now() - 1).toISOString();
+ getTaskById.mockResolvedValue(task());
+ expect(await sweepRepoStateVerifications()).toMatchObject({ checked: 1 });
+ const { recordCompletionCleanup } = await import('./cosAgentLifecycle.js');
+ expect(recordCompletionCleanup).toHaveBeenCalledWith('agent-1', prior.startedAt, expect.objectContaining({ replayAttempts: 6, retryAt: expect.any(String) }));
+ expect(addTask).not.toHaveBeenCalled();
+ });
+
it('keeps missing policy pending with an updated retry time rather than granting authority or starving other audits', async () => {
getTaskById.mockResolvedValue(null);
readBranchCompletionPolicies.mockResolvedValue([archived()]);
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index b3952694a2..91dccc86a4 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -3,8 +3,8 @@ import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';
* Agent Worktree Cleanup
*
* Post-completion worktree handling for agents: merge-or-PR the worktree
- * branch, drive the multi-reviewer review-loop follow-up, and auto-create
- * recovery tasks when a merge or PR creation fails. Extracted from
+ * branch, drive the multi-reviewer review-loop follow-up, and retain
+ * completion recovery when a merge or PR creation fails. Extracted from
* agentLifecycle.js as a self-contained leaf so the completion-cleanup
* orchestrator (agentCompletionCleanup.js) can import it without a circular
* dependency back into agentLifecycle.js. Consumers import these functions from
@@ -29,7 +29,6 @@ import { PATHS } from '../lib/fileUtils.js';
import { execGit } from '../lib/execGit.js';
import { isRetryHoldOwner, clearedRetryHoldMetadata } from '../lib/taskRetryHold.js';
import { resolveTaskTargetBranch, resolveTaskRemoteBranch, shouldStripTaskTargetBranch } from '../lib/taskTargetBranch.js';
-import { RECOVERY_TASK_PREFIX } from './recoveryTasks.js';
import { ServerError } from '../lib/errorHandler.js';
import { detectForgeCli } from '../lib/gitForge.js';
import { normalizeForkHead } from '../lib/forkHead.js';
@@ -1465,90 +1464,24 @@ async function resumeReviewLoopCoordinator(taskId, prUrl, prBranch, reviewTarget
return { id: taskId, resumed: true };
}
-/**
- * Auto-create a recovery task when a worktree merge or PR creation fails, so stale
- * branches don't accumulate in managed app repos and block future agent work.
- */
+/** Retain warning recovery on the original completion receipt. No new agent,
+ * retry budget or personal obligation is created by a cleanup warning. */
export async function spawnMergeRecoveryTask(cleanupWarnings, agentId, task, appName, sourceWorkspace) {
- let staleBranch = null;
- let isMergeFail = false;
-
- for (const w of cleanupWarnings) {
- const mergeMatch = w.match(/Auto-merge failed for branch (\S+)/);
- if (mergeMatch) { staleBranch = mergeMatch[1]; isMergeFail = true; break; }
-
- const prMatch = w.match(/PR creation failed for branch (\S+?):/);
- if (prMatch) { staleBranch = prMatch[1]; break; }
- }
-
- if (!staleBranch || !sourceWorkspace) return;
-
- // Review-only runs must not delegate to a recovery agent that can land work.
- if (resolvePrCompletion(task?.metadata) === PR_COMPLETIONS.LEAVE_OPEN) {
- emitLog('warn', `Review-only branch ${staleBranch} needs manual recovery; automatic recovery suppressed`, { agentId, staleBranch });
- return null;
- }
-
- const appId = task?.metadata?.app;
-
- if (isMergeFail) {
- const defaultBr = await git.getDefaultBranch(sourceWorkspace).catch(() => null) || 'main';
- const recoveryTask = await addTask({
- description: `${RECOVERY_TASK_PREFIX} Resolve merge conflict and clean up stale branch ${staleBranch} in ${appName}`,
- priority: 'HIGH',
- app: appId,
- isRecovery: true,
- context: `An agent failed to auto-merge branch "${staleBranch}" back to ${defaultBr} in ${sourceWorkspace}. `
- + `Resolve this by: (1) checking if the branch's changes are already on ${defaultBr} (superseded by other commits), `
- + `and if so, delete the branch with "git branch -D ${staleBranch}"; `
- + `(2) if the changes are NOT on ${defaultBr}, attempt "git merge ${staleBranch} --no-edit" from ${defaultBr}, resolve any conflicts, and commit; `
- + `(3) after merging or determining the branch is stale, delete it with "git branch -D ${staleBranch}". `
- + `Original agent: ${agentId}, original task: ${task?.description || 'unknown'}.`,
- useWorktree: false,
- }, 'user').catch(err => {
- emitLog('warn', `Failed to create merge recovery task: ${err.message}`, { agentId, staleBranch });
- return null;
- });
- if (recoveryTask) {
- emitLog('info', `🔧 Auto-created merge recovery task for stale branch ${staleBranch}`, { agentId, appName });
- }
- return recoveryTask;
- } else {
- // PR/MR creation failed — spawn an agent to investigate and retry. Pick gh vs
- // glab based on the repo's forge so the recovery agent gets commands that
- // actually work against this remote.
- const [{ cli }, detectedBase] = await Promise.all([
- git.resolveForgeForRepo(sourceWorkspace).catch(() => ({ cli: 'gh' })),
- git.getDefaultBranch(sourceWorkspace).catch(() => null)
- ]);
- const targetBase = detectedBase || 'main';
- const isGitLab = cli === 'glab';
- const reqWord = isGitLab ? 'MR' : 'PR';
- const listCmd = isGitLab
- ? `glab mr list --source-branch ${staleBranch}`
- : `gh pr list --head ${staleBranch}`;
- const createCmd = isGitLab
- ? `glab mr create --source-branch ${staleBranch} --target-branch ${targetBase} --title '...' --description '...'`
- : `gh pr create --head ${staleBranch} --base ${targetBase} --title '...' --body '...'`;
-
- const recoveryTask = await addTask({
- description: `${RECOVERY_TASK_PREFIX} Investigate and retry failed ${reqWord} for branch ${staleBranch} in ${appName}`,
- priority: 'HIGH',
- app: appId,
- isRecovery: true,
- context: `An agent pushed branch "${staleBranch}" to ${sourceWorkspace} but automated ${reqWord} creation failed. `
- + `Investigate by: (1) checking if a ${reqWord} already exists for this branch: "${listCmd}"; `
- + `(2) if no ${reqWord} exists, review the branch changes and create one: "${createCmd}"; `
- + `(3) if the branch is stale or changes are already on ${targetBase}, delete the remote branch: "git push origin --delete ${staleBranch}". `
- + `Original agent: ${agentId}, original task: ${task?.description || 'unknown'}.`,
- useWorktree: false,
- }, 'user').catch(err => {
- emitLog('warn', `Failed to create ${reqWord} recovery task: ${err.message}`, { agentId, staleBranch });
- return null;
- });
- if (recoveryTask) {
- emitLog('info', `🔧 Auto-created ${reqWord} recovery task for branch ${staleBranch}`, { agentId, appName, cli });
- }
- return recoveryTask;
+ if (!cleanupWarnings?.length || !agentId || !sourceWorkspace) return null;
+ // Draft/review-only delivery remains with its explicit owner. A warning
+ // cannot grant the completion sweep authority to retire that checkout.
+ if (resolvePrCompletion(task?.metadata) === PR_COMPLETIONS.LEAVE_OPEN) return null;
+ const { getAgentRecord, recordCleanupOutcome, recordRepoStateVerification } = await import('./cosAgentLifecycle.js');
+ const agent = await getAgentRecord(agentId);
+ const branch = agent?.metadata?.worktreeBranch || agent?.metadata?.claimBranch
+ || cleanupWarnings.map(warning => warning.match(/(?:Auto-merge failed|PR creation failed) for branch (\S+?)(?=:|\s|$)/)?.[1]).find(Boolean);
+ if (branch) {
+ const recorded = await recordCleanupOutcome(agentId, { status: 'pending', reason: 'cleanup-warning-pending', branch,
+ path: agent?.metadata?.worktreePath || null, warnings: cleanupWarnings, sourceWorkspace, taskId: task?.id || null });
+ if (!recorded) throw new Error('cleanup-warning-receipt-unavailable');
}
+ const pending = await recordRepoStateVerification(agentId, { status: 'pending', reasons: ['cleanup-warning-pending'] });
+ if (!pending) throw new Error('cleanup-warning-replay-unavailable');
+ emitLog('info', 'Cleanup warning retained for deterministic completion recovery', { agentId, appName });
+ return null;
}
diff --git a/server/services/brainTaskThreads.js b/server/services/brainTaskThreads.js
index 5538e7708c..0df4e4584f 100644
--- a/server/services/brainTaskThreads.js
+++ b/server/services/brainTaskThreads.js
@@ -1,5 +1,6 @@
/** Bridge CoS tasks into the Brain bullet journal without creating noise. */
+import { TIMED_COOLDOWN_BLOCKED_CATEGORIES } from '../lib/taskBlockCategories.js';
import * as brainStorage from './brainStorage.js';
const TASK_REF_KIND = 'cos.task';
@@ -20,3 +21,60 @@ export async function ensureTaskThread({ taskId, title, nextAction, notes, prior
source: 'cos',
});
}
+
+// Exact legacy producer signatures, not a title-prefix filter. Only pristine
+// records with the original single task reference qualify for reconciliation.
+function legacyCleanupThread(thread, task) {
+ if (!task || thread.source !== 'cos' || thread.status !== 'open' || thread.priority !== 'high'
+ || !thread.createdAt || thread.updatedAt !== thread.createdAt || thread.refs?.length !== 1
+ || thread.refs[0].kind !== TASK_REF_KIND || thread.refs[0].id !== task.id
+ || thread.refs[0].label !== thread.title || thread.dueAt || thread.waitingOn
+ || thread.pinned || thread.tags?.length) return false;
+ // Unknown and actionable holds remain visible. A spent cognitive budget is
+ // not permission to conceal a real decision, credential or authorization gap.
+ if (task.approvalRequired || task.metadata?.blocker
+ || (!['pending', 'in_progress', 'completed', 'cancelled'].includes(task.status)
+ && !(task.status === 'blocked' && TIMED_COOLDOWN_BLOCKED_CATEGORIES.has(task.metadata?.blockedCategory)))
+ || task.metadata?.taskOutcome?.recovery?.mode === 'approval') return false;
+ const origin = task.metadata?.recoveryOrigin;
+ const repoMatch = task.description?.match(/^\[Recovery\] Finish incomplete cleanup for branch (.+) in (.+)$/);
+ if (origin?.subsystem === 'repository-cleanup' && origin.parentAgentId && repoMatch) {
+ return thread.title === `[Recovery] Finish incomplete cleanup for branch ${repoMatch[1]}`
+ && thread.nextAction === 'Let the recovery task finish, then verify the branch and worktree are clean.';
+ }
+ // The older warning producer had no recoveryOrigin. Its complete task and
+ // context signatures provide provenance; never infer it from a title alone.
+ const warningMatch = task.description?.match(/^\[Recovery\] (?:Resolve merge conflict and clean up stale branch|Investigate and retry failed (?:PR|MR) for branch) (\S+) in (.+)$/);
+ return Boolean(task.metadata?.isRecovery === true && warningMatch
+ && /^An agent (?:failed to auto-merge branch|pushed branch) "/.test(task.metadata?.context || '')
+ && /Original agent: .+, original task: .+\.$/.test(task.metadata?.context || '')
+ && thread.title === `Agent cleanup issue: ${warningMatch[2]}`
+ && thread.nextAction === 'Let the recovery task resolve the cleanup warning, then verify the worktree.');
+}
+
+/** Recoverable archive only: preserves content, refs and unresolved CoS work.
+ * Default dry run returns aggregate evidence suitable for logs or issue reports.
+ * A queued per-record recheck prevents clobbering a concurrent personal edit.
+ */
+export async function reconcileCleanupThreads({ dryRun = true } = {}) {
+ const { getTaskById } = await import('./cosTaskStore.js');
+ const threads = await brainStorage.getThreads();
+ let candidates = 0;
+ let archived = 0;
+ for (const thread of threads) {
+ if (thread.source !== 'cos' || thread.status !== 'open' || thread.refs?.length !== 1
+ || thread.refs[0]?.kind !== TASK_REF_KIND) continue;
+ const task = await getTaskById(thread.refs[0].id);
+ if (!legacyCleanupThread(thread, task)) continue;
+ candidates++;
+ if (dryRun) continue;
+ const updated = await brainStorage.updateWith('threads', thread.id, async fresh => {
+ const liveTask = await getTaskById(thread.refs[0].id);
+ if (!legacyCleanupThread(fresh, liveTask)) return null;
+ return { status: 'archived', cleanupReconciliation: { producer: 'cos-cleanup-v1',
+ taskId: liveTask.id, archivedAt: new Date().toISOString(), reason: 'machine-owned-obligation' } };
+ });
+ if (updated) archived++;
+ }
+ return { dryRun, candidates, archived };
+}
diff --git a/server/services/brainTaskThreads.test.js b/server/services/brainTaskThreads.test.js
new file mode 100644
index 0000000000..fdf0d5598f
--- /dev/null
+++ b/server/services/brainTaskThreads.test.js
@@ -0,0 +1,64 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+const state = vi.hoisted(() => ({ threads: [], tasks: new Map(), beforeUpdate: null }));
+vi.mock('./brainStorage.js', () => ({
+ getThreads: async () => structuredClone(state.threads),
+ updateWith: async (_type, id, update) => {
+ const row = state.threads.find(thread => thread.id === id);
+ state.beforeUpdate?.(row);
+ const patch = await update(row);
+ if (!patch) return null;
+ Object.assign(row, patch, { updatedAt: '2026-10-25T00:00:00Z' });
+ return row;
+ },
+}));
+vi.mock('./cosTaskStore.js', () => ({ getTaskById: async id => state.tasks.get(id) }));
+import { reconcileCleanupThreads } from './brainTaskThreads.js';
+const fixture = (id, taskPatch = {}, threadPatch = {}) => {
+ const title = '[Recovery] Finish incomplete cleanup for branch claim/example';
+ state.tasks.set(id, { id, status: 'pending', description: `${title} in Example`,
+ metadata: { recoveryOrigin: { subsystem: 'repository-cleanup', parentAgentId: 'original' } }, ...taskPatch });
+ state.threads.push({ id, source: 'cos', status: 'open', priority: 'high', title,
+ nextAction: 'Let the recovery task finish, then verify the branch and worktree are clean.',
+ refs: [{ kind: 'cos.task', id, label: title }], notes: 'Synthetic diagnostic',
+ createdAt: '2026-10-24T00:00:00Z', updatedAt: '2026-10-24T00:00:00Z', ...threadPatch });
+};
+beforeEach(() => { state.threads = []; state.tasks.clear(); state.beforeUpdate = null; });
+describe('legacy cleanup obligation reconciliation', () => {
+ it('previews, archives recoverably and remains idempotent without completing machine work', async () => {
+ fixture('owned');
+ expect(await reconcileCleanupThreads()).toEqual({ dryRun: true, candidates: 1, archived: 0 });
+ expect(state.threads[0].status).toBe('open');
+ expect(await reconcileCleanupThreads({ dryRun: false })).toEqual({ dryRun: false, candidates: 1, archived: 1 });
+ expect(state.threads[0]).toMatchObject({ status: 'archived', notes: 'Synthetic diagnostic',
+ cleanupReconciliation: { taskId: 'owned', reason: 'machine-owned-obligation' } });
+ expect(state.tasks.get('owned').status).toBe('pending');
+ expect(await reconcileCleanupThreads({ dryRun: false })).toMatchObject({ candidates: 0, archived: 0 });
+ });
+ it('preserves personal edits, missing provenance and real decision/credential blockers', async () => {
+ fixture('edited', {}, { updatedAt: '2026-10-24T01:00:00Z' });
+ fixture('personal', {}, { source: 'manual' });
+ fixture('unknown', { metadata: {} });
+ fixture('credential', { status: 'blocked', metadata: { blockedCategory: 'provider-config' } });
+ fixture('decision', { approvalRequired: true });
+ fixture('missing'); state.tasks.delete('missing');
+ expect(await reconcileCleanupThreads({ dryRun: false })).toMatchObject({ candidates: 0, archived: 0 });
+ expect(state.threads.every(row => row.status === 'open')).toBe(true);
+ });
+ it('archives exact warning-producer and timed machine holds while preserving exhausted decisions', async () => {
+ fixture('timed', { status: 'blocked', metadata: { blockedCategory: 'worktree-busy', recoveryOrigin: { subsystem: 'repository-cleanup', parentAgentId: 'original' } } });
+ fixture('exhausted', { status: 'blocked', metadata: { blockedCategory: 'task-recovery-exhausted' } });
+ const title = 'Agent cleanup issue: Example';
+ fixture('warning', { description: '[Recovery] Investigate and retry failed PR for branch claim/example in Example',
+ metadata: { isRecovery: true, context: 'An agent pushed branch "claim/example" to /synthetic but automated PR creation failed. Original agent: original, original task: Example.' } }, { title, nextAction: 'Let the recovery task resolve the cleanup warning, then verify the worktree.',
+ refs: [{ kind: 'cos.task', id: 'warning', label: title }] });
+ expect(await reconcileCleanupThreads({ dryRun: false })).toMatchObject({ candidates: 2, archived: 2 });
+ expect(state.threads.find(row => row.id === 'exhausted').status).toBe('open');
+ });
+
+ it('rechecks a personal edit inside the serialized update', async () => {
+ fixture('race');
+ state.beforeUpdate = row => { row.updatedAt = '2026-10-24T02:00:00Z'; row.notes = 'My own next step'; };
+ expect(await reconcileCleanupThreads({ dryRun: false })).toMatchObject({ candidates: 1, archived: 0 });
+ expect(state.threads[0].notes).toBe('My own next step');
+ });
+});
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 0f2c6e9047..9ea7558017 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -45,6 +45,8 @@ vi.mock('../lib/uuid.js', () => ({
}));
vi.mock('./cosAgentLifecycle.js', () => ({
+ recordCleanupOutcome: vi.fn(async () => true),
+ recordRepoStateVerification: vi.fn(async () => true),
registerAgent: vi.fn().mockResolvedValue(undefined),
updateAgent: vi.fn().mockResolvedValue(undefined),
completeAgent: vi.fn().mockResolvedValue(undefined),
@@ -2206,188 +2208,26 @@ describe('spawnReviewLoopFollowUp', () => {
});
});
-describe('spawnMergeRecoveryTask', () => {
- let logEvents;
- let captureLog;
-
- beforeEach(() => {
- logEvents = [];
- captureLog = entry => logEvents.push(entry);
- cosEvents.on('log', captureLog);
+describe('cleanup warning recovery', () => {
+ it('leaves failed persistence unresolved instead of claiming recovery was recorded', async () => {
+ const { recordRepoStateVerification } = await import('./cosAgentLifecycle.js');
+ vi.mocked(recordRepoStateVerification).mockResolvedValueOnce(false);
+ await expect(spawnMergeRecoveryTask(['cleanup unavailable'], 'agent-1', { id: 'original' }, 'Example', '/mock/workspace'))
+ .rejects.toThrow('cleanup-warning-replay-unavailable');
});
- afterEach(() => {
- cosEvents.off('log', captureLog);
- });
-
- beforeEach(() => {
+ it('keeps repeated failures on the original durable receipt without successor tasks', async () => {
+ const { recordCleanupOutcome, recordRepoStateVerification } = await import('./cosAgentLifecycle.js');
vi.clearAllMocks();
- addTask.mockResolvedValue({ id: 'task-recovery' });
- });
-
- it('should create a recovery task when merge failure warning is present', async () => {
- const warnings = ['Auto-merge failed for branch cos/task-abc123/agent-1 — branch preserved for manual recovery'];
- const task = { id: 'task-original', description: 'Fix deps', metadata: { app: 'sparsetree' } };
-
- const result = await spawnMergeRecoveryTask(warnings, 'agent-1', task, 'SparseTree', '/mock/workspace');
-
- expect(addTask).toHaveBeenCalledTimes(1);
- expect(addTask).toHaveBeenCalledWith(
- expect.objectContaining({
- description: expect.stringContaining('[Recovery]'),
- priority: 'HIGH',
- app: 'sparsetree',
- context: expect.stringContaining('cos/task-abc123/agent-1'),
- useWorktree: false,
- }),
- 'user'
- );
- expect(result).toEqual({ id: 'task-recovery' });
- expect(logEvents).toContainEqual(expect.objectContaining({
- level: 'info',
- message: expect.stringContaining('Auto-created merge recovery task'),
- }));
- });
-
- it('should include branch name and repo path in recovery context', async () => {
- const warnings = ['Auto-merge failed for branch feature/my-branch — branch preserved for manual recovery'];
- const task = { id: 'task-1', description: 'Original task', metadata: { app: 'myapp' } };
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', task, 'MyApp', '/mock/workspace');
-
- const call = addTask.mock.calls[0];
- expect(call[0].context).toContain('feature/my-branch');
- expect(call[0].context).toContain('/mock/workspace');
- expect(call[0].context).toContain('agent-1');
- expect(call[0].description).toContain('feature/my-branch');
- expect(call[0].description).toContain('MyApp');
- });
-
- it('should not create a task when no merge failure warning exists', async () => {
- const warnings = ['Worktree cleanup failed: some other error'];
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', {}, 'TestApp', '/mock/workspace');
-
- expect(addTask).not.toHaveBeenCalled();
- });
-
- it('should not create a task when warnings array is empty', async () => {
- await spawnMergeRecoveryTask([], 'agent-1', {}, 'TestApp', '/mock/workspace');
-
- expect(addTask).not.toHaveBeenCalled();
- });
-
- it('should not create a task when sourceWorkspace is undefined', async () => {
- const warnings = ['Auto-merge failed for branch cos/task-abc/agent-1 — branch preserved'];
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', {}, 'TestApp', undefined);
-
- expect(addTask).not.toHaveBeenCalled();
- });
-
- it('should not create a task when sourceWorkspace is null', async () => {
- const warnings = ['Auto-merge failed for branch cos/task-abc/agent-1 — branch preserved'];
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', {}, 'TestApp', null);
-
- expect(addTask).not.toHaveBeenCalled();
- });
-
- // Rejected persistence must retain its diagnostic without claiming a task exists.
- it.each([
- ['merge', 'Auto-merge failed for branch cos/task-abc/agent-1 — branch preserved', 'gh'],
- ['PR', 'PR creation failed for branch cos/task-abc/agent-1: forge error. Worktree preserved.', 'gh'],
- ['MR', 'PR creation failed for branch cos/task-abc/agent-1: forge error. Worktree preserved.', 'glab'],
- ])('should handle addTask failure gracefully for %s recovery', async (kind, warning, cli) => {
- if (kind !== 'merge') git.resolveForgeForRepo.mockResolvedValueOnce({ cli });
- addTask.mockRejectedValue(new Error('write failed'));
-
- await expect(spawnMergeRecoveryTask(
- [warning], 'agent-1', { metadata: {} }, 'TestApp', '/mock/workspace'
- )).resolves.toBeNull();
-
- expect(addTask).toHaveBeenCalledTimes(1);
- expect(logEvents).toContainEqual(expect.objectContaining({
- level: 'warn',
- message: `Failed to create ${kind} recovery task: write failed`,
- agentId: 'agent-1',
- staleBranch: 'cos/task-abc/agent-1',
- }));
- expect(logEvents.some(entry => entry.message.includes('Auto-created'))).toBe(false);
- });
-
- it('should use "unknown" for task description when not provided', async () => {
- const warnings = ['Auto-merge failed for branch cos/branch-1 — branch preserved'];
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', { metadata: {} }, 'TestApp', '/mock/workspace');
-
- const call = addTask.mock.calls[0];
- expect(call[0].context).toContain('original task: unknown');
- });
-
- it('should create a PR recovery task when a PR creation failure warning is present', async () => {
- const warnings = ['PR creation failed for branch cos/task-xyz/agent-1: GraphQL: some error. Worktree preserved for manual PR creation.'];
- const task = { id: 'task-original', description: 'Add feature', metadata: { app: 'myapp' } };
-
- const result = await spawnMergeRecoveryTask(warnings, 'agent-1', task, 'MyApp', '/mock/workspace');
-
- expect(addTask).toHaveBeenCalledTimes(1);
- expect(addTask).toHaveBeenCalledWith(
- expect.objectContaining({
- description: expect.stringContaining('[Recovery]'),
- priority: 'HIGH',
- app: 'myapp',
- context: expect.stringContaining('cos/task-xyz/agent-1'),
- useWorktree: false,
- }),
- 'user'
- );
- expect(result).toEqual({ id: 'task-recovery' });
- expect(logEvents).toContainEqual(expect.objectContaining({
- level: 'info',
- message: expect.stringContaining('Auto-created PR recovery task'),
- }));
- });
-
- it('should include branch name and workspace in PR recovery context', async () => {
- const warnings = ['PR creation failed for branch feature/my-pr-branch: gh exited with code 1. Worktree preserved for manual PR creation.'];
- const task = { id: 'task-1', description: 'Original task', metadata: { app: 'myapp' } };
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', task, 'MyApp', '/mock/workspace');
-
- const call = addTask.mock.calls[0];
- expect(call[0].context).toContain('feature/my-pr-branch');
- expect(call[0].context).toContain('/mock/workspace');
- expect(call[0].description).toContain('feature/my-pr-branch');
- expect(call[0].description).toContain('MyApp');
- });
-
- it('should not create a PR recovery task when sourceWorkspace is missing', async () => {
- const warnings = ['PR creation failed for branch cos/task-xyz/agent-1: some error. Worktree preserved for manual PR creation.'];
-
- await spawnMergeRecoveryTask(warnings, 'agent-1', {}, 'TestApp', null);
-
+ for (let i = 0; i < 3; i++) {
+ await spawnMergeRecoveryTask(['PR creation failed for branch claim/example: unavailable forge'], 'agent-1',
+ { id: 'original', metadata: { isRecovery: true } }, 'Example', '/mock/workspace');
+ }
expect(addTask).not.toHaveBeenCalled();
- });
-
- it('emits glab/MR commands when the source workspace is a GitLab repo', async () => {
- git.resolveForgeForRepo.mockResolvedValueOnce({ cli: 'glab', env: process.env, host: 'gitlab.com', owner: 'mygroup', account: null });
-
- const warnings = ['PR creation failed for branch feature/x: glab error. Worktree preserved for manual PR creation.'];
- const task = { id: 'task-gl', description: 'Add thing', metadata: { app: 'gl-app' } };
-
- await spawnMergeRecoveryTask(warnings, 'agent-gl', task, 'GitLabApp', '/mock/gl-workspace');
-
- expect(addTask).toHaveBeenCalledTimes(1);
- const call = addTask.mock.calls[0][0];
- expect(call.description).toContain('MR');
- expect(call.context).toContain('glab mr list --source-branch feature/x');
- expect(call.context).toContain('glab mr create --source-branch feature/x --target-branch main');
- expect(call.context).not.toContain('gh pr ');
- expect(logEvents).toContainEqual(expect.objectContaining({
- level: 'info',
- message: expect.stringContaining('Auto-created MR recovery task'),
+ expect(recordCleanupOutcome).toHaveBeenCalledWith('agent-1', expect.objectContaining({
+ status: 'pending', reason: 'cleanup-warning-pending', taskId: 'original',
}));
+ expect(recordRepoStateVerification).toHaveBeenCalledWith('agent-1', { status: 'pending', reasons: ['cleanup-warning-pending'] });
});
});