Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve cleanup ownership and evidence across agent recovery

Adam Eivy5c54afa87aed
diff --git a/server/lib/taskBlockCategories.js b/server/lib/taskBlockCategories.js
index 0ec2906285..5e5f5bc62a 100644
--- a/server/lib/taskBlockCategories.js
+++ b/server/lib/taskBlockCategories.js
@@ -58,6 +58,8 @@ export const TIMED_COOLDOWN_BLOCKED_CATEGORIES = new Set([
*/
export const PAUSED_BLOCKED_CATEGORIES = new Set([
...TIMED_COOLDOWN_BLOCKED_CATEGORIES,
+ 'task-approval', // A user decision must retain its checkout and never silently expire.
+ 'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.
'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.
'app-unresolved', // the task's app has no usable Repository Path
'workspace-invalid', // the resolved workspace isn't a usable directory
@@ -86,6 +88,8 @@ export const USER_DECISION_BLOCKED_CATEGORIES = new Set([
'user-terminated', // user explicitly stopped the agent
AGENT_PAUSED_CATEGORY, // user paused; resumable on demand
'challenge-escalation', // parked awaiting the user's arbitration
+ 'task-approval', // A user decision must retain its checkout and never silently expire.
+ 'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.
'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.
'app-unresolved',
'workspace-invalid',
diff --git a/server/lib/taskRetryHold.js b/server/lib/taskRetryHold.js
index d03017e2b6..a9a2747ecc 100644
--- a/server/lib/taskRetryHold.js
+++ b/server/lib/taskRetryHold.js
@@ -109,3 +109,7 @@ export function isStaleRetryHold(metadata, now = Date.now(), graceMs = RETRY_HOL
if (!Number.isFinite(since)) return true;
return now - since >= graceMs;
}
+
+/** Repository verification owns these tasks across completion and restarts. */
+export const isDeliveryCleanupHeld = metadata => metadata?.deliveryRecovery === 'cleanup'
+ && typeof metadata.deliveryRecoveryAgentId === 'string' && metadata.deliveryRecoveryAgentId.length > 0;
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 669a39c0a0..37973a8db0 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1,5 +1,5 @@
import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';
-import { retryHoldMetadata } from '../lib/taskRetryHold.js';
+import { retryHoldMetadata, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';
import { PR_COMPLETIONS, resolvePrCompletion, PR_MISSING_CATEGORY, prClaimWasVerified } from '../lib/prDisposition.js';
import { isAuditTaskType } from '../lib/auditCatalog.js';
import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';
@@ -1257,8 +1257,12 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
);
}
+ const savedCleanup = isDeliveryCleanupHeld(task?.metadata) && task.metadata.deliveryRecoveryAgentId === agent.id;
+ const cleanupAllowed = savedCleanup || (executionSuccess && validationPassed !== false
+ && (agent.metadata?.isWorktree === true || isTruthyMeta(agent.metadata?.claimPicksOwnBranch)));
+ const recoveryOutcome = !cleanupAllowed && taskOutcome?.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome;
const recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'
- ? resolveUndeliveredTaskUpdate(task, (!executionSuccess || validationPassed === false || (!agent.metadata?.isWorktree && !isTruthyMeta(agent.metadata?.claimPicksOwnBranch))) && taskOutcome.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome, agent.id) : null;
+ ? resolveUndeliveredTaskUpdate(task, recoveryOutcome, agent.id) : null;
if (recoveryUpdate) {
const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user');
if (update?.error) throw new Error(`Could not persist recovered task outcome: ${update.error}`);
diff --git a/server/services/agentFinalization.outputHooks.test.js b/server/services/agentFinalization.outputHooks.test.js
index 8bb8e082e0..92ec52c1cb 100644
--- a/server/services/agentFinalization.outputHooks.test.js
+++ b/server/services/agentFinalization.outputHooks.test.js
@@ -397,6 +397,22 @@ describe('recovery output-hook dispatch (#3182)', () => {
} finally { isProgrammaticIoTaskType.mockReturnValue(true); }
});
+ it('retains an already persisted cleanup handoff when its run is recovered after a crash', async () => {
+ const { mkdtemp, writeFile, rm } = await import('node:fs/promises');
+ const { tmpdir } = await import('node:os');
+ const workspacePath = await mkdtemp(join(tmpdir(), 'cleanup-restart-'));
+ const taskOutcome = { status: 'incomplete', reason: 'Only preservation remains.', recovery: { mode: 'cleanup' } };
+ const task = { ...TASK, metadata: { ...TASK.metadata, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' } };
+ persistedAgent.metadata = { workspacePath, isWorktree: true };
+ try {
+ await writeFile(join(workspacePath, '.agent-done-agent-example'), JSON.stringify({ taskOutcome }));
+ await retireDeadAgent({ agent: persistedAgent, task, success: false, exitCode: 1, duration: 1000 });
+ expect(updateTask).toHaveBeenLastCalledWith(TASK.id, expect.objectContaining({ status: 'in_progress',
+ metadata: expect.objectContaining({ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' }) }), TASK.taskType);
+ expect(completeAgent).toHaveBeenLastCalledWith('agent-example', expect.objectContaining({ success: false, cleanupPending: true }));
+ } finally { await rm(workspacePath, { recursive: true, force: true }); }
+ });
+
it('stops repeated delivery recovery at the existing budget with concrete choices', async () => {
const taskOutcome = { status: 'incomplete', reason: 'Validation remains red.' };
await finalizeAgent({ agentId: 'agent-example', task: { ...TASK, metadata: { ...TASK.metadata, failureCount: MAX_TASK_RETRIES - 1 } },
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index a620fb3d9c..a4282ae5f1 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -36,7 +36,7 @@ import { activeAgents, runnerAgents, completingAgents, spawningTasks, userTermin
// for handleOrphanedTask. Importing them from their own leaf modules is what
// lets that edge be a plain static import instead of a dynamic-import dodge.
import { cleanupAgentWorktree, resolveTaskResumePatch } from './agentWorktreeCleanup.js';
-import { RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY, isRetryHeld, clearedRetryHoldMetadata } from '../lib/taskRetryHold.js';
+import { RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY, isRetryHeld, clearedRetryHoldMetadata, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';
import { CLAIM_METADATA_KEYS } from './cosTaskClaim.js';
import { REQUEUED_AT_KEY, LAST_SPAWNED_AT_KEY } from '../lib/taskRequeue.js';
import { resolveTaskTargetBranch } from '../lib/taskTargetBranch.js';
@@ -1490,6 +1490,7 @@ export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agent
return;
}
+ if (isDeliveryCleanupHeld(task.metadata)) return;
const ctx = { taskId, agentId, task, agentMetadata, agentStartedAt };
const guard = classifyOrphanGuard(task, agentMetadata);
if (guard) return ORPHAN_GUARD_HANDLERS[guard](ctx);
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index 483ebc5447..42cd3b77c0 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -799,6 +799,13 @@ describe('handleOrphanedTask — persisted outcome shapes and guard precedence (
vi.useRealTimers();
});
+ it('does not rescue commits or requeue a task owned by repository cleanup after restart', async () => {
+ await run(taskWith({ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-dead' }));
+ expect(updateTask).not.toHaveBeenCalled();
+ expect(committedDuringRun).not.toHaveBeenCalled();
+ expect(resolveTaskResumePatch).not.toHaveBeenCalled();
+ });
+
it('requeues a genuine orphan: charges a retry, stamps the orphan breadcrumb, clears interruptedByRestart', async () => {
resolveTaskResumePatch.mockResolvedValueOnce({ resumeBranch: 'branch-a' });
await run(taskWith({ keep: 'x', orphanRetryCount: 1, totalSpawnCount: 2, interruptedByRestart: true }));
diff --git a/server/services/cos.js b/server/services/cos.js
index 814009057a..8686b7d3fb 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -23,7 +23,7 @@ import { join } from 'path';
import { getActiveProvider } from './providers.js';
import { isInternalTaskId } from '../lib/taskParser.js';
import { unclaimedTaskIds, runningAgentsByTaskId, spawningAgentForTask, spawnClaimAgeMs } from '../lib/cosSpawnWindow.js';
-import { isRetryHeld, isStaleRetryHold } from '../lib/taskRetryHold.js';
+import { isRetryHeld, isStaleRetryHold, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';
import { clearStaleActiveAgents } from './appActivity.js';
// The single Priority-0 on-demand loop body, shared with the evaluateTasks
// engine in cosTaskGenerator.js so the two can no longer drift (#6618).
@@ -883,6 +883,7 @@ export async function resetOrphanedTasks({ bootRecovery = false } = {}) {
// bypasses: the hold is armed AFTER `completeAgent`, so a crash-restart
// inside that 60s window would otherwise skip the very task the boot pass
// exists to recover and strand it until the 15-minute periodic sweep.
+ if (isDeliveryCleanupHeld(task.metadata)) continue;
const held = isRetryHeld(task.metadata);
const recoverHold = held && (bootRecovery || isStaleRetryHold(task.metadata));
if (held && !recoverHold) {
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index 070166fc17..df5c301c64 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -2035,6 +2035,14 @@ describe('resetOrphanedTasks — latest run outcome (#12057)', () => {
afterEach(() => vi.restoreAllMocks());
+ it.each([false, true])('leaves durable cleanup handoffs to verification during boot=%s', async bootRecovery => {
+ taskStore.getAllTasks.mockResolvedValue({ user: { grouped: { in_progress: [{ id: taskId,
+ status: 'in_progress', metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: latest.id } }] } }, cos: { grouped: {} } });
+ await resetOrphanedTasks({ bootRecovery });
+ expect(taskStore.updateTask).not.toHaveBeenCalled();
+ expect(agentManagement.handleOrphanedTask).not.toHaveBeenCalled();
+ });
+
it('recovers the latest failed rerun despite an older successful run', async () => {
await resetOrphanedTasks();
diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.js
index bbd4da4602..11e3a546b1 100644
--- a/server/services/cosAgentLifecycle.js
+++ b/server/services/cosAgentLifecycle.js
@@ -288,7 +288,7 @@ export async function completeAgent(agentId, result = {}) {
let completedCheckouts;
if (agent.metadata?.claimPicksOwnBranch || agent.metadata?.isWorktree) {
completedCheckouts = [];
- if (result.success === true) {
+ if (result.success === true || result.cleanupPending === true) {
const { captureCompletedCheckouts } = await import('./completedCheckoutEvidence.js');
completedCheckouts = await captureCompletedCheckouts(agent, { success: true }).catch(() => []);
}
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index 3af715402b..86f20434c1 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -543,7 +543,7 @@ describe('completeAgent idempotence (#3384)', () => {
cosEvents.removeAllListeners('agent:completed');
});
- it('archives checkout evidence captured while the successful claim still owns its bindings', async () => {
+ it.each([true, false])('archives checkout evidence before releasing a delivered or cleanup-pending claim (success=%s)', async success => {
const agentId = 'claim-completion';
const receipt = { version: 1, agentId, branch: 'claim/issue-1' };
mockCosState.state.agents[agentId] = { id: agentId, status: 'running', startedAt: '2026-01-01T00:00:00Z',
@@ -553,7 +553,9 @@ describe('completeAgent idempotence (#3384)', () => {
expect(mockCosState.state.agents[agentId].status).toBe('running');
return [receipt];
});
- const result = await completeAgent(agentId, { success: true });
+ const result = await completeAgent(agentId, { success, cleanupPending: !success });
+ expect(result.result.success).toBe(success);
+ expect(result.metadata.repoStateVerification.status).toBe('pending');
expect(result.metadata.completedCheckouts).toEqual([receipt]);
const archived = JSON.parse(await readFile(join(mockCosState.agentsDir, result.completedAt.slice(0, 10), agentId, 'metadata.json'), 'utf8'));
expect(archived.metadata.completedCheckouts).toEqual([receipt]);
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 26adea2ccd..9bc5f3ac50 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -2472,6 +2472,16 @@ describe('cosTaskStore — stale failure-artifact reaper (#2619)', () => {
// CLI provider — so the reaper must leave it alone no matter how old it gets.
// Flipping it to `completed` with `resolution: 'auto-expired'` would report
// work nobody dropped as done, and federate that to every peer (#6193).
+ it.each(['task-approval', 'task-recovery-exhausted'])('preserves the checkout and human decision for stale %s', async blockedCategory => {
+ await addTask({ description: 'delivery decision', id: 'task-decision', priority: 'HIGH' }, 'user', { now: NOW });
+ await updateTask('task-decision', { metadata: { existingBranch: 'fix/retained', resumedFromAgentId: 'agent-old', resumeWorktreePath: '/example/retained' } }, 'user', { now: NOW });
+ await updateTask('task-decision', { status: 'blocked', metadata: { blockedCategory, blockedAt: daysAgo(90) } }, 'user', { now: NOW });
+ expect((await sweepResolvedFailureTasks({ now: NOW })).reaped).toBe(0);
+ const retained = await getTaskById('task-decision');
+ expect(retained.status).toBe('blocked');
+ expect(retained.metadata).toMatchObject({ existingBranch: 'fix/retained', resumedFromAgentId: 'agent-old', resumeWorktreePath: '/example/retained' });
+ });
+
it('never auto-expires a provider-config block, however stale', async () => {
await addTask({ description: 'api-only provider pinned', id: 'task-pc-old', priority: 'HIGH' }, 'user', { now: NOW });
await updateTask('task-pc-old', { status: 'blocked', metadata: { blockedCategory: 'provider-config', blockedAt: daysAgo(90) } }, 'user', { now: NOW });