Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve cleanup ownership and evidence across agent recovery
Adam Eivy5c54afa87aed
diff --git a/server/lib/taskBlockCategories.js b/server/lib/taskBlockCategories.jsindex 0ec2906285..5e5f5bc62a 100644--- a/server/lib/taskBlockCategories.js+++ b/server/lib/taskBlockCategories.js@@ -58,6 +58,8 @@ export const TIMED_COOLDOWN_BLOCKED_CATEGORIES = new Set([*/export const PAUSED_BLOCKED_CATEGORIES = new Set([...TIMED_COOLDOWN_BLOCKED_CATEGORIES,+ 'task-approval', // A user decision must retain its checkout and never silently expire.+ 'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.'app-unresolved', // the task's app has no usable Repository Path'workspace-invalid', // the resolved workspace isn't a usable directory@@ -86,6 +88,8 @@ export const USER_DECISION_BLOCKED_CATEGORIES = new Set(['user-terminated', // user explicitly stopped the agentAGENT_PAUSED_CATEGORY, // user paused; resumable on demand'challenge-escalation', // parked awaiting the user's arbitration+ 'task-approval', // A user decision must retain its checkout and never silently expire.+ 'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.'app-unresolved','workspace-invalid',diff --git a/server/lib/taskRetryHold.js b/server/lib/taskRetryHold.jsindex d03017e2b6..a9a2747ecc 100644--- a/server/lib/taskRetryHold.js+++ b/server/lib/taskRetryHold.js@@ -109,3 +109,7 @@ export function isStaleRetryHold(metadata, now = Date.now(), graceMs = RETRY_HOLif (!Number.isFinite(since)) return true;return now - since >= graceMs;}++/** Repository verification owns these tasks across completion and restarts. */+export const isDeliveryCleanupHeld = metadata => metadata?.deliveryRecovery === 'cleanup'+ && typeof metadata.deliveryRecoveryAgentId === 'string' && metadata.deliveryRecoveryAgentId.length > 0;diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.jsindex 669a39c0a0..37973a8db0 100644--- a/server/services/agentFinalization.js+++ b/server/services/agentFinalization.js@@ -1,5 +1,5 @@import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';-import { retryHoldMetadata } from '../lib/taskRetryHold.js';+import { retryHoldMetadata, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';import { PR_COMPLETIONS, resolvePrCompletion, PR_MISSING_CATEGORY, prClaimWasVerified } from '../lib/prDisposition.js';import { isAuditTaskType } from '../lib/auditCatalog.js';import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';@@ -1257,8 +1257,12 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration);}+ const savedCleanup = isDeliveryCleanupHeld(task?.metadata) && task.metadata.deliveryRecoveryAgentId === agent.id;+ const cleanupAllowed = savedCleanup || (executionSuccess && validationPassed !== false+ && (agent.metadata?.isWorktree === true || isTruthyMeta(agent.metadata?.claimPicksOwnBranch)));+ const recoveryOutcome = !cleanupAllowed && taskOutcome?.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome;const recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'- ? resolveUndeliveredTaskUpdate(task, (!executionSuccess || validationPassed === false || (!agent.metadata?.isWorktree && !isTruthyMeta(agent.metadata?.claimPicksOwnBranch))) && taskOutcome.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome, agent.id) : null;+ ? resolveUndeliveredTaskUpdate(task, recoveryOutcome, agent.id) : null;if (recoveryUpdate) {const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user');if (update?.error) throw new Error(`Could not persist recovered task outcome: ${update.error}`);diff --git a/server/services/agentFinalization.outputHooks.test.js b/server/services/agentFinalization.outputHooks.test.jsindex 8bb8e082e0..92ec52c1cb 100644--- a/server/services/agentFinalization.outputHooks.test.js+++ b/server/services/agentFinalization.outputHooks.test.js@@ -397,6 +397,22 @@ describe('recovery output-hook dispatch (#3182)', () => {} finally { isProgrammaticIoTaskType.mockReturnValue(true); }});+ it('retains an already persisted cleanup handoff when its run is recovered after a crash', async () => {+ const { mkdtemp, writeFile, rm } = await import('node:fs/promises');+ const { tmpdir } = await import('node:os');+ const workspacePath = await mkdtemp(join(tmpdir(), 'cleanup-restart-'));+ const taskOutcome = { status: 'incomplete', reason: 'Only preservation remains.', recovery: { mode: 'cleanup' } };+ const task = { ...TASK, metadata: { ...TASK.metadata, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' } };+ persistedAgent.metadata = { workspacePath, isWorktree: true };+ try {+ await writeFile(join(workspacePath, '.agent-done-agent-example'), JSON.stringify({ taskOutcome }));+ await retireDeadAgent({ agent: persistedAgent, task, success: false, exitCode: 1, duration: 1000 });+ expect(updateTask).toHaveBeenLastCalledWith(TASK.id, expect.objectContaining({ status: 'in_progress',+ metadata: expect.objectContaining({ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' }) }), TASK.taskType);+ expect(completeAgent).toHaveBeenLastCalledWith('agent-example', expect.objectContaining({ success: false, cleanupPending: true }));+ } finally { await rm(workspacePath, { recursive: true, force: true }); }+ });+it('stops repeated delivery recovery at the existing budget with concrete choices', async () => {const taskOutcome = { status: 'incomplete', reason: 'Validation remains red.' };await finalizeAgent({ agentId: 'agent-example', task: { ...TASK, metadata: { ...TASK.metadata, failureCount: MAX_TASK_RETRIES - 1 } },diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex a620fb3d9c..a4282ae5f1 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -36,7 +36,7 @@ import { activeAgents, runnerAgents, completingAgents, spawningTasks, userTermin// for handleOrphanedTask. Importing them from their own leaf modules is what// lets that edge be a plain static import instead of a dynamic-import dodge.import { cleanupAgentWorktree, resolveTaskResumePatch } from './agentWorktreeCleanup.js';-import { RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY, isRetryHeld, clearedRetryHoldMetadata } from '../lib/taskRetryHold.js';+import { RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY, isRetryHeld, clearedRetryHoldMetadata, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';import { CLAIM_METADATA_KEYS } from './cosTaskClaim.js';import { REQUEUED_AT_KEY, LAST_SPAWNED_AT_KEY } from '../lib/taskRequeue.js';import { resolveTaskTargetBranch } from '../lib/taskTargetBranch.js';@@ -1490,6 +1490,7 @@ export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agentreturn;}+ if (isDeliveryCleanupHeld(task.metadata)) return;const ctx = { taskId, agentId, task, agentMetadata, agentStartedAt };const guard = classifyOrphanGuard(task, agentMetadata);if (guard) return ORPHAN_GUARD_HANDLERS[guard](ctx);diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.jsindex 483ebc5447..42cd3b77c0 100644--- a/server/services/agentManagement.test.js+++ b/server/services/agentManagement.test.js@@ -799,6 +799,13 @@ describe('handleOrphanedTask — persisted outcome shapes and guard precedence (vi.useRealTimers();});+ it('does not rescue commits or requeue a task owned by repository cleanup after restart', async () => {+ await run(taskWith({ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-dead' }));+ expect(updateTask).not.toHaveBeenCalled();+ expect(committedDuringRun).not.toHaveBeenCalled();+ expect(resolveTaskResumePatch).not.toHaveBeenCalled();+ });+it('requeues a genuine orphan: charges a retry, stamps the orphan breadcrumb, clears interruptedByRestart', async () => {resolveTaskResumePatch.mockResolvedValueOnce({ resumeBranch: 'branch-a' });await run(taskWith({ keep: 'x', orphanRetryCount: 1, totalSpawnCount: 2, interruptedByRestart: true }));diff --git a/server/services/cos.js b/server/services/cos.jsindex 814009057a..8686b7d3fb 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -23,7 +23,7 @@ import { join } from 'path';import { getActiveProvider } from './providers.js';import { isInternalTaskId } from '../lib/taskParser.js';import { unclaimedTaskIds, runningAgentsByTaskId, spawningAgentForTask, spawnClaimAgeMs } from '../lib/cosSpawnWindow.js';-import { isRetryHeld, isStaleRetryHold } from '../lib/taskRetryHold.js';+import { isRetryHeld, isStaleRetryHold, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';import { clearStaleActiveAgents } from './appActivity.js';// The single Priority-0 on-demand loop body, shared with the evaluateTasks// engine in cosTaskGenerator.js so the two can no longer drift (#6618).@@ -883,6 +883,7 @@ export async function resetOrphanedTasks({ bootRecovery = false } = {}) {// bypasses: the hold is armed AFTER `completeAgent`, so a crash-restart// inside that 60s window would otherwise skip the very task the boot pass// exists to recover and strand it until the 15-minute periodic sweep.+ if (isDeliveryCleanupHeld(task.metadata)) continue;const held = isRetryHeld(task.metadata);const recoverHold = held && (bootRecovery || isStaleRetryHold(task.metadata));if (held && !recoverHold) {diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex 070166fc17..df5c301c64 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -2035,6 +2035,14 @@ describe('resetOrphanedTasks — latest run outcome (#12057)', () => {afterEach(() => vi.restoreAllMocks());+ it.each([false, true])('leaves durable cleanup handoffs to verification during boot=%s', async bootRecovery => {+ taskStore.getAllTasks.mockResolvedValue({ user: { grouped: { in_progress: [{ id: taskId,+ status: 'in_progress', metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: latest.id } }] } }, cos: { grouped: {} } });+ await resetOrphanedTasks({ bootRecovery });+ expect(taskStore.updateTask).not.toHaveBeenCalled();+ expect(agentManagement.handleOrphanedTask).not.toHaveBeenCalled();+ });+it('recovers the latest failed rerun despite an older successful run', async () => {await resetOrphanedTasks();diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.jsindex bbd4da4602..11e3a546b1 100644--- a/server/services/cosAgentLifecycle.js+++ b/server/services/cosAgentLifecycle.js@@ -288,7 +288,7 @@ export async function completeAgent(agentId, result = {}) {let completedCheckouts;if (agent.metadata?.claimPicksOwnBranch || agent.metadata?.isWorktree) {completedCheckouts = [];- if (result.success === true) {+ if (result.success === true || result.cleanupPending === true) {const { captureCompletedCheckouts } = await import('./completedCheckoutEvidence.js');completedCheckouts = await captureCompletedCheckouts(agent, { success: true }).catch(() => []);}diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.jsindex 3af715402b..86f20434c1 100644--- a/server/services/cosAgentLifecycle.test.js+++ b/server/services/cosAgentLifecycle.test.js@@ -543,7 +543,7 @@ describe('completeAgent idempotence (#3384)', () => {cosEvents.removeAllListeners('agent:completed');});- it('archives checkout evidence captured while the successful claim still owns its bindings', async () => {+ it.each([true, false])('archives checkout evidence before releasing a delivered or cleanup-pending claim (success=%s)', async success => {const agentId = 'claim-completion';const receipt = { version: 1, agentId, branch: 'claim/issue-1' };mockCosState.state.agents[agentId] = { id: agentId, status: 'running', startedAt: '2026-01-01T00:00:00Z',@@ -553,7 +553,9 @@ describe('completeAgent idempotence (#3384)', () => {expect(mockCosState.state.agents[agentId].status).toBe('running');return [receipt];});- const result = await completeAgent(agentId, { success: true });+ const result = await completeAgent(agentId, { success, cleanupPending: !success });+ expect(result.result.success).toBe(success);+ expect(result.metadata.repoStateVerification.status).toBe('pending');expect(result.metadata.completedCheckouts).toEqual([receipt]);const archived = JSON.parse(await readFile(join(mockCosState.agentsDir, result.completedAt.slice(0, 10), agentId, 'metadata.json'), 'utf8'));expect(archived.metadata.completedCheckouts).toEqual([receipt]);diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.jsindex 26adea2ccd..9bc5f3ac50 100644--- a/server/services/cosTaskStore.test.js+++ b/server/services/cosTaskStore.test.js@@ -2472,6 +2472,16 @@ describe('cosTaskStore — stale failure-artifact reaper (#2619)', () => {// CLI provider — so the reaper must leave it alone no matter how old it gets.// Flipping it to `completed` with `resolution: 'auto-expired'` would report// work nobody dropped as done, and federate that to every peer (#6193).+ it.each(['task-approval', 'task-recovery-exhausted'])('preserves the checkout and human decision for stale %s', async blockedCategory => {+ await addTask({ description: 'delivery decision', id: 'task-decision', priority: 'HIGH' }, 'user', { now: NOW });+ await updateTask('task-decision', { metadata: { existingBranch: 'fix/retained', resumedFromAgentId: 'agent-old', resumeWorktreePath: '/example/retained' } }, 'user', { now: NOW });+ await updateTask('task-decision', { status: 'blocked', metadata: { blockedCategory, blockedAt: daysAgo(90) } }, 'user', { now: NOW });+ expect((await sweepResolvedFailureTasks({ now: NOW })).reaped).toBe(0);+ const retained = await getTaskById('task-decision');+ expect(retained.status).toBe('blocked');+ expect(retained.metadata).toMatchObject({ existingBranch: 'fix/retained', resumedFromAgentId: 'agent-old', resumeWorktreePath: '/example/retained' });+ });+it('never auto-expires a provider-config block, however stale', async () => {await addTask({ description: 'api-only provider pinned', id: 'task-pc-old', priority: 'HIGH' }, 'user', { now: NOW });await updateTask('task-pc-old', { status: 'blocked', metadata: { blockedCategory: 'provider-config', blockedAt: daysAgo(90) } }, 'user', { now: NOW });