Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: finish cleanup handoffs through verified retirement policy
Adam Eivya7960ae4fbdf
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.jsindex 37973a8db0..d7e9e657c3 100644--- a/server/services/agentFinalization.js+++ b/server/services/agentFinalization.js@@ -1271,6 +1271,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, durationsuccess: effectiveSuccess,executionSuccess,cleanupPending: recoveryUpdate?.metadata?.deliveryRecoveryAgentId === agent.id,+ cleanupDeliveryVerified: savedCleanup,taskOutcome,validationPassed,exitCode,diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.jsindex acd41be9fd..292c87e7af 100644--- a/server/services/agentRepoStateVerification.js+++ b/server/services/agentRepoStateVerification.js@@ -82,7 +82,7 @@ const PENDING_OWNER_STATUSES = new Set(['pending', 'in_progress', 'blocked', 'ch* @param {object|null} app - the managed app record (carries `pendingMergePrs`)* @returns {Promise<boolean>}*/-async function branchHasPendingOwner(branchName, app) {+async function branchHasPendingOwner(branchName, app, cleanupOwner = null) {// `resolveTaskTargetBranch` is the canonical "which branch does this task own"// reader — matching `reviewLoopPRBranch` by hand here would miss a retry holding// the same branch through `existingBranch`.@@ -94,7 +94,9 @@ async function branchHasPendingOwner(branchName, app) {const tasks = await readAllTasksFlat().catch(() => null);if (!tasks) return null;const claimedByTask = tasks.some(t =>- PENDING_OWNER_STATUSES.has(t.status) && resolveTaskTargetBranch(t.metadata) === branchName+ !(cleanupOwner && t.id === cleanupOwner.taskId && t.metadata?.deliveryRecovery === 'cleanup'+ && t.metadata.deliveryRecoveryAgentId === cleanupOwner.agentId)+ && PENDING_OWNER_STATUSES.has(t.status) && resolveTaskTargetBranch(t.metadata) === branchName);if (claimedByTask) return true;@@ -253,6 +255,26 @@ export async function verifyAgentRepoState(input) {if (verificationRuns.has(input.agentId)) return verificationRuns.get(input.agentId);const cleanupPending = input.agentState?.result?.cleanupPending === true;const run = verifyRunBranches({ ...input, success: input.success || cleanupPending }).then(async result => {+ const outcomes = result.branches || [result];+ const unsupported = outcomes.find(item => [REPO_STATE_SKIPS.DISABLED, REPO_STATE_SKIPS.NOT_WORKTREE,+ REPO_STATE_SKIPS.PERSISTENT_WORKTREE, REPO_STATE_SKIPS.DISCARDED_WORKTREE,+ 'retirement-app-unavailable', 'retirement-policy-not-legacy-completed'].includes(item.skipReason)+ || (item.skipReason === 'cleanup-delivery-unverified' && item.observed?.prState === 'OPEN'));+ if (cleanupPending && unsupported) {+ const { resolveUndeliveredTaskUpdate } = await import('./agentErrorAnalysis.js');+ const outcome = { status: 'incomplete', reason: `Cleanup verification cannot run (${unsupported.skipReason}). ${input.task.metadata?.taskOutcome?.reason || ''}`.trim() };+ const planned = resolveUndeliveredTaskUpdate(input.task, outcome, input.agentId);+ const saved = await updateTask(input.task.id, planned, input.task.taskType || 'user', { expectedStatus: 'in_progress',+ expectedMetadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: input.agentId } });+ if (saved?.status !== planned.status || saved?.error || saved?.statusChanged) return { ...result, skipReason: 'cleanup-transfer-unverified' };+ if (planned.metadata.retryPendingCleanup) {+ const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');+ await releaseRetryHold({ agentId: input.agentId, task: input.task, success: false, agentMetadata: input.agentState.metadata });+ }+ const { recordRepoStateVerification } = await import('./cosAgentLifecycle.js');+ await recordRepoStateVerification(input.agentId, { status: 'completed', reasons: ['cleanup-transferred-to-recovery'] });+ return { ...result, skipReason: 'cleanup-transferred-to-recovery' };+ }if (cleanupPending && result.verified) {const updated = await updateTask(input.task.id, { status: 'completed', metadata: {deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined,@@ -423,7 +445,7 @@ async function verifyAgentBranchState({ agentId, task, agentState, success, prExconst worktreePath = isTruthyMeta(metadata.claimPicksOwnBranch) ? metadata.workspacePath : metadata.workspacePath || join(PATHS.worktrees, agentId);const { record, blockWith } = branchLedger(agentId, { branch: branchName, path: worktreePath });const enabled = repoStateVerificationEnabled(app);- const owned = enabled ? await branchHasPendingOwner(branchName, app).catch(() => null) : false;+ const owned = enabled ? await branchHasPendingOwner(branchName, app, agentState?.result?.cleanupPending === true ? { taskId: task.id, agentId } : null).catch(() => null) : false;if (owned === null) return blockWith('ownership-unreadable', { skipReason: REPO_STATE_SKIPS.GATE_UNREADABLE });if (owned) await record('pending', 'existing-completion-owner');diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.jsindex d50b205972..b7dec1e8a9 100644--- a/server/services/agentRepoStateVerification.test.js+++ b/server/services/agentRepoStateVerification.test.js@@ -21,6 +21,7 @@ vi.mock('./branchCompletionPolicy.js', async () => ({import { describe, it, expect, beforeEach, vi } from 'vitest';+vi.mock('./agentWorktreeCleanup.js', () => ({ releaseRetryHold: vi.fn(async () => ({})) }));vi.mock('./repoCleanupRecovery.js', () => ({ inspectOrQueueCompletedReview: vi.fn(async () => ({ disposition: 'held', reason: 'delivery-ci-or-review-pending' })) }));vi.mock('./repoCleanupObservation.js', async original => ({ ...await original(), readCheckoutPathPresence: vi.fn(async () => false), readLocalBranchPresence: vi.fn(async () => false) }));vi.mock('../lib/gitRemote.js', () => ({ getOriginInfo: vi.fn(async () => ({ host: 'github.com', fullName: 'example/repo', isGithub: true })) }));@@ -80,6 +81,7 @@ import { readClaimOwnershipAgents } from './cosClaimOwnership.js';import { recordCleanupOutcome, reserveRepoCleanupRecovery, recordRepoStateRecovery, recordRepoStateVerification } from './cosAgentLifecycle.js';import { verifyAgentRepoState, sweepRepoStateVerifications, REPO_STATE_REMEDIATIONS } from './agentRepoStateVerification.js';import { readBranchCompletionPolicies } from './branchCompletionPolicy.js';+import { releaseRetryHold } from './agentWorktreeCleanup.js';import { getTaskById, updateTask } from './cosTaskStore.js';import { recordBranchReconcileOutcome } from './taskSchedule.js';import { findMissingCheckoutPreservation, prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';@@ -147,6 +149,7 @@ beforeEach(() => {inspectOrQueueCompletedReview.mockResolvedValue({ disposition: 'held', reason: 'delivery-ci-or-review-pending' });readBranchCompletionPolicies.mockResolvedValue([]);getTaskById.mockResolvedValue(null);+ updateTask.mockResolvedValue({ status: 'completed' });recordBranchReconcileOutcome.mockResolvedValue({ retirementObligations: [{ ...RELEASE, createdAt: CREATED_AT }] });prepareInventoriedCheckoutRetirement.mockResolvedValue({ ok: true, review: review() });retireInventoriedCheckout.mockResolvedValue({ ok: true, archiveDir: '/private/archive', branchDeleted: true });@@ -171,6 +174,27 @@ beforeEach(() => {});describe('verifyAgentRepoState — clean runs', () => {+ it('transfers unsupported cleanup verification back to bounded recovery', async () => {+ updateTask.mockResolvedValueOnce({ status: 'in_progress' });+ const state = { ...agentState({ isPersistentWorktree: true, repoStateVerification: { status: 'pending' } }),+ result: { success: false, cleanupPending: true } };+ expect(await run({ success: false, agentState: state })).toMatchObject({ skipReason: 'cleanup-transferred-to-recovery' });+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'in_progress',+ metadata: expect.objectContaining({ failureCount: 1, retryPendingCleanup: 'agent-1' }) }), 'user', expect.any(Object));+ expect(releaseRetryHold).toHaveBeenCalledOnce();+ expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed', reasons: ['cleanup-transferred-to-recovery'] }));+ });++ it('ignores the exact cleanup task itself while preserving a different pending owner', async () => {+ const own = { ...task(), status: 'in_progress', metadata: { ...task().metadata,+ existingBranch: BRANCH, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-1' } };+ const state = { ...agentState(), result: { success: false, cleanupPending: true } };+ queuedTasks([own]);+ expect((await run({ success: false, agentState: state, task: own })).skipReason).not.toBe('follow-up-pending');+ queuedTasks([own, { ...own, id: 'other-task' }]);+ expect(await run({ success: false, agentState: state, task: own })).toMatchObject({ skipReason: 'follow-up-pending' });+ });+it('settles only the cleanup handoff owned by this run after archive and integration proof', async () => {const completed = { version: 1, agentId: 'agent-1', repoPath: '/repo', branch: BRANCH, path: WORKTREE, head: HEAD,startedAt: CREATED_AT, capturedAt: CREATED_AT };diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.jsindex b6a6c8bb1d..e92ee59849 100644--- a/server/services/agentWorktreeCleanup.js+++ b/server/services/agentWorktreeCleanup.js@@ -992,6 +992,15 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }});if (!persisted) return {};+ if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted'].includes(persisted.metadata?.blockedCategory)) {+ const metadata = agentMetadata === undefined ? (await getAgentRecord(agentId))?.metadata : agentMetadata;+ const patch = await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });+ const result = await updateTask(task.id, { metadata: patch }, task.taskType || persisted.taskType || 'user', {+ expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt },+ });+ return result?.error || result?.statusChanged ? {} : patch;+ }+if (isLegacyDeepAudit(task.metadata)) {if (persisted.status !== 'in_progress' || !isRetryHoldOwner(persisted.metadata, agentId)) return {};const metadata = agentMetadata === undefineddiff --git a/server/services/branchCompletionPolicy.js b/server/services/branchCompletionPolicy.jsindex 3b2f82a888..5db65efad6 100644--- a/server/services/branchCompletionPolicy.js+++ b/server/services/branchCompletionPolicy.js@@ -136,9 +136,9 @@ function reviewedIncarnation(input, policies) {if (!isAbsolute(input.checkoutPath || '') || samePath(input.checkoutPath, input.repoPath)|| !input.prUrl || input.verifiedPrHead !== input.tip|| !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(input.tip || '')) return null;- const owners = policies.filter(policy => policy.agent.status === 'completed' && policy.agent.result?.success === true+ const owners = policies.filter(policy => policy.agent.status === 'completed' && (policy.agent.result?.success === true || completedCleanupForRetirement(policy, input))&& policy.completion === PR_COMPLETIONS.REVIEW_THEN_MERGE- && samePath(policy.agent.metadata?.workspacePath, input.checkoutPath)+ && (samePath(policy.agent.metadata?.workspacePath, input.checkoutPath) || completedCleanupForRetirement(policy, input))&& retirementPr(policy) === input.prUrl && (!policy.tip || policy.tip === input.tip)&& (!reviewTarget(policy) || (reviewTarget(policy).branch === input.branch && reviewTarget(policy).url === input.prUrl))&& Number.isFinite(Date.parse(policy.agent.startedAt))@@ -168,6 +168,19 @@ function reviewedIncarnation(input, policies) {return { current, prior };}+/** Cleanup handoff is retirement evidence only under fresh merged-head and+ * server-captured checkout identity. It never grants merge/review authority.+ */+function completedCleanupForRetirement(policy, input) {+ const agent = policy.agent;+ const result = agent.result;+ return result?.cleanupPending === true && (result.executionSuccess === true || result.cleanupDeliveryVerified === true) && result.validationPassed !== false+ && result.taskOutcome?.recovery?.mode === 'cleanup' && input.verifiedPrHead === input.tip && !!input.tip+ && agent.metadata?.completedCheckouts?.some(row => row.version === 1 && row.agentId === agent.id+ && row.repoPath === input.repoPath && row.branch === input.branch && row.path === input.checkoutPath+ && row.head === input.tip && row.incarnation?.pointer && row.incarnation?.gitdir);+}+/** Operator review settles successful completed review obligations. PR-bound* obligations additionally need a fresh exact-head, exact-checkout incarnation.* This is review evidence, never unattended consent or a saved policy rewrite.@@ -180,7 +193,7 @@ export function reviewedRetirementPolicy(input) {const incarnation = reviewedIncarnation(input, decision.policies || []);const policies = incarnation?.current || decision.policies || [];if (policies.some(p => p.completion === PR_COMPLETIONS.LEAVE_OPEN)) return { reason: held };- if (policies.some(p => p.agent.status !== 'completed' || p.agent.result?.success !== true+ if (policies.some(p => p.agent.status !== 'completed' || (p.agent.result?.success !== true && !completedCleanupForRetirement(p, input))|| p.completion !== PR_COMPLETIONS.REVIEW_THEN_MERGE|| (retirementPr(p) && (!incarnation || retirementPr(p) !== input.prUrl))|| (reviewTarget(p) && (reviewTarget(p).branch !== input.branch || (p.prUrl && p.prUrl !== reviewTarget(p).url)))diff --git a/server/services/branchCompletionPolicy.test.js b/server/services/branchCompletionPolicy.test.jsindex ea3d5a08cb..cdd498fa97 100644--- a/server/services/branchCompletionPolicy.test.js+++ b/server/services/branchCompletionPolicy.test.js@@ -295,6 +295,24 @@ describe('reviewed reused-branch checkout incarnation', () => {return [current, prior];}+ it('permits retirement of a cleanup handoff only with its captured checkout and fresh merged-head proof', () => {+ const agents = owners();+ const current = agents[0];+ current.result = { success: false, executionSuccess: true, cleanupPending: true,+ taskOutcome: { status: 'incomplete', recovery: { mode: 'cleanup' } } };+ current.metadata.completedCheckouts = [{ version: 1, agentId: current.id, repoPath, branch, path: checkoutPath,+ head: runtime.tip, incarnation: { pointer: { ino: '1' }, gitdir: { ino: '2' } } }];+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeNull();+ expect(reviewedRetirementPolicy({ ...input(), agents, verifiedPrHead: null }).reason).toBeTruthy();+ current.metadata.workspacePath = repoPath; // A self-managed claim runs from the primary checkout.+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeNull();+ current.result.cleanupPending = false;+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeTruthy();+ current.result.cleanupPending = true;+ current.metadata.completedCheckouts[0].head = 'b'.repeat(40);+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeTruthy();+ });+it('separates explicit prior checkout history only under fresh current PR/head proof', () => {const agents = owners();const result = reviewedRetirementPolicy({ ...input(), agents });diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.jsindex 97a1b8632b..8ebdb96b36 100644--- a/server/services/cleanupAgentWorktree.test.js+++ b/server/services/cleanupAgentWorktree.test.js@@ -1724,6 +1724,16 @@ describe('releaseRetryHold', () => {// The direct-CLI / TUI shape: neither spawn path holds the agent record, so the// helper has to read it for the worktree fields — via the transcript-free// `getAgentRecord`, since a long TUI run's output.txt is megabytes.+ it.each(['task-approval', 'task-recovery-exhausted'])('records a first attempt resume pointer for %s without dispatching it', async blockedCategory => {+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now' } });+ git.getWorktreeBranches.mockResolvedValue(new Set([DEAD_BRANCH]));+ existsSyncMock.mockReturnValue(true);+ await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });+ expect(updateTask).toHaveBeenCalledWith('task-1', { metadata: expect.objectContaining({+ existingBranch: DEAD_BRANCH, resumedFromAgentId: 'agent-x', resumeWorktreePath: DEAD_TREE,+ }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now' } });+ });+it('records the pointer for a failed run whose task is still pending', async () => {await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false });diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.jsindex 0b6ba251c4..d49e85dcd1 100644--- a/server/services/cosTaskStore.js+++ b/server/services/cosTaskStore.js@@ -735,6 +735,9 @@ async function preparePauseRelease(taskId, updates) {* than each re-testing `updates.status !== 'in_progress'`.*/const TRANSITION_METADATA_CLEARS = Object.freeze([+ // An explicit retry or new cycle cancels the old cleanup coordinator.+ ['isRequeue', ['deliveryRecovery', 'deliveryRecoveryAgentId']],+ ['isRevive', ['deliveryRecovery', 'deliveryRecoveryAgentId']],// Leaving `blocked` by any door — a dedupe revive, an autopilot re-dispatch, an// orphan-cooldown expiry, or a human unblocking it from the task list. Clearing// only in `resumeAgent` left every one of those paths running a task that stilldiff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.jsindex 9bc5f3ac50..9f9ea47ea3 100644--- a/server/services/cosTaskStore.test.js+++ b/server/services/cosTaskStore.test.js@@ -1728,6 +1728,16 @@ describe('cosTaskStore.updateTask', () => {// The pointer is spent once a task is done or parked for a human: a PERPETUAL// task id gets re-queued for unrelated work later, and a stale existingBranch// would attach that fresh run to a long-merged branch.+ it('invalidates a completed run cleanup handoff before an explicit retry starts', async () => {+ await addTask({ description: 'cleanup handoff', id: 'task-cleanup-retry' }, 'user');+ await updateTask('task-cleanup-retry', { status: 'in_progress', metadata: {+ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-old',+ } }, 'user');+ const retried = await updateTask('task-cleanup-retry', { status: 'pending' }, 'user');+ expect(retried.metadata.deliveryRecovery).toBeUndefined();+ expect(retried.metadata.deliveryRecoveryAgentId).toBeUndefined();+ });+it('drops the resume pointer on a terminal status', async () => {await addTask({ description: 'res', id: 'task-res' }, 'user');await updateTask('task-res', {