Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: finish cleanup handoffs through verified retirement policy

Adam Eivya7960ae4fbdf
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 37973a8db0..d7e9e657c3 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1271,6 +1271,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
success: effectiveSuccess,
executionSuccess,
cleanupPending: recoveryUpdate?.metadata?.deliveryRecoveryAgentId === agent.id,
+ cleanupDeliveryVerified: savedCleanup,
taskOutcome,
validationPassed,
exitCode,
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.js
index acd41be9fd..292c87e7af 100644
--- a/server/services/agentRepoStateVerification.js
+++ b/server/services/agentRepoStateVerification.js
@@ -82,7 +82,7 @@ const PENDING_OWNER_STATUSES = new Set(['pending', 'in_progress', 'blocked', 'ch
* @param {object|null} app - the managed app record (carries `pendingMergePrs`)
* @returns {Promise<boolean>}
*/
-async function branchHasPendingOwner(branchName, app) {
+async function branchHasPendingOwner(branchName, app, cleanupOwner = null) {
// `resolveTaskTargetBranch` is the canonical "which branch does this task own"
// reader — matching `reviewLoopPRBranch` by hand here would miss a retry holding
// the same branch through `existingBranch`.
@@ -94,7 +94,9 @@ async function branchHasPendingOwner(branchName, app) {
const tasks = await readAllTasksFlat().catch(() => null);
if (!tasks) return null;
const claimedByTask = tasks.some(t =>
- PENDING_OWNER_STATUSES.has(t.status) && resolveTaskTargetBranch(t.metadata) === branchName
+ !(cleanupOwner && t.id === cleanupOwner.taskId && t.metadata?.deliveryRecovery === 'cleanup'
+ && t.metadata.deliveryRecoveryAgentId === cleanupOwner.agentId)
+ && PENDING_OWNER_STATUSES.has(t.status) && resolveTaskTargetBranch(t.metadata) === branchName
);
if (claimedByTask) return true;
@@ -253,6 +255,26 @@ export async function verifyAgentRepoState(input) {
if (verificationRuns.has(input.agentId)) return verificationRuns.get(input.agentId);
const cleanupPending = input.agentState?.result?.cleanupPending === true;
const run = verifyRunBranches({ ...input, success: input.success || cleanupPending }).then(async result => {
+ const outcomes = result.branches || [result];
+ const unsupported = outcomes.find(item => [REPO_STATE_SKIPS.DISABLED, REPO_STATE_SKIPS.NOT_WORKTREE,
+ REPO_STATE_SKIPS.PERSISTENT_WORKTREE, REPO_STATE_SKIPS.DISCARDED_WORKTREE,
+ 'retirement-app-unavailable', 'retirement-policy-not-legacy-completed'].includes(item.skipReason)
+ || (item.skipReason === 'cleanup-delivery-unverified' && item.observed?.prState === 'OPEN'));
+ if (cleanupPending && unsupported) {
+ const { resolveUndeliveredTaskUpdate } = await import('./agentErrorAnalysis.js');
+ const outcome = { status: 'incomplete', reason: `Cleanup verification cannot run (${unsupported.skipReason}). ${input.task.metadata?.taskOutcome?.reason || ''}`.trim() };
+ const planned = resolveUndeliveredTaskUpdate(input.task, outcome, input.agentId);
+ const saved = await updateTask(input.task.id, planned, input.task.taskType || 'user', { expectedStatus: 'in_progress',
+ expectedMetadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: input.agentId } });
+ if (saved?.status !== planned.status || saved?.error || saved?.statusChanged) return { ...result, skipReason: 'cleanup-transfer-unverified' };
+ if (planned.metadata.retryPendingCleanup) {
+ const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');
+ await releaseRetryHold({ agentId: input.agentId, task: input.task, success: false, agentMetadata: input.agentState.metadata });
+ }
+ const { recordRepoStateVerification } = await import('./cosAgentLifecycle.js');
+ await recordRepoStateVerification(input.agentId, { status: 'completed', reasons: ['cleanup-transferred-to-recovery'] });
+ return { ...result, skipReason: 'cleanup-transferred-to-recovery' };
+ }
if (cleanupPending && result.verified) {
const updated = await updateTask(input.task.id, { status: 'completed', metadata: {
deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined,
@@ -423,7 +445,7 @@ async function verifyAgentBranchState({ agentId, task, agentState, success, prEx
const worktreePath = isTruthyMeta(metadata.claimPicksOwnBranch) ? metadata.workspacePath : metadata.workspacePath || join(PATHS.worktrees, agentId);
const { record, blockWith } = branchLedger(agentId, { branch: branchName, path: worktreePath });
const enabled = repoStateVerificationEnabled(app);
- const owned = enabled ? await branchHasPendingOwner(branchName, app).catch(() => null) : false;
+ const owned = enabled ? await branchHasPendingOwner(branchName, app, agentState?.result?.cleanupPending === true ? { taskId: task.id, agentId } : null).catch(() => null) : false;
if (owned === null) return blockWith('ownership-unreadable', { skipReason: REPO_STATE_SKIPS.GATE_UNREADABLE });
if (owned) await record('pending', 'existing-completion-owner');
diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.js
index d50b205972..b7dec1e8a9 100644
--- a/server/services/agentRepoStateVerification.test.js
+++ b/server/services/agentRepoStateVerification.test.js
@@ -21,6 +21,7 @@ vi.mock('./branchCompletionPolicy.js', async () => ({
import { describe, it, expect, beforeEach, vi } from 'vitest';
+vi.mock('./agentWorktreeCleanup.js', () => ({ releaseRetryHold: vi.fn(async () => ({})) }));
vi.mock('./repoCleanupRecovery.js', () => ({ inspectOrQueueCompletedReview: vi.fn(async () => ({ disposition: 'held', reason: 'delivery-ci-or-review-pending' })) }));
vi.mock('./repoCleanupObservation.js', async original => ({ ...await original(), readCheckoutPathPresence: vi.fn(async () => false), readLocalBranchPresence: vi.fn(async () => false) }));
vi.mock('../lib/gitRemote.js', () => ({ getOriginInfo: vi.fn(async () => ({ host: 'github.com', fullName: 'example/repo', isGithub: true })) }));
@@ -80,6 +81,7 @@ import { readClaimOwnershipAgents } from './cosClaimOwnership.js';
import { recordCleanupOutcome, reserveRepoCleanupRecovery, recordRepoStateRecovery, recordRepoStateVerification } from './cosAgentLifecycle.js';
import { verifyAgentRepoState, sweepRepoStateVerifications, REPO_STATE_REMEDIATIONS } from './agentRepoStateVerification.js';
import { readBranchCompletionPolicies } from './branchCompletionPolicy.js';
+import { releaseRetryHold } from './agentWorktreeCleanup.js';
import { getTaskById, updateTask } from './cosTaskStore.js';
import { recordBranchReconcileOutcome } from './taskSchedule.js';
import { findMissingCheckoutPreservation, prepareInventoriedCheckoutRetirement, retireInventoriedCheckout } from './heldCheckoutRecovery.js';
@@ -147,6 +149,7 @@ beforeEach(() => {
inspectOrQueueCompletedReview.mockResolvedValue({ disposition: 'held', reason: 'delivery-ci-or-review-pending' });
readBranchCompletionPolicies.mockResolvedValue([]);
getTaskById.mockResolvedValue(null);
+ updateTask.mockResolvedValue({ status: 'completed' });
recordBranchReconcileOutcome.mockResolvedValue({ retirementObligations: [{ ...RELEASE, createdAt: CREATED_AT }] });
prepareInventoriedCheckoutRetirement.mockResolvedValue({ ok: true, review: review() });
retireInventoriedCheckout.mockResolvedValue({ ok: true, archiveDir: '/private/archive', branchDeleted: true });
@@ -171,6 +174,27 @@ beforeEach(() => {
});
describe('verifyAgentRepoState — clean runs', () => {
+ it('transfers unsupported cleanup verification back to bounded recovery', async () => {
+ updateTask.mockResolvedValueOnce({ status: 'in_progress' });
+ const state = { ...agentState({ isPersistentWorktree: true, repoStateVerification: { status: 'pending' } }),
+ result: { success: false, cleanupPending: true } };
+ expect(await run({ success: false, agentState: state })).toMatchObject({ skipReason: 'cleanup-transferred-to-recovery' });
+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'in_progress',
+ metadata: expect.objectContaining({ failureCount: 1, retryPendingCleanup: 'agent-1' }) }), 'user', expect.any(Object));
+ expect(releaseRetryHold).toHaveBeenCalledOnce();
+ expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed', reasons: ['cleanup-transferred-to-recovery'] }));
+ });
+
+ it('ignores the exact cleanup task itself while preserving a different pending owner', async () => {
+ const own = { ...task(), status: 'in_progress', metadata: { ...task().metadata,
+ existingBranch: BRANCH, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-1' } };
+ const state = { ...agentState(), result: { success: false, cleanupPending: true } };
+ queuedTasks([own]);
+ expect((await run({ success: false, agentState: state, task: own })).skipReason).not.toBe('follow-up-pending');
+ queuedTasks([own, { ...own, id: 'other-task' }]);
+ expect(await run({ success: false, agentState: state, task: own })).toMatchObject({ skipReason: 'follow-up-pending' });
+ });
+
it('settles only the cleanup handoff owned by this run after archive and integration proof', async () => {
const completed = { version: 1, agentId: 'agent-1', repoPath: '/repo', branch: BRANCH, path: WORKTREE, head: HEAD,
startedAt: CREATED_AT, capturedAt: CREATED_AT };
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index b6a6c8bb1d..e92ee59849 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -992,6 +992,15 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }
});
if (!persisted) return {};
+ if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted'].includes(persisted.metadata?.blockedCategory)) {
+ const metadata = agentMetadata === undefined ? (await getAgentRecord(agentId))?.metadata : agentMetadata;
+ const patch = await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });
+ const result = await updateTask(task.id, { metadata: patch }, task.taskType || persisted.taskType || 'user', {
+ expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt },
+ });
+ return result?.error || result?.statusChanged ? {} : patch;
+ }
+
if (isLegacyDeepAudit(task.metadata)) {
if (persisted.status !== 'in_progress' || !isRetryHoldOwner(persisted.metadata, agentId)) return {};
const metadata = agentMetadata === undefined
diff --git a/server/services/branchCompletionPolicy.js b/server/services/branchCompletionPolicy.js
index 3b2f82a888..5db65efad6 100644
--- a/server/services/branchCompletionPolicy.js
+++ b/server/services/branchCompletionPolicy.js
@@ -136,9 +136,9 @@ function reviewedIncarnation(input, policies) {
if (!isAbsolute(input.checkoutPath || '') || samePath(input.checkoutPath, input.repoPath)
|| !input.prUrl || input.verifiedPrHead !== input.tip
|| !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(input.tip || '')) return null;
- const owners = policies.filter(policy => policy.agent.status === 'completed' && policy.agent.result?.success === true
+ const owners = policies.filter(policy => policy.agent.status === 'completed' && (policy.agent.result?.success === true || completedCleanupForRetirement(policy, input))
&& policy.completion === PR_COMPLETIONS.REVIEW_THEN_MERGE
- && samePath(policy.agent.metadata?.workspacePath, input.checkoutPath)
+ && (samePath(policy.agent.metadata?.workspacePath, input.checkoutPath) || completedCleanupForRetirement(policy, input))
&& retirementPr(policy) === input.prUrl && (!policy.tip || policy.tip === input.tip)
&& (!reviewTarget(policy) || (reviewTarget(policy).branch === input.branch && reviewTarget(policy).url === input.prUrl))
&& Number.isFinite(Date.parse(policy.agent.startedAt))
@@ -168,6 +168,19 @@ function reviewedIncarnation(input, policies) {
return { current, prior };
}
+/** Cleanup handoff is retirement evidence only under fresh merged-head and
+ * server-captured checkout identity. It never grants merge/review authority.
+ */
+function completedCleanupForRetirement(policy, input) {
+ const agent = policy.agent;
+ const result = agent.result;
+ return result?.cleanupPending === true && (result.executionSuccess === true || result.cleanupDeliveryVerified === true) && result.validationPassed !== false
+ && result.taskOutcome?.recovery?.mode === 'cleanup' && input.verifiedPrHead === input.tip && !!input.tip
+ && agent.metadata?.completedCheckouts?.some(row => row.version === 1 && row.agentId === agent.id
+ && row.repoPath === input.repoPath && row.branch === input.branch && row.path === input.checkoutPath
+ && row.head === input.tip && row.incarnation?.pointer && row.incarnation?.gitdir);
+}
+
/** Operator review settles successful completed review obligations. PR-bound
* obligations additionally need a fresh exact-head, exact-checkout incarnation.
* This is review evidence, never unattended consent or a saved policy rewrite.
@@ -180,7 +193,7 @@ export function reviewedRetirementPolicy(input) {
const incarnation = reviewedIncarnation(input, decision.policies || []);
const policies = incarnation?.current || decision.policies || [];
if (policies.some(p => p.completion === PR_COMPLETIONS.LEAVE_OPEN)) return { reason: held };
- if (policies.some(p => p.agent.status !== 'completed' || p.agent.result?.success !== true
+ if (policies.some(p => p.agent.status !== 'completed' || (p.agent.result?.success !== true && !completedCleanupForRetirement(p, input))
|| p.completion !== PR_COMPLETIONS.REVIEW_THEN_MERGE
|| (retirementPr(p) && (!incarnation || retirementPr(p) !== input.prUrl))
|| (reviewTarget(p) && (reviewTarget(p).branch !== input.branch || (p.prUrl && p.prUrl !== reviewTarget(p).url)))
diff --git a/server/services/branchCompletionPolicy.test.js b/server/services/branchCompletionPolicy.test.js
index ea3d5a08cb..cdd498fa97 100644
--- a/server/services/branchCompletionPolicy.test.js
+++ b/server/services/branchCompletionPolicy.test.js
@@ -295,6 +295,24 @@ describe('reviewed reused-branch checkout incarnation', () => {
return [current, prior];
}
+ it('permits retirement of a cleanup handoff only with its captured checkout and fresh merged-head proof', () => {
+ const agents = owners();
+ const current = agents[0];
+ current.result = { success: false, executionSuccess: true, cleanupPending: true,
+ taskOutcome: { status: 'incomplete', recovery: { mode: 'cleanup' } } };
+ current.metadata.completedCheckouts = [{ version: 1, agentId: current.id, repoPath, branch, path: checkoutPath,
+ head: runtime.tip, incarnation: { pointer: { ino: '1' }, gitdir: { ino: '2' } } }];
+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeNull();
+ expect(reviewedRetirementPolicy({ ...input(), agents, verifiedPrHead: null }).reason).toBeTruthy();
+ current.metadata.workspacePath = repoPath; // A self-managed claim runs from the primary checkout.
+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeNull();
+ current.result.cleanupPending = false;
+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeTruthy();
+ current.result.cleanupPending = true;
+ current.metadata.completedCheckouts[0].head = 'b'.repeat(40);
+ expect(reviewedRetirementPolicy({ ...input(), agents }).reason).toBeTruthy();
+ });
+
it('separates explicit prior checkout history only under fresh current PR/head proof', () => {
const agents = owners();
const result = reviewedRetirementPolicy({ ...input(), agents });
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 97a1b8632b..8ebdb96b36 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -1724,6 +1724,16 @@ describe('releaseRetryHold', () => {
// The direct-CLI / TUI shape: neither spawn path holds the agent record, so the
// helper has to read it for the worktree fields — via the transcript-free
// `getAgentRecord`, since a long TUI run's output.txt is megabytes.
+ it.each(['task-approval', 'task-recovery-exhausted'])('records a first attempt resume pointer for %s without dispatching it', async blockedCategory => {
+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now' } });
+ git.getWorktreeBranches.mockResolvedValue(new Set([DEAD_BRANCH]));
+ existsSyncMock.mockReturnValue(true);
+ await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });
+ expect(updateTask).toHaveBeenCalledWith('task-1', { metadata: expect.objectContaining({
+ existingBranch: DEAD_BRANCH, resumedFromAgentId: 'agent-x', resumeWorktreePath: DEAD_TREE,
+ }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now' } });
+ });
+
it('records the pointer for a failed run whose task is still pending', async () => {
await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false });
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index 0b6ba251c4..d49e85dcd1 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -735,6 +735,9 @@ async function preparePauseRelease(taskId, updates) {
* than each re-testing `updates.status !== 'in_progress'`.
*/
const TRANSITION_METADATA_CLEARS = Object.freeze([
+ // An explicit retry or new cycle cancels the old cleanup coordinator.
+ ['isRequeue', ['deliveryRecovery', 'deliveryRecoveryAgentId']],
+ ['isRevive', ['deliveryRecovery', 'deliveryRecoveryAgentId']],
// Leaving `blocked` by any door — a dedupe revive, an autopilot re-dispatch, an
// orphan-cooldown expiry, or a human unblocking it from the task list. Clearing
// only in `resumeAgent` left every one of those paths running a task that still
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 9bc5f3ac50..9f9ea47ea3 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -1728,6 +1728,16 @@ describe('cosTaskStore.updateTask', () => {
// The pointer is spent once a task is done or parked for a human: a PERPETUAL
// task id gets re-queued for unrelated work later, and a stale existingBranch
// would attach that fresh run to a long-merged branch.
+ it('invalidates a completed run cleanup handoff before an explicit retry starts', async () => {
+ await addTask({ description: 'cleanup handoff', id: 'task-cleanup-retry' }, 'user');
+ await updateTask('task-cleanup-retry', { status: 'in_progress', metadata: {
+ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-old',
+ } }, 'user');
+ const retried = await updateTask('task-cleanup-retry', { status: 'pending' }, 'user');
+ expect(retried.metadata.deliveryRecovery).toBeUndefined();
+ expect(retried.metadata.deliveryRecoveryAgentId).toBeUndefined();
+ });
+
it('drops the resume pointer on a terminal status', async () => {
await addTask({ description: 'res', id: 'task-res' }, 'user');
await updateTask('task-res', {