Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: fence recovery decisions and reuse their task owner
Adam Eivy5c528eb856d9
diff --git a/server/services/agentCompletionCleanup.js b/server/services/agentCompletionCleanup.jsindex e164648f11..4fba114620 100644--- a/server/services/agentCompletionCleanup.js+++ b/server/services/agentCompletionCleanup.js@@ -584,6 +584,9 @@ async function reportWorktreeCleanupWarnings({ agentId, task, cleanupWarnings })const { getAgent: getAgentForResult } = await import('./cosAgentLifecycle.js');const currentAgent = await getAgentForResult(agentId).catch(() => null);await updateAgent(agentId, { result: { ...currentAgent?.result, warnings: cleanupWarnings } });+ // The original task already owns a bounded retry, cleanup proof, or human+ // decision. Another recovery task would race it or bypass its safety stop.+ if (currentAgent?.result?.deliveryRecoveryOwned === true) return;const appName = task?.metadata?.appName || task?.metadata?.app || 'PortOS';void spawnMergeRecoveryTask(cleanupWarnings, agentId, task, appName, currentAgent?.metadata?.sourceWorkspace).then(recoveryTask => {diff --git a/server/services/agentCompletionCleanup.test.js b/server/services/agentCompletionCleanup.test.jsindex 0e68bc6cc8..f9f7fb7754 100644--- a/server/services/agentCompletionCleanup.test.js+++ b/server/services/agentCompletionCleanup.test.js@@ -804,6 +804,15 @@ describe.each(['runner', 'spawner'])('%s completion side effects', (path) => {expect(handleCreativeDirectorCompletion.mock.invocationCallOrder[0]).toBeLessThan(cleanupAgentWorktree.mock.invocationCallOrder[0]);});+ it('retains cleanup warnings without spawning a second recovery when the original task owns recovery', async () => {+ const warnings = ['Worktree preserved pending its original recovery owner'];+ getAgent.mockResolvedValue({ id: 'a1', status: 'completed', result: { success: false, deliveryRecoveryOwned: true }, metadata: {} });+ cleanupAgentWorktree.mockResolvedValueOnce(warnings);+ await complete({}, false);+ expect(spawnMergeRecoveryTask).not.toHaveBeenCalled();+ expect(updateAgent).toHaveBeenCalledWith('a1', { result: { success: false, deliveryRecoveryOwned: true, warnings } });+ });+it('preserves the result and publishes cleanup warnings with a recovery task', async () => {const warnings = ['Merge failed: resolve example conflict'];cleanupAgentWorktree.mockResolvedValueOnce(warnings);diff --git a/server/services/agentErrorAnalysis.js b/server/services/agentErrorAnalysis.jsindex e3d0e69ad1..22120591dd 100644--- a/server/services/agentErrorAnalysis.js+++ b/server/services/agentErrorAnalysis.js@@ -1581,13 +1581,13 @@ export function resolveUndeliveredTaskUpdate(task, outcome, agentId, now = Date.deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: agentId, taskOutcome: outcome } };}if (!approval && decision.status !== 'blocked') return { status: decision.status,- metadata: { ...task.metadata, ...decision.metadataUpdates, blockedReason: undefined, blockedCategory: undefined, blockedAt: undefined, deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined, taskOutcome: outcome } };+ metadata: { ...task.metadata, ...decision.metadataUpdates, deliveryDecisionAgentId: undefined, blockedReason: undefined, blockedCategory: undefined, blockedAt: undefined, deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined, taskOutcome: outcome } };const options = approval ? outcome.recovery.options : ['Fix the reported cause and retry the retained work with a fresh retry budget.','Keep the task stopped and preserve its work; do not spend more agent runs.',];return { status: 'blocked', metadata: { ...task.metadata, ...decision.metadataUpdates,- taskOutcome: outcome, blockedCategory: approval ? 'task-approval' : 'task-recovery-exhausted',+ taskOutcome: outcome, deliveryDecisionAgentId: agentId, blockedCategory: approval ? 'task-approval' : 'task-recovery-exhausted',blockedAt: new Date(now).toISOString(),blockedReason: `${outcome.reason}\n${approval ? 'Approval required before proceeding.' : 'Automatic recovery stopped at the retry/spawn safety limit.'}\nOptions:\n${options.map((option, index) => `${index + 1}. ${option}`).join('\n')}`,} };diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.jsindex 33a314e258..a00a0caa66 100644--- a/server/services/agentFinalization.js+++ b/server/services/agentFinalization.js@@ -1270,6 +1270,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, durationawait completeAgent(agent.id, {success: effectiveSuccess,executionSuccess,+ ...(recoveryUpdate ? { deliveryRecoveryOwned: true } : {}),...(recoveryUpdate?.metadata?.deliveryRecoveryAgentId === agent.id ? { cleanupPending: true } : {}),...(savedCleanup ? { cleanupDeliveryVerified: true } : {}),taskOutcome,@@ -1294,7 +1295,8 @@ export async function retireDeadAgent({ agent, task, success, exitCode, durationawait removeCompletionSentinel({ agentId: agent.id, agentState: agent }).catch(err => emitLog('warn', `Completion sentinel removal failed for ${agent.id}: ${err.message}`, { agentId: agent.id }));- if (recoveryUpdate?.metadata?.retryPendingCleanup) {+ if (recoveryUpdate?.metadata?.retryPendingCleanup+ || ['task-approval', 'task-recovery-exhausted'].includes(recoveryUpdate?.metadata?.blockedCategory)) {const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');await releaseRetryHold({ agentId: agent.id, task, success: false, agentMetadata: agent.metadata }).catch(err => emitLog('warn', `Recovered delivery retry deferred for ${agent.id}: ${err.message}`, { agentId: agent.id }));@@ -1963,7 +1965,11 @@ export async function finalizeAgent({// failure.await completeAgent(agentId, { ...completionRecord({task, verdict: deliveryVerdict, hookResult, fidelity, executionInterruption, exitCode, duration, outputBuffer, validationPassed,- }), executionSuccess, taskOutcome, ...(taskUpdate.metadata?.deliveryRecoveryAgentId === agentId ? { cleanupPending: true } : {}) });+ }), executionSuccess, taskOutcome,+ ...(['task-approval', 'task-recovery-exhausted'].includes(taskUpdate.metadata?.blockedCategory)+ || taskUpdate.metadata?.retryPendingCleanup === agentId || taskUpdate.metadata?.deliveryRecoveryAgentId === agentId+ ? { deliveryRecoveryOwned: true } : {}),+ ...(taskUpdate.metadata?.deliveryRecoveryAgentId === agentId ? { cleanupPending: true } : {}) });if (runId) {// Every failed verdict overrides exit-code/commit rescue in run tracking,diff --git a/server/services/agentFinalization.retireDeadAgent.test.js b/server/services/agentFinalization.retireDeadAgent.test.jsindex fccaa38b9c..65742ee202 100644--- a/server/services/agentFinalization.retireDeadAgent.test.js+++ b/server/services/agentFinalization.retireDeadAgent.test.js@@ -129,6 +129,16 @@ describe('retireDeadAgent (#8440)', () => {} finally { await rm(workspacePath, { recursive: true, force: true }); }});+ it('preserves the first attempt pointer when a recovered outcome needs approval', async () => {+ const taskOutcome = { status: 'blocked', reason: 'Authorized budget exhausted.', recovery: {+ mode: 'approval', options: ['Increase the budget and retry.', 'Preserve the stopped work.'],+ } };+ await retireDeadAgent({ agent: { ...AGENT, result: { taskOutcome } }, task: TASK, success: true, exitCode: 0, duration: 100 });+ expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.objectContaining({ status: 'blocked',+ metadata: expect.objectContaining({ deliveryDecisionAgentId: AGENT.id }) }), 'user');+ expect(releaseRetryHold).toHaveBeenCalledWith({ agentId: AGENT.id, task: TASK, success: false, agentMetadata: AGENT.metadata });+ });+it('accepts a recovered no-op only after its output hook verifies delivery', async () => {const taskOutcome = { status: 'no-action-needed', reason: 'Invented verification found no remaining work.' };const hook = vi.fn(async () => ({ accepted: true }));diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.jsindex bf19e6ea34..84d3bf0f27 100644--- a/server/services/agentRepoStateVerification.js+++ b/server/services/agentRepoStateVerification.js@@ -275,7 +275,8 @@ export async function verifyAgentRepoState(input) {const unsupported = outcomes.find(item => [REPO_STATE_SKIPS.DISABLED, REPO_STATE_SKIPS.NOT_WORKTREE,REPO_STATE_SKIPS.PERSISTENT_WORKTREE, REPO_STATE_SKIPS.DISCARDED_WORKTREE,'retirement-app-unavailable', 'retirement-policy-not-legacy-completed'].includes(item.skipReason)- || (item.skipReason === 'cleanup-delivery-unverified' && item.observed?.prState === 'OPEN'));+ || (item.skipReason === 'cleanup-delivery-unverified' && (['OPEN', 'CLOSED'].includes(item.observed?.prState)+ || (item.observed?.prState === null && item.observed?.prUrl === null && item.observed?.unreadable?.length === 0))));if (cleanupPending && unsupported) {const { resolveUndeliveredTaskUpdate } = await import('./agentErrorAnalysis.js');const outcome = { status: 'incomplete', reason: `Cleanup verification cannot run (${unsupported.skipReason}). ${input.task.metadata?.taskOutcome?.reason || ''}`.trim() };diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.jsindex 9b46e60a5a..f3cbe2baa9 100644--- a/server/services/agentRepoStateVerification.test.js+++ b/server/services/agentRepoStateVerification.test.js@@ -195,6 +195,17 @@ describe('verifyAgentRepoState — clean runs', () => {expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed', reasons: ['cleanup-transferred-to-recovery'] }));});+ it.each(['OPEN', 'CLOSED'])('transfers a definitive %s undelivered PR out of the cleanup hold', async prState => {+ updateTask.mockResolvedValueOnce({ status: 'in_progress' });+ localBranch(true);+ existsSync.mockReturnValue(true);+ findPullRequestForBranch.mockResolvedValue({ status: 'found', detail: prState, url: 'https://github.com/example/repo/pull/7' });+ const state = { ...agentState(), result: { success: false, cleanupPending: true } };+ expect(await run({ success: false, agentState: state })).toMatchObject({ skipReason: 'cleanup-transferred-to-recovery' });+ expect(releaseRetryHold).toHaveBeenCalledOnce();+ expect(addTask).not.toHaveBeenCalled();+ });+it('ignores the exact cleanup task itself while preserving a different pending owner', async () => {const own = { ...task(), status: 'in_progress', metadata: { ...task().metadata,existingBranch: BRANCH, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-1' } };diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.jsindex e92ee59849..b3952694a2 100644--- a/server/services/agentWorktreeCleanup.js+++ b/server/services/agentWorktreeCleanup.js@@ -993,10 +993,11 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }if (!persisted) return {};if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted'].includes(persisted.metadata?.blockedCategory)) {+ if (persisted.metadata.deliveryDecisionAgentId !== agentId) return {};const metadata = agentMetadata === undefined ? (await getAgentRecord(agentId))?.metadata : agentMetadata;const patch = await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });const result = await updateTask(task.id, { metadata: patch }, task.taskType || persisted.taskType || 'user', {- expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt },+ expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt, deliveryDecisionAgentId: agentId },});return result?.error || result?.statusChanged ? {} : patch;}diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.jsindex 8ebdb96b36..0f2c6e9047 100644--- a/server/services/cleanupAgentWorktree.test.js+++ b/server/services/cleanupAgentWorktree.test.js@@ -1724,14 +1724,22 @@ describe('releaseRetryHold', () => {// The direct-CLI / TUI shape: neither spawn path holds the agent record, so the// helper has to read it for the worktree fields — via the transcript-free// `getAgentRecord`, since a long TUI run's output.txt is megabytes.+ it('never lets an older completion replace the current decision owner resume pointer', async () => {+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: {+ blockedCategory: 'task-approval', blockedAt: 'now', deliveryDecisionAgentId: 'agent-new',+ } });+ await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });+ expect(updateTask).not.toHaveBeenCalled();+ });+it.each(['task-approval', 'task-recovery-exhausted'])('records a first attempt resume pointer for %s without dispatching it', async blockedCategory => {- getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now' } });+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now', deliveryDecisionAgentId: 'agent-x' } });git.getWorktreeBranches.mockResolvedValue(new Set([DEAD_BRANCH]));existsSyncMock.mockReturnValue(true);await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });expect(updateTask).toHaveBeenCalledWith('task-1', { metadata: expect.objectContaining({existingBranch: DEAD_BRANCH, resumedFromAgentId: 'agent-x', resumeWorktreePath: DEAD_TREE,- }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now' } });+ }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now', deliveryDecisionAgentId: 'agent-x' } });});it('records the pointer for a failed run whose task is still pending', async () => {diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.jsindex d49e85dcd1..3a7ce92c35 100644--- a/server/services/cosTaskStore.js+++ b/server/services/cosTaskStore.js@@ -745,7 +745,7 @@ const TRANSITION_METADATA_CLEARS = Object.freeze([// still-paused agent record then read its own pause as spent and spawned a SECOND// agent on a fresh task. The failure counters go too, or the revived task// re-blocks on the budget it blocked with.- ['leavesBlocked', ['blocker', 'blockedReason', 'blockedCategory', 'blockedAt', 'failureCount', 'lastErrorCategory', 'lastFailureAt', ...PAUSE_METADATA_KEYS]],+ ['leavesBlocked', ['blocker', 'blockedReason', 'blockedCategory', 'blockedAt', 'deliveryDecisionAgentId', 'failureCount', 'lastErrorCategory', 'lastFailureAt', ...PAUSE_METADATA_KEYS]],// A retry hold (#3373) only means anything while the task is `in_progress` waiting// on a cleanup to resolve its resume pointer. Any other status the task reaches —// terminal, or a requeue some other path performed — retires it, so drop the marker