Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: fence recovery decisions and reuse their task owner

Adam Eivy5c528eb856d9
diff --git a/server/services/agentCompletionCleanup.js b/server/services/agentCompletionCleanup.js
index e164648f11..4fba114620 100644
--- a/server/services/agentCompletionCleanup.js
+++ b/server/services/agentCompletionCleanup.js
@@ -584,6 +584,9 @@ async function reportWorktreeCleanupWarnings({ agentId, task, cleanupWarnings })
const { getAgent: getAgentForResult } = await import('./cosAgentLifecycle.js');
const currentAgent = await getAgentForResult(agentId).catch(() => null);
await updateAgent(agentId, { result: { ...currentAgent?.result, warnings: cleanupWarnings } });
+ // The original task already owns a bounded retry, cleanup proof, or human
+ // decision. Another recovery task would race it or bypass its safety stop.
+ if (currentAgent?.result?.deliveryRecoveryOwned === true) return;
const appName = task?.metadata?.appName || task?.metadata?.app || 'PortOS';
void spawnMergeRecoveryTask(cleanupWarnings, agentId, task, appName, currentAgent?.metadata?.sourceWorkspace).then(recoveryTask => {
diff --git a/server/services/agentCompletionCleanup.test.js b/server/services/agentCompletionCleanup.test.js
index 0e68bc6cc8..f9f7fb7754 100644
--- a/server/services/agentCompletionCleanup.test.js
+++ b/server/services/agentCompletionCleanup.test.js
@@ -804,6 +804,15 @@ describe.each(['runner', 'spawner'])('%s completion side effects', (path) => {
expect(handleCreativeDirectorCompletion.mock.invocationCallOrder[0]).toBeLessThan(cleanupAgentWorktree.mock.invocationCallOrder[0]);
});
+ it('retains cleanup warnings without spawning a second recovery when the original task owns recovery', async () => {
+ const warnings = ['Worktree preserved pending its original recovery owner'];
+ getAgent.mockResolvedValue({ id: 'a1', status: 'completed', result: { success: false, deliveryRecoveryOwned: true }, metadata: {} });
+ cleanupAgentWorktree.mockResolvedValueOnce(warnings);
+ await complete({}, false);
+ expect(spawnMergeRecoveryTask).not.toHaveBeenCalled();
+ expect(updateAgent).toHaveBeenCalledWith('a1', { result: { success: false, deliveryRecoveryOwned: true, warnings } });
+ });
+
it('preserves the result and publishes cleanup warnings with a recovery task', async () => {
const warnings = ['Merge failed: resolve example conflict'];
cleanupAgentWorktree.mockResolvedValueOnce(warnings);
diff --git a/server/services/agentErrorAnalysis.js b/server/services/agentErrorAnalysis.js
index e3d0e69ad1..22120591dd 100644
--- a/server/services/agentErrorAnalysis.js
+++ b/server/services/agentErrorAnalysis.js
@@ -1581,13 +1581,13 @@ export function resolveUndeliveredTaskUpdate(task, outcome, agentId, now = Date.
deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: agentId, taskOutcome: outcome } };
}
if (!approval && decision.status !== 'blocked') return { status: decision.status,
- metadata: { ...task.metadata, ...decision.metadataUpdates, blockedReason: undefined, blockedCategory: undefined, blockedAt: undefined, deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined, taskOutcome: outcome } };
+ metadata: { ...task.metadata, ...decision.metadataUpdates, deliveryDecisionAgentId: undefined, blockedReason: undefined, blockedCategory: undefined, blockedAt: undefined, deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined, taskOutcome: outcome } };
const options = approval ? outcome.recovery.options : [
'Fix the reported cause and retry the retained work with a fresh retry budget.',
'Keep the task stopped and preserve its work; do not spend more agent runs.',
];
return { status: 'blocked', metadata: { ...task.metadata, ...decision.metadataUpdates,
- taskOutcome: outcome, blockedCategory: approval ? 'task-approval' : 'task-recovery-exhausted',
+ taskOutcome: outcome, deliveryDecisionAgentId: agentId, blockedCategory: approval ? 'task-approval' : 'task-recovery-exhausted',
blockedAt: new Date(now).toISOString(),
blockedReason: `${outcome.reason}\n${approval ? 'Approval required before proceeding.' : 'Automatic recovery stopped at the retry/spawn safety limit.'}\nOptions:\n${options.map((option, index) => `${index + 1}. ${option}`).join('\n')}`,
} };
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 33a314e258..a00a0caa66 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1270,6 +1270,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
await completeAgent(agent.id, {
success: effectiveSuccess,
executionSuccess,
+ ...(recoveryUpdate ? { deliveryRecoveryOwned: true } : {}),
...(recoveryUpdate?.metadata?.deliveryRecoveryAgentId === agent.id ? { cleanupPending: true } : {}),
...(savedCleanup ? { cleanupDeliveryVerified: true } : {}),
taskOutcome,
@@ -1294,7 +1295,8 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
await removeCompletionSentinel({ agentId: agent.id, agentState: agent })
.catch(err => emitLog('warn', `Completion sentinel removal failed for ${agent.id}: ${err.message}`, { agentId: agent.id }));
- if (recoveryUpdate?.metadata?.retryPendingCleanup) {
+ if (recoveryUpdate?.metadata?.retryPendingCleanup
+ || ['task-approval', 'task-recovery-exhausted'].includes(recoveryUpdate?.metadata?.blockedCategory)) {
const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');
await releaseRetryHold({ agentId: agent.id, task, success: false, agentMetadata: agent.metadata })
.catch(err => emitLog('warn', `Recovered delivery retry deferred for ${agent.id}: ${err.message}`, { agentId: agent.id }));
@@ -1963,7 +1965,11 @@ export async function finalizeAgent({
// failure.
await completeAgent(agentId, { ...completionRecord({
task, verdict: deliveryVerdict, hookResult, fidelity, executionInterruption, exitCode, duration, outputBuffer, validationPassed,
- }), executionSuccess, taskOutcome, ...(taskUpdate.metadata?.deliveryRecoveryAgentId === agentId ? { cleanupPending: true } : {}) });
+ }), executionSuccess, taskOutcome,
+ ...(['task-approval', 'task-recovery-exhausted'].includes(taskUpdate.metadata?.blockedCategory)
+ || taskUpdate.metadata?.retryPendingCleanup === agentId || taskUpdate.metadata?.deliveryRecoveryAgentId === agentId
+ ? { deliveryRecoveryOwned: true } : {}),
+ ...(taskUpdate.metadata?.deliveryRecoveryAgentId === agentId ? { cleanupPending: true } : {}) });
if (runId) {
// Every failed verdict overrides exit-code/commit rescue in run tracking,
diff --git a/server/services/agentFinalization.retireDeadAgent.test.js b/server/services/agentFinalization.retireDeadAgent.test.js
index fccaa38b9c..65742ee202 100644
--- a/server/services/agentFinalization.retireDeadAgent.test.js
+++ b/server/services/agentFinalization.retireDeadAgent.test.js
@@ -129,6 +129,16 @@ describe('retireDeadAgent (#8440)', () => {
} finally { await rm(workspacePath, { recursive: true, force: true }); }
});
+ it('preserves the first attempt pointer when a recovered outcome needs approval', async () => {
+ const taskOutcome = { status: 'blocked', reason: 'Authorized budget exhausted.', recovery: {
+ mode: 'approval', options: ['Increase the budget and retry.', 'Preserve the stopped work.'],
+ } };
+ await retireDeadAgent({ agent: { ...AGENT, result: { taskOutcome } }, task: TASK, success: true, exitCode: 0, duration: 100 });
+ expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.objectContaining({ status: 'blocked',
+ metadata: expect.objectContaining({ deliveryDecisionAgentId: AGENT.id }) }), 'user');
+ expect(releaseRetryHold).toHaveBeenCalledWith({ agentId: AGENT.id, task: TASK, success: false, agentMetadata: AGENT.metadata });
+ });
+
it('accepts a recovered no-op only after its output hook verifies delivery', async () => {
const taskOutcome = { status: 'no-action-needed', reason: 'Invented verification found no remaining work.' };
const hook = vi.fn(async () => ({ accepted: true }));
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.js
index bf19e6ea34..84d3bf0f27 100644
--- a/server/services/agentRepoStateVerification.js
+++ b/server/services/agentRepoStateVerification.js
@@ -275,7 +275,8 @@ export async function verifyAgentRepoState(input) {
const unsupported = outcomes.find(item => [REPO_STATE_SKIPS.DISABLED, REPO_STATE_SKIPS.NOT_WORKTREE,
REPO_STATE_SKIPS.PERSISTENT_WORKTREE, REPO_STATE_SKIPS.DISCARDED_WORKTREE,
'retirement-app-unavailable', 'retirement-policy-not-legacy-completed'].includes(item.skipReason)
- || (item.skipReason === 'cleanup-delivery-unverified' && item.observed?.prState === 'OPEN'));
+ || (item.skipReason === 'cleanup-delivery-unverified' && (['OPEN', 'CLOSED'].includes(item.observed?.prState)
+ || (item.observed?.prState === null && item.observed?.prUrl === null && item.observed?.unreadable?.length === 0))));
if (cleanupPending && unsupported) {
const { resolveUndeliveredTaskUpdate } = await import('./agentErrorAnalysis.js');
const outcome = { status: 'incomplete', reason: `Cleanup verification cannot run (${unsupported.skipReason}). ${input.task.metadata?.taskOutcome?.reason || ''}`.trim() };
diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.js
index 9b46e60a5a..f3cbe2baa9 100644
--- a/server/services/agentRepoStateVerification.test.js
+++ b/server/services/agentRepoStateVerification.test.js
@@ -195,6 +195,17 @@ describe('verifyAgentRepoState — clean runs', () => {
expect(recordRepoStateVerification).toHaveBeenLastCalledWith('agent-1', expect.objectContaining({ status: 'completed', reasons: ['cleanup-transferred-to-recovery'] }));
});
+ it.each(['OPEN', 'CLOSED'])('transfers a definitive %s undelivered PR out of the cleanup hold', async prState => {
+ updateTask.mockResolvedValueOnce({ status: 'in_progress' });
+ localBranch(true);
+ existsSync.mockReturnValue(true);
+ findPullRequestForBranch.mockResolvedValue({ status: 'found', detail: prState, url: 'https://github.com/example/repo/pull/7' });
+ const state = { ...agentState(), result: { success: false, cleanupPending: true } };
+ expect(await run({ success: false, agentState: state })).toMatchObject({ skipReason: 'cleanup-transferred-to-recovery' });
+ expect(releaseRetryHold).toHaveBeenCalledOnce();
+ expect(addTask).not.toHaveBeenCalled();
+ });
+
it('ignores the exact cleanup task itself while preserving a different pending owner', async () => {
const own = { ...task(), status: 'in_progress', metadata: { ...task().metadata,
existingBranch: BRANCH, deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-1' } };
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index e92ee59849..b3952694a2 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -993,10 +993,11 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }
if (!persisted) return {};
if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted'].includes(persisted.metadata?.blockedCategory)) {
+ if (persisted.metadata.deliveryDecisionAgentId !== agentId) return {};
const metadata = agentMetadata === undefined ? (await getAgentRecord(agentId))?.metadata : agentMetadata;
const patch = await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });
const result = await updateTask(task.id, { metadata: patch }, task.taskType || persisted.taskType || 'user', {
- expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt },
+ expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt, deliveryDecisionAgentId: agentId },
});
return result?.error || result?.statusChanged ? {} : patch;
}
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 8ebdb96b36..0f2c6e9047 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -1724,14 +1724,22 @@ describe('releaseRetryHold', () => {
// The direct-CLI / TUI shape: neither spawn path holds the agent record, so the
// helper has to read it for the worktree fields — via the transcript-free
// `getAgentRecord`, since a long TUI run's output.txt is megabytes.
+ it('never lets an older completion replace the current decision owner resume pointer', async () => {
+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: {
+ blockedCategory: 'task-approval', blockedAt: 'now', deliveryDecisionAgentId: 'agent-new',
+ } });
+ await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
it.each(['task-approval', 'task-recovery-exhausted'])('records a first attempt resume pointer for %s without dispatching it', async blockedCategory => {
- getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now' } });
+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: { blockedCategory, blockedAt: 'now', deliveryDecisionAgentId: 'agent-x' } });
git.getWorktreeBranches.mockResolvedValue(new Set([DEAD_BRANCH]));
existsSyncMock.mockReturnValue(true);
await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });
expect(updateTask).toHaveBeenCalledWith('task-1', { metadata: expect.objectContaining({
existingBranch: DEAD_BRANCH, resumedFromAgentId: 'agent-x', resumeWorktreePath: DEAD_TREE,
- }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now' } });
+ }) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now', deliveryDecisionAgentId: 'agent-x' } });
});
it('records the pointer for a failed run whose task is still pending', async () => {
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index d49e85dcd1..3a7ce92c35 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -745,7 +745,7 @@ const TRANSITION_METADATA_CLEARS = Object.freeze([
// still-paused agent record then read its own pause as spent and spawned a SECOND
// agent on a fresh task. The failure counters go too, or the revived task
// re-blocks on the budget it blocked with.
- ['leavesBlocked', ['blocker', 'blockedReason', 'blockedCategory', 'blockedAt', 'failureCount', 'lastErrorCategory', 'lastFailureAt', ...PAUSE_METADATA_KEYS]],
+ ['leavesBlocked', ['blocker', 'blockedReason', 'blockedCategory', 'blockedAt', 'deliveryDecisionAgentId', 'failureCount', 'lastErrorCategory', 'lastFailureAt', ...PAUSE_METADATA_KEYS]],
// A retry hold (#3373) only means anything while the task is `in_progress` waiting
// on a cleanup to resolve its resume pointer. Any other status the task reaches —
// terminal, or a requeue some other path performed — retires it, so drop the marker