Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover failed task reruns despite earlier success (#12057)

Adam Eivyccf7bf5c01df
diff --git a/server/services/cos.js b/server/services/cos.js
index eb2a79b47c..0c640bb393 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -802,7 +802,7 @@ export async function forceSpawnTask(taskId) {
* immediately instead of waiting out its liveness grace, which on the periodic
* pass is what stops the sweep stealing a task mid-cleanup.
*/
-async function resetOrphanedTasks({ bootRecovery = false } = {}) {
+export async function resetOrphanedTasks({ bootRecovery = false } = {}) {
const state = await loadState();
const { user: userTaskData, cos: cosTaskData } = await getAllTasks();
@@ -842,16 +842,6 @@ async function resetOrphanedTasks({ bootRecovery = false } = {}) {
.map(a => a.taskId)
);
- // Track tasks whose agents completed successfully — if handleAgentCompletion's
- // updateTask call failed silently (e.g., file write race after server restart),
- // we should complete the task here rather than treating it as orphaned.
- const successfullyCompletedTaskIds = new Map();
- for (const agent of Object.values(state.agents)) {
- if (agent.status === 'completed' && agent.result?.success) {
- successfullyCompletedTaskIds.set(agent.taskId, agent.id);
- }
- }
-
emitLog('debug', `Running agents: ${runningAgentTaskIds.length}, recently completed: ${recentlyCompletedTaskIds.size}`, { taskIds: runningAgentTaskIds });
// Route orphaned tasks through handleOrphanedTask for consistent retry counting,
@@ -925,9 +915,10 @@ async function resetOrphanedTasks({ bootRecovery = false } = {}) {
emitLog('debug', `Skipping task ${task.id} — agent recently completed, awaiting task status update`, { taskId: task.id });
continue;
}
- // If the agent completed successfully but task wasn't updated (silent updateTask failure),
- // complete the task now instead of treating it as orphaned
- const successAgentId = successfullyCompletedTaskIds.get(task.id);
+ // Only the latest run can settle a missed task update. An older success
+ // must not hide a failed rerun that needs orphan recovery.
+ const lastAgent = lastAgentByTask.get(task.id);
+ const successAgentId = lastAgent?.status === 'completed' && lastAgent.result?.success ? lastAgent.id : null;
if (successAgentId) {
emitLog('warn', `🔧 Task ${task.id} still in_progress but agent ${successAgentId} completed successfully — completing task now (missed update)`, { taskId: task.id, agentId: successAgentId });
await updateTask(task.id, { status: 'completed' }, task.taskType || (isInternalTaskId(task.id) ? 'internal' : 'user'));
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index 08877a481b..2816d77622 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -31,11 +31,11 @@
* `availableSlots <= 0` short-circuit flips a clear red flag.
*/
-import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
-import { firstLine, isPerpetualRefillCandidate, perpetualRefillPlan } from './cos.js';
+import { firstLine, isPerpetualRefillCandidate, perpetualRefillPlan, resetOrphanedTasks } from './cos.js';
import { canQueueImprovementTasks, DEFAULT_STATE } from './cosState.js';
import { createDequeueCapacity, countRunningAgentsByLocalEndpoint, isIdleTierEligible, isUserTaskRunnableUnattended } from './cosDequeue.js';
import { generateTasksMarkdown, parseTasksMarkdown } from '../lib/taskParser.js';
@@ -1998,6 +1998,61 @@ describe('cos.js source — priority + capacity invariants', () => {
});
});
+describe('resetOrphanedTasks — latest run outcome (#12057)', () => {
+ let taskStore;
+ let agentManagement;
+ const taskId = 'task-rerun';
+ const older = {
+ id: 'agent-older', taskId, status: 'completed', result: { success: true },
+ startedAt: '2026-01-01T00:00:00Z', completedAt: '2026-01-01T00:01:00Z',
+ };
+ const latest = {
+ id: 'agent-latest', taskId, status: 'completed', result: { success: false },
+ startedAt: '2026-01-01T01:00:00Z', completedAt: '2026-01-01T01:01:00Z',
+ metadata: { worktreePath: '/synthetic/resume-worktree', branch: 'resume-branch' },
+ };
+
+ beforeEach(async () => {
+ const state = await import('./cosState.js');
+ taskStore = await import('./cosTaskStore.js');
+ const identity = await import('./instanceIdentity.js');
+ agentManagement = await import('./agentManagement.js');
+ // Put the older run last in the map: chronology, not insertion order,
+ // determines which run can settle this task.
+ vi.spyOn(state, 'loadState').mockResolvedValue({ agents: { latest, older } });
+ vi.spyOn(identity, 'ensureInstanceId').mockResolvedValue('synthetic-instance');
+ vi.spyOn(taskStore, 'getAllTasks').mockResolvedValue({
+ user: { exists: true, grouped: { in_progress: [{ id: taskId, status: 'in_progress', metadata: {} }] } },
+ cos: { exists: false },
+ });
+ vi.spyOn(taskStore, 'updateTask').mockResolvedValue({});
+ vi.spyOn(agentManagement, 'handleOrphanedTask').mockResolvedValue({});
+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-01-01T02:00:00Z'));
+ });
+
+ afterEach(() => vi.restoreAllMocks());
+
+ it('recovers the latest failed rerun despite an older successful run', async () => {
+ await resetOrphanedTasks();
+
+ expect(taskStore.updateTask).not.toHaveBeenCalled();
+ expect(agentManagement.handleOrphanedTask).toHaveBeenCalledExactlyOnceWith(
+ taskId, latest.id, taskStore.getTaskById,
+ { agentMetadata: latest.metadata, agentStartedAt: latest.startedAt }
+ );
+ });
+
+ it('completes a missed task update when the latest run succeeded', async () => {
+ const state = await import('./cosState.js');
+ state.loadState.mockResolvedValue({ agents: { latest: { ...latest, result: { success: true } }, older } });
+
+ await resetOrphanedTasks();
+
+ expect(taskStore.updateTask).toHaveBeenCalledExactlyOnceWith(taskId, { status: 'completed' }, 'user');
+ expect(agentManagement.handleOrphanedTask).not.toHaveBeenCalled();
+ });
+});
+
// A failed run's retry is held non-spawnable (`in_progress` + marker) while its
// worktree cleanup resolves the resume pointer (#3373). The boot/health-check
// sweep walks in_progress tasks, so it must leave a LIVE hold alone — requeueing