Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: recover failed task reruns despite earlier success (#12057)
Adam Eivyccf7bf5c01df
diff --git a/server/services/cos.js b/server/services/cos.jsindex eb2a79b47c..0c640bb393 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -802,7 +802,7 @@ export async function forceSpawnTask(taskId) {* immediately instead of waiting out its liveness grace, which on the periodic* pass is what stops the sweep stealing a task mid-cleanup.*/-async function resetOrphanedTasks({ bootRecovery = false } = {}) {+export async function resetOrphanedTasks({ bootRecovery = false } = {}) {const state = await loadState();const { user: userTaskData, cos: cosTaskData } = await getAllTasks();@@ -842,16 +842,6 @@ async function resetOrphanedTasks({ bootRecovery = false } = {}) {.map(a => a.taskId));- // Track tasks whose agents completed successfully — if handleAgentCompletion's- // updateTask call failed silently (e.g., file write race after server restart),- // we should complete the task here rather than treating it as orphaned.- const successfullyCompletedTaskIds = new Map();- for (const agent of Object.values(state.agents)) {- if (agent.status === 'completed' && agent.result?.success) {- successfullyCompletedTaskIds.set(agent.taskId, agent.id);- }- }-emitLog('debug', `Running agents: ${runningAgentTaskIds.length}, recently completed: ${recentlyCompletedTaskIds.size}`, { taskIds: runningAgentTaskIds });// Route orphaned tasks through handleOrphanedTask for consistent retry counting,@@ -925,9 +915,10 @@ async function resetOrphanedTasks({ bootRecovery = false } = {}) {emitLog('debug', `Skipping task ${task.id} — agent recently completed, awaiting task status update`, { taskId: task.id });continue;}- // If the agent completed successfully but task wasn't updated (silent updateTask failure),- // complete the task now instead of treating it as orphaned- const successAgentId = successfullyCompletedTaskIds.get(task.id);+ // Only the latest run can settle a missed task update. An older success+ // must not hide a failed rerun that needs orphan recovery.+ const lastAgent = lastAgentByTask.get(task.id);+ const successAgentId = lastAgent?.status === 'completed' && lastAgent.result?.success ? lastAgent.id : null;if (successAgentId) {emitLog('warn', `🔧 Task ${task.id} still in_progress but agent ${successAgentId} completed successfully — completing task now (missed update)`, { taskId: task.id, agentId: successAgentId });await updateTask(task.id, { status: 'completed' }, task.taskType || (isInternalTaskId(task.id) ? 'internal' : 'user'));diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex 08877a481b..2816d77622 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -31,11 +31,11 @@* `availableSlots <= 0` short-circuit flips a clear red flag.*/-import { describe, it, expect, vi, beforeEach } from 'vitest';+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';import { readFileSync } from 'fs';import { fileURLToPath } from 'url';import { dirname, join } from 'path';-import { firstLine, isPerpetualRefillCandidate, perpetualRefillPlan } from './cos.js';+import { firstLine, isPerpetualRefillCandidate, perpetualRefillPlan, resetOrphanedTasks } from './cos.js';import { canQueueImprovementTasks, DEFAULT_STATE } from './cosState.js';import { createDequeueCapacity, countRunningAgentsByLocalEndpoint, isIdleTierEligible, isUserTaskRunnableUnattended } from './cosDequeue.js';import { generateTasksMarkdown, parseTasksMarkdown } from '../lib/taskParser.js';@@ -1998,6 +1998,61 @@ describe('cos.js source — priority + capacity invariants', () => {});});+describe('resetOrphanedTasks — latest run outcome (#12057)', () => {+ let taskStore;+ let agentManagement;+ const taskId = 'task-rerun';+ const older = {+ id: 'agent-older', taskId, status: 'completed', result: { success: true },+ startedAt: '2026-01-01T00:00:00Z', completedAt: '2026-01-01T00:01:00Z',+ };+ const latest = {+ id: 'agent-latest', taskId, status: 'completed', result: { success: false },+ startedAt: '2026-01-01T01:00:00Z', completedAt: '2026-01-01T01:01:00Z',+ metadata: { worktreePath: '/synthetic/resume-worktree', branch: 'resume-branch' },+ };++ beforeEach(async () => {+ const state = await import('./cosState.js');+ taskStore = await import('./cosTaskStore.js');+ const identity = await import('./instanceIdentity.js');+ agentManagement = await import('./agentManagement.js');+ // Put the older run last in the map: chronology, not insertion order,+ // determines which run can settle this task.+ vi.spyOn(state, 'loadState').mockResolvedValue({ agents: { latest, older } });+ vi.spyOn(identity, 'ensureInstanceId').mockResolvedValue('synthetic-instance');+ vi.spyOn(taskStore, 'getAllTasks').mockResolvedValue({+ user: { exists: true, grouped: { in_progress: [{ id: taskId, status: 'in_progress', metadata: {} }] } },+ cos: { exists: false },+ });+ vi.spyOn(taskStore, 'updateTask').mockResolvedValue({});+ vi.spyOn(agentManagement, 'handleOrphanedTask').mockResolvedValue({});+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-01-01T02:00:00Z'));+ });++ afterEach(() => vi.restoreAllMocks());++ it('recovers the latest failed rerun despite an older successful run', async () => {+ await resetOrphanedTasks();++ expect(taskStore.updateTask).not.toHaveBeenCalled();+ expect(agentManagement.handleOrphanedTask).toHaveBeenCalledExactlyOnceWith(+ taskId, latest.id, taskStore.getTaskById,+ { agentMetadata: latest.metadata, agentStartedAt: latest.startedAt }+ );+ });++ it('completes a missed task update when the latest run succeeded', async () => {+ const state = await import('./cosState.js');+ state.loadState.mockResolvedValue({ agents: { latest: { ...latest, result: { success: true } }, older } });++ await resetOrphanedTasks();++ expect(taskStore.updateTask).toHaveBeenCalledExactlyOnceWith(taskId, { status: 'completed' }, 'user');+ expect(agentManagement.handleOrphanedTask).not.toHaveBeenCalled();+ });+});+// A failed run's retry is held non-spawnable (`in_progress` + marker) while its// worktree cleanup resolves the resume pointer (#3373). The boot/health-check// sweep walks in_progress tasks, so it must leave a LIVE hold alone — requeueing