Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Investigation retry — put a failure-blocked task back in the queue as soon as
* the investigation that diagnosed it finishes.
*
* The failure loop used to be open at both ends. #3714 closed the first half:
* an agent failure files an investigation task that runs UNATTENDED, so nobody
* has to approve the diagnosis. But the original task stayed `blocked` forever
* after the fix landed — the only ways out were a human unblocking it by hand or
* the 14-day reaper quietly auto-expiring it as stale. So the fix shipped and
* the work it unblocked never ran.
*
* This module closes the second half: when an investigation task completes, the
* tasks it named in `metadata.affectedTasks` are revived automatically. Bounded
* by `MAX_AUTO_RETRIES_PER_TASK` so an unresolvable cause can't spin
* investigate → retry → fail → investigate forever — past the budget the task
* stays blocked, saying so, and the next same-cause investigation trips the
* `repeat-fingerprint` hold, which IS the point where a human is the right
* answer.
*
* Imports `cosTaskStore` directly rather than the `cos.js` facade: `cos.js`
* subscribes to this module's entry point, so going back through the facade
* would close a load-time cycle.
*/
import { getAllTasks, reviveBlockedTask, updateTask } from './cosTaskStore.js';
import { emitLog } from './cosEvents.js';
import {
MAX_AUTO_RETRIES_PER_TASK,
RETRY_SKIP_REASONS,
affectedTaskIds,
autoRetryMetadata,
couldReleaseBlockedTasks,
resolveInvestigationRetryTargets,
} from '../lib/investigationTasks.js';
// What a task's `blockedReason` becomes once the auto-retry loop gives up on it,
// so the queue says "the loop stopped trying" rather than still showing whatever
// failure blocked it two retries ago. Every other retry budget in CoS stamps its
// exhaustion onto the task; this one would otherwise report only to the journal,
// which is the one place the user won't look.
const RETRY_EXHAUSTED_CATEGORY = 'auto-retry-exhausted';
/**
* Revive every still-blocked task the just-completed `investigation` was
* tracking. A no-op (returning zero retried) for anything that is not a
* genuinely-finished investigation, so the caller can hand it every completed
* task without pre-filtering.
*
* Never throws: it runs off an event listener, outside the request lifecycle.
*
* @param {object} investigation the task that just reached `completed`
* @returns {Promise<{ retried: string[], skipped: Array<{ taskId: string, reason: string }> }>}
*/
export async function retryTasksResolvedByInvestigation(investigation, { now = Date.now() } = {}) {
// Answered from the task in hand, before any I/O — every task completion on the
// install reaches this, and the reaper alone flips up to 50 investigations to
// `completed` per sweep that release nothing (see couldReleaseBlockedTasks).
if (!couldReleaseBlockedTasks(investigation)) return { retried: [], skipped: [] };
// Index ONLY the ids this investigation names — usually one — rather than
// cloning both whole queues to look them up. `getAllTasks` returns each queue's
// raw tasks without a `taskType`, but the revive has to write back to the file
// the task actually lives in, so stamp it from the queue it came out of.
const wanted = new Set(affectedTaskIds(investigation));
const { user, cos } = await getAllTasks();
const tasksById = new Map();
for (const [taskType, list] of [['user', user?.tasks || []], ['internal', cos?.tasks || []]]) {
for (const task of list) if (wanted.has(task.id)) tasksById.set(task.id, { ...task, taskType });
}
const { targets, skipped } = resolveInvestigationRetryTargets({ investigation, tasksById });
const retried = [];
for (const task of targets) {
// `reviveBlockedTask` clears the block + failure/spawn budgets and emits
// `tasks:changed` action `unblocked`, which wakes the dequeue — so the retry
// starts on its own without a second signal from here.
const result = await reviveBlockedTask(
task.id,
{ metadata: autoRetryMetadata(task, investigation.id, now) },
task.taskType
).catch((err) => ({ error: err.message }));
if (result?.error) {
emitLog('warn', `⚠️ Auto-retry of task ${task.id} after investigation ${investigation.id} failed: ${result.error}`, {
taskId: task.id, investigationTaskId: investigation.id
});
continue;
}
retried.push(task.id);
}
await markExhaustedTasks(skipped, tasksById, investigation.id, now);
if (retried.length > 0) {
emitLog('info', `🔁 Investigation ${investigation.id} resolved — auto-retrying ${retried.length} blocked task(s): ${retried.join(', ')}`, {
taskId: investigation.id, retried: retried.length, skipped: skipped.length
});
} else if (skipped.length > 0) {
emitLog('debug', `⏭️ Investigation ${investigation.id} revived nothing: ${skipped.map(s => `${s.taskId} (${s.reason})`).join(', ')}`, {
taskId: investigation.id, skipped: skipped.length
});
}
return { retried, skipped };
}
/**
* Re-label the tasks whose auto-retry budget just ran out, so the task list says
* the loop gave up instead of still showing the original failure. Idempotent —
* a task already carrying the category is left alone, so a second investigation
* naming it doesn't rewrite the same reason. Best-effort: a failed re-label must
* not lose the retries that did land.
*/
async function markExhaustedTasks(skipped, tasksById, investigationId, now) {
for (const { taskId, reason } of skipped) {
if (reason !== RETRY_SKIP_REASONS.BUDGET_EXHAUSTED) continue;
const task = tasksById.get(taskId);
if (task?.metadata?.blockedCategory === RETRY_EXHAUSTED_CATEGORY) continue;
await updateTask(taskId, {
metadata: {
blockedReason: `Auto-retry gave up after ${MAX_AUTO_RETRIES_PER_TASK} attempts — the same cause keeps blocking this task`,
blockedCategory: RETRY_EXHAUSTED_CATEGORY,
autoRetryExhaustedAt: new Date(now).toISOString(),
}
}, task?.taskType || 'internal').catch((err) => {
emitLog('warn', `⚠️ Could not mark task ${taskId} auto-retry-exhausted after investigation ${investigationId}: ${err.message}`, { taskId });
});
}
}