Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
refactor: orphan-task recovery decides retry budget and guard order in a pure module (#12945)
[._.]/ Adam Eivyaa850bd0b2c2
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex a1592545fe..d8bf7f6b0b 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -1,9 +1,9 @@-import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';import { hasClaimFlowContract } from '../lib/claimFlowTaskTypes.js';import { isFalsyMeta, isTruthyMeta } from '../lib/metadataFlags.js';import { maintenance } from '../lib/maintenanceAdmission.js';import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';import { reconcileDeadOwnerReservations } from './deadOwnerReservations.js';+import { MAX_ORPHAN_RETRIES, ORPHAN_GUARDS, buildOrphanOutcome, classifyOrphanGuard, decideOrphanRetry } from './agentOrphanRecovery.js';/*** Agent Management*@@ -54,11 +54,6 @@ import * as shellService from './shell.js';const ROOT_DIR = PATHS.root;-// Max retries before creating investigation task-const MAX_ORPHAN_RETRIES = 3;-// Minimum cooldown between orphan retries (30 minutes)-const ORPHAN_RETRY_COOLDOWN_MS = 30 * 60 * 1000;-// Startup has two callers for this sweep: cos.start() awaits it before the// first dequeue, while the spawner's delayed safety sweep runs shortly after// its event wiring. Keep the whole sweep single-flight so both callers cannot@@ -1490,292 +1485,205 @@ export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agent// Without this, correctness depended on `cleanupOrphanedAgents` happening to// run before `resetOrphanedTasks` in cos.js's boot sequence.const wasInterrupted = interrupted ?? (agentMetadata?.interruptedBy === HOST_SHUTDOWN_REASON);- // Consume the breadcrumb the moment it's honored, exactly as the marker is- // consumed after the sweep. It is written once and would otherwise stay on the- // agent record forever: `resetOrphanedTasks` keys on the most recent agent per- // task, so a respawn that dies before creating its own agent record would keep- // re-deriving `wasInterrupted` from this same stale breadcrumb — and, because- // an interrupted run bypasses the cooldown, respawn on EVERY 15-minute sweep- // instead of once per 30 minutes.- if (wasInterrupted && agentMetadata?.interruptedBy === HOST_SHUTDOWN_REASON) {- await updateAgent(agentId, { metadata: { interruptedBy: null } })- .catch(err => emitLog('warn', `Could not clear interrupted breadcrumb on ${agentId}: ${err.message}`, { agentId }));- }- if (isPrivateSecurityTask({ metadata: agentMetadata })) {- const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');- await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });- }+ await consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted);+ if (isPrivateSecurityTask({ metadata: agentMetadata })) await removePrivateSecurityScratch(agentId);+const task = await getTaskByIdFn(taskId).catch(() => null);if (!task) {emitLog('warn', `Could not find task ${taskId} for orphaned agent ${agentId}`, { taskId, agentId });return;}- // User stops and policy refusals remain terminal across recovery sweeps.- if (task.status === 'blocked' && ['user-terminated', 'content-refusal', 'content-filtered'].includes(task.metadata?.blockedCategory)) {- emitLog('info', `⏭️ Skipping orphaned task ${taskId} — ${task.metadata.blockedCategory}`, { taskId, agentId });- return;- }+ const ctx = { taskId, agentId, task, agentMetadata, agentStartedAt };+ const guard = classifyOrphanGuard(task, agentMetadata);+ if (guard) return ORPHAN_GUARD_HANDLERS[guard](ctx);+ if (await completeIfCommitted(ctx)) return;- // Lost in-memory source scope cannot validate a recovered assessment. Never- // hand its transcript to an ordinary investigation or retry provider.- if (isPrivateSecurityTask(task) || isPrivateSecurityTask({ metadata: agentMetadata })) {- if (task.status === 'completed') return;- const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');- await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });- const taskUpdate = await stampLiExecutionVerdict({ status: 'blocked', metadata: { ...task.metadata,- blockedAt: new Date().toISOString(), blockedCategory: 'private-security-assessment-failed',- blockedReason: 'Private assessment was interrupted. Inspect its local output and explicitly rerun it; no automatic investigation or provider fallback will run.',- } }, task, { success: false });- await updateTask(taskId, taskUpdate, task.taskType || 'user');- return;- }+ const decision = decideOrphanRetry({ task, wasInterrupted, now: Date.now() });+ await applyOrphanDecision(ctx, decision);+}- // If a prior orphan in this sweep already routed the task through this handler- // (max-retries → investigation task created, or orphan-cooldown → blocked until later),- // skip — otherwise each additional orphaned agent for the same task spawns a- // duplicate investigation task and inflates orphanRetryCount past its ceiling.- if (task.status === 'blocked' &&- (task.metadata?.blockedCategory === 'max-retries' ||- task.metadata?.blockedCategory === 'orphan-cooldown')) {- emitLog('info', `⏭️ Skipping orphaned task ${taskId} — already handled (${task.metadata.blockedCategory})`, {- taskId, agentId, blockedCategory: task.metadata.blockedCategory- });- return;- }+/**+ * Consume the interrupted breadcrumb the moment it's honored, exactly as the+ * marker is consumed after the sweep. It is written once and would otherwise stay+ * on the agent record forever: `resetOrphanedTasks` keys on the most recent agent+ * per task, so a respawn that dies before creating its own agent record would keep+ * re-deriving `wasInterrupted` from this same stale breadcrumb — and, because an+ * interrupted run bypasses the cooldown, respawn on EVERY 15-minute sweep instead+ * of once per 30 minutes.+ */+async function consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted) {+ if (!wasInterrupted || agentMetadata?.interruptedBy !== HOST_SHUTDOWN_REASON) return;+ await updateAgent(agentId, { metadata: { interruptedBy: null } })+ .catch(err => emitLog('warn', `Could not clear interrupted breadcrumb on ${agentId}: ${err.message}`, { agentId }));+}- // Skip tasks already completed- if (task.status === 'completed') {- emitLog('debug', `⏭️ Skipping orphaned task ${taskId} — already completed`, { taskId, agentId });- return;- }+async function removePrivateSecurityScratch(agentId) {+ const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');+ await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });+}- // A crash/restart after the refusal was saved must not authorize a retry or- // commit rescue. Do not let an older refused run block a newer invocation.- const interruption = agentMetadata?.executionInterruption;- if (['content-refusal', 'content-filtered'].includes(interruption?.category)) {+function skipTerminalBlockedOrphan({ taskId, agentId, task }) {+ emitLog('info', `⏭️ Skipping orphaned task ${taskId} — ${task.metadata.blockedCategory}`, { taskId, agentId });+}++function skipHandledOrphan({ taskId, agentId, task }) {+ const { blockedCategory } = task.metadata;+ emitLog('info', `⏭️ Skipping orphaned task ${taskId} — already handled (${blockedCategory})`, { taskId, agentId, blockedCategory });+}++function skipCompletedOrphan({ taskId, agentId }) {+ emitLog('debug', `⏭️ Skipping orphaned task ${taskId} — already completed`, { taskId, agentId });+}++async function recoverPrivateSecurityOrphan({ taskId, agentId, task }) {+ if (task.status === 'completed') return;+ await removePrivateSecurityScratch(agentId);+ const taskUpdate = await stampLiExecutionVerdict({ status: 'blocked', metadata: { ...task.metadata,+ blockedAt: new Date().toISOString(), blockedCategory: 'private-security-assessment-failed',+ blockedReason: 'Private assessment was interrupted. Inspect its local output and explicitly rerun it; no automatic investigation or provider fallback will run.',+ } }, task, { success: false });+ await updateTask(taskId, taskUpdate, task.taskType || 'user');+}++async function recoverRefusedOrphan({ taskId, task, agentMetadata, agentStartedAt }) {+ const interruption = agentMetadata.executionInterruption;+ // Do not let an older refused run block a newer invocation.+ const startedAt = toEpochMs(agentStartedAt);+ if (startedAt && toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]) > startedAt) return;+ await updateTask(taskId, await stampLiExecutionVerdict({ status: 'blocked', metadata: {+ ...task.metadata, ...clearedRetryHoldMetadata(), executionInterruption: interruption,+ blockedAt: new Date().toISOString(), blockedCategory: interruption.category,+ blockedReason: 'Provider safety refusal; automatic continuation, retries and provider fallback are disabled.',+ } }, task, { success: false }), task.taskType || 'user');+}++async function recoverDeepAuditOrphan({ taskId, agentId, task, agentMetadata, agentStartedAt }) {+ if (task.status !== 'in_progress') return;+ const held = isRetryHeld(task.metadata);+ if (held && task.metadata[RETRY_HOLD_KEY] !== agentId) return;+ if (!held) {+ const spawnedAt = toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]);const startedAt = toEpochMs(agentStartedAt);- if (startedAt && toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]) > startedAt) return;- await updateTask(taskId, await stampLiExecutionVerdict({ status: 'blocked', metadata: {- ...task.metadata, ...clearedRetryHoldMetadata(), executionInterruption: interruption,- blockedAt: new Date().toISOString(), blockedCategory: interruption.category,- blockedReason: 'Provider safety refusal; automatic continuation, retries and provider fallback are disabled.',- } }, task, { success: false }), task.taskType || 'user');- return;+ // A legacy/unknown owner cannot authorize parking a newer invocation.+ if (!spawnedAt || !startedAt || spawnedAt > startedAt) return;+ const agents = await getAgents();+ if (!Array.isArray(agents) || agents.some(entry => entry.taskId === taskId && entry.id !== agentId+ && (entry.status === 'running' || toEpochMs(entry.startedAt) > startedAt))) return;}+ const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata });+ await updateTask(taskId, { status: 'blocked', metadata: {+ ...resumePatch, ...clearedRetryHoldMetadata(), blockedCategory: 'deep-audit-partial',+ blockedReason: 'Interrupted Deep audit; inspect checkpoint and resume explicitly', blockedAt: new Date().toISOString(),+ } }, task.taskType || 'user', { expectedStatus: 'in_progress',+ expectedMetadata: held ? { [RETRY_HOLD_KEY]: agentId } : { [LAST_SPAWNED_AT_KEY]: task.metadata[LAST_SPAWNED_AT_KEY] },+ });+}- // Recovery of a dead Deep invocation is a checkpoint, never permission to- // auto-launch another provider call or to treat commits as completed coverage.- if (isLegacyDeepAudit(task.metadata)) {- if (task.status !== 'in_progress') return;- const held = isRetryHeld(task.metadata);- if (held && task.metadata[RETRY_HOLD_KEY] !== agentId) return;- if (!held) {- const spawnedAt = toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]);- const startedAt = toEpochMs(agentStartedAt);- // A legacy/unknown owner cannot authorize parking a newer invocation.- if (!spawnedAt || !startedAt || spawnedAt > startedAt) return;- const agents = await getAgents();- if (!Array.isArray(agents) || agents.some(entry => entry.taskId === taskId && entry.id !== agentId- && (entry.status === 'running' || toEpochMs(entry.startedAt) > startedAt))) return;- }- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata });- await updateTask(taskId, { status: 'blocked', metadata: {- ...resumePatch, ...clearedRetryHoldMetadata(), blockedCategory: 'deep-audit-partial',- blockedReason: 'Interrupted Deep audit; inspect checkpoint and resume explicitly', blockedAt: new Date().toISOString(),- } }, task.taskType || 'user', { expectedStatus: 'in_progress',- expectedMetadata: held ? { [RETRY_HOLD_KEY]: agentId } : { [LAST_SPAWNED_AT_KEY]: task.metadata[LAST_SPAWNED_AT_KEY] },- });+/**+ * The run's verdict is already persisted — it failed, and was budgeted a retry —+ * so this is not a fresh orphan: finish the transition the dead process started+ * (resolve the pointer, flip to `pending`, drop the marker) rather than charging+ * it orphan budget for a decision already made. Precedes the commit check: a failed+ * run that committed is exactly the case the resume pointer exists for, and+ * completing the task on that evidence would discard the retry finalizeAgent granted.+ */+async function recoverRetryHeldOrphan({ taskId, agentId, task, agentMetadata }) {+ const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {+ emitLog('warn', `Resume pointer for held task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });+ return {};+ });+ const targetBranch = resolveTaskTargetBranch(resumePatch);+ const result = await updateTask(taskId, {+ status: 'pending',+ metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }+ }, task.taskType || 'user', {+ expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },+ });+ if (result?.error) {+ emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });return;}+ if (result?.statusChanged) return;+ emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {+ taskId, agentId, branchName: targetBranch+ });+}- // A retry hold whose process died mid-transition (#3373). The run's verdict is- // already persisted — it failed, and was budgeted a retry — so this is not a- // fresh orphan: finish the transition the dead process started (resolve the- // pointer, flip to `pending`, drop the marker) rather than charging it orphan- // budget for a decision that was already made. Runs BEFORE the commit check- // below: a failed run that committed is exactly the case the resume pointer- // exists for, and completing the task on that evidence would discard the retry- // finalizeAgent already granted.- if (isRetryHeld(task.metadata)) {- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {- emitLog('warn', `Resume pointer for held task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });- return {};- });- const targetBranch = resolveTaskTargetBranch(resumePatch);- const result = await updateTask(taskId, {- status: 'pending',- metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }- }, task.taskType || 'user', {- expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },- });- if (result?.error) {- emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });- return;- }- if (result?.statusChanged) return;- emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {- taskId, agentId, branchName: targetBranch- });- return;- }+// One handler per `ORPHAN_GUARDS` reason; the precedence lives in that list.+const ORPHAN_GUARD_HANDLERS = {+ 'terminal-block': skipTerminalBlockedOrphan,+ 'private-security': recoverPrivateSecurityOrphan,+ 'already-handled': skipHandledOrphan,+ 'completed': skipCompletedOrphan,+ 'content-refusal': recoverRefusedOrphan,+ 'deep-audit': recoverDeepAuditOrphan,+ 'retry-held': recoverRetryHeldOrphan,+};+for (const { reason } of ORPHAN_GUARDS) {+ if (!ORPHAN_GUARD_HANDLERS[reason]) throw new Error(`No orphan recovery handler for guard "${reason}"`);+}- // Check if the agent actually committed work before treating as orphaned.- // Scoped to the dead run's OWN window (#3637): the retired task-id commit marker- // was never emitted by anything, and an unbounded `git log` would credit this- // task with any commit in the repo — including another agent's.- // Both shapes the agent record can carry — the persisted ISO string and a raw- // epoch-ms number (see toEpochMs for why a bare Date.parse drops the latter).- const orphanRunStartedAt = toEpochMs(agentStartedAt);- const commitFound = Number.isFinite(orphanRunStartedAt)- && await committedDuringRun(agentMetadata?.workspacePath || ROOT_DIR, orphanRunStartedAt);- if (commitFound) {- emitLog('info', `✅ Orphaned agent ${agentId} actually completed work - commit found for task ${taskId}`, { taskId, agentId });- const taskUpdate = await stampLiExecutionVerdict({ status: 'completed' }, task, { success: true });- await updateTask(taskId, taskUpdate, task.taskType || 'user');- return;- }+/**+ * Did the agent actually commit work before it died? Scoped to the dead run's OWN+ * window (#3637): the retired task-id commit marker was never emitted by anything,+ * and an unbounded `git log` would credit this task with any commit in the repo —+ * including another agent's. `toEpochMs` accepts both shapes the agent record can+ * carry (the persisted ISO string and a raw epoch-ms number); without a start time+ * the probe is skipped.+ */+async function completeIfCommitted({ taskId, agentId, task, agentMetadata, agentStartedAt }) {+ const runStartedAt = toEpochMs(agentStartedAt);+ const commitFound = Number.isFinite(runStartedAt)+ && await committedDuringRun(agentMetadata?.workspacePath || ROOT_DIR, runStartedAt);+ if (!commitFound) return false;+ emitLog('info', `✅ Orphaned agent ${agentId} actually completed work - commit found for task ${taskId}`, { taskId, agentId });+ const taskUpdate = await stampLiExecutionVerdict({ status: 'completed' }, task, { success: true });+ await updateTask(taskId, taskUpdate, task.taskType || 'user');+ return true;+}- // Get current retry count from task metadata.- //- // Retry budget and the cooldown exist to stop a task that keeps KILLING ITS OWN- // agent from spawning forever. A PortOS host restart is not that: the agent was- // healthy and PortOS took it down (#3202). Charging it a retry — and arming the- // 30-minute cooldown — punishes the task for our maintenance, and three routine- // restarts were enough to exhaust the budget and block the task outright. So an- // interrupted run costs no retry and arms no cooldown; it just requeues with a- // resume pointer. `MAX_TOTAL_SPAWNS` still applies, so nothing can spawn- // unboundedly even under a restart loop.- const priorOrphanRetries = Number(task.metadata?.orphanRetryCount) || 0;- const retryCount = wasInterrupted ? priorOrphanRetries : priorOrphanRetries + 1;- const taskType = task.taskType || 'user';-- // `totalSpawnCount` is incremented when a task goes `in_progress`, so the run a- // restart destroyed already consumed a spawn. REFUND it: that spawn produced no- // work and was ended by PortOS, not by the task. Without the refund the fix is- // only half-done — the ceiling moves from 3 restarts to MAX_TOTAL_SPAWNS, after- // which the task is still blocked `max-retries` AND an "[Auto-Fix] Investigate- // repeated agent orphaning" task is filed blaming an agent that never failed.- // The refund keeps a real ceiling on genuine failures (each still costs a- // spawn) while making a restart cost nothing.- const priorTotalSpawns = Number(task.metadata?.totalSpawnCount) || 0;- const totalSpawns = wasInterrupted ? Math.max(0, priorTotalSpawns - 1) : priorTotalSpawns;-- // Block if total spawn count across all retry types is exhausted- const totalExceeded = totalSpawns >= MAX_TOTAL_SPAWNS;-- // Enforce cooldown between orphan retries- const lastOrphanedAt = task.metadata?.lastOrphanedAt ? new Date(task.metadata.lastOrphanedAt).getTime() : 0;- const cooldownRemaining = lastOrphanedAt ? ORPHAN_RETRY_COOLDOWN_MS - (Date.now() - lastOrphanedAt) : 0;- const inCooldown = !wasInterrupted && cooldownRemaining > 0;-- // An interrupted run is exempt from the orphan ceiling for the same reason it- // is exempt from the counter: it isn't one of the failures the ceiling counts.- // `totalExceeded` still gates it, so a restart loop can't spawn without bound.- const withinOrphanBudget = wasInterrupted || retryCount < MAX_ORPHAN_RETRIES;-- if (withinOrphanBudget && !totalExceeded && !inCooldown) {- emitLog('info', wasInterrupted+function logOrphanDecision(taskId, decision) {+ const { outcome, interrupted, retryCount, totalSpawns, cooldownMinutes, reason } = decision;+ if (outcome === 'requeue') {+ emitLog('info', interrupted? `Resuming task ${taskId} interrupted by a PortOS restart (no retry charged; total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`: `Resetting orphaned task ${taskId} for retry (attempt ${retryCount}/${MAX_ORPHAN_RETRIES}, total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`, {- taskId,- retryCount,- totalSpawns,- interrupted: wasInterrupted,- maxRetries: MAX_ORPHAN_RETRIES+ taskId, retryCount, totalSpawns, interrupted, maxRetries: MAX_ORPHAN_RETRIES});+ } else if (outcome === 'cooldown') {+ emitLog('info', `⏳ Orphan retry for task ${taskId} in cooldown (${cooldownMinutes}m remaining)`, { taskId, cooldownMinutes, retryCount });+ } else {+ emitLog('warn', `Task ${taskId} blocked: ${reason}, creating investigation task`, { taskId, retryCount, totalSpawns });+ }+}- // Point the retry at whatever the dead run left behind — its branch, or the- // whole worktree when the process died mid-edit and the uncommitted work is- // still sitting in it. Folded into THIS write rather than stamped separately- // beforehand: the flip to `pending` emits `tasks:changed`, which can spawn the- // retry immediately, so a pointer written afterwards could land too late.- // Fails open to "start clean": requeueing the task matters far more than- // resuming it, and a throw here would strand it in_progress until the next sweep.- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {+/** Persist the retry decision: requeue, orphan-cooldown block, or max-retries block + investigation. */+async function applyOrphanDecision({ taskId, agentId, task, agentMetadata }, decision) {+ logOrphanDecision(taskId, decision);+ const requeue = decision.outcome === 'requeue';+ // Point the retry at whatever the dead run left behind — its branch, or the+ // whole worktree when the process died mid-edit and the uncommitted work is+ // still sitting in it. Folded into THIS write rather than stamped separately+ // beforehand: the flip to `pending` emits `tasks:changed`, which can spawn the+ // retry immediately, so a pointer written afterwards could land too late.+ // Fails open to "start clean": requeueing the task matters far more than+ // resuming it, and a throw here would strand it in_progress until the next sweep.+ const resumePatch = requeue+ ? await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {emitLog('warn', `Resume pointer for task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });return {};- });-- // An interrupted run records WHAT happened without touching the orphan- // bookkeeping: `lastOrphanedAt` stays as-is so a later genuine orphan's- // cooldown is still measured from the last genuine orphan, not from our- // restart. The genuine-orphan branch clears `interruptedByRestart` — the- // metadata spread below carries every prior key forward, so leaving it set- // would make a task that was once restarted look permanently restart-- // interrupted, through this orphan and every later one.- const now = new Date().toISOString();- const recoveryMetadata = wasInterrupted- ? {- // Persist the refunded count (see the totalSpawns note above) so the- // restart's spawn is genuinely given back rather than just ignored once.- totalSpawnCount: totalSpawns,- lastInterruptedAt: now,- lastInterruptedAgentId: agentId,- interruptedByRestart: true,- }- : {- orphanRetryCount: retryCount,- lastOrphanedAt: now,- lastOrphanedAgentId: agentId,- interruptedByRestart: false,- };-- await updateTask(taskId, {- status: 'pending',- metadata: {- ...task.metadata,- ...recoveryMetadata,- ...resumePatch- }- }, taskType);- } else if (inCooldown && retryCount < MAX_ORPHAN_RETRIES && !totalExceeded) {- const cooldownMinutes = Math.ceil(cooldownRemaining / 60000);- emitLog('info', `⏳ Orphan retry for task ${taskId} in cooldown (${cooldownMinutes}m remaining)`, {- taskId, cooldownMinutes, retryCount- });-- await updateTask(taskId, await stampLiExecutionVerdict({- status: 'blocked',- metadata: {- ...task.metadata,- orphanRetryCount: retryCount,- lastOrphanedAt: new Date().toISOString(),- lastOrphanedAgentId: agentId,- // Cleared for the same reason the requeue branch clears it: the metadata- // spread carries every prior key forward, so a task interrupted once- // would read as restart-interrupted through every later outcome.- interruptedByRestart: false,- blockedReason: `Orphan retry cooldown (${cooldownMinutes}m remaining)`,- blockedCategory: 'orphan-cooldown',- blockedAt: new Date().toISOString(),- cooldownUntil: new Date(Date.now() + cooldownRemaining).toISOString()- }- }, task, { success: false }), taskType);- } else {- const reason = totalExceeded- ? `total spawns exceeded (${totalSpawns}/${MAX_TOTAL_SPAWNS})`- : `orphan retries exceeded (${retryCount}/${MAX_ORPHAN_RETRIES})`;- emitLog('warn', `Task ${taskId} blocked: ${reason}, creating investigation task`, {- taskId,- retryCount,- totalSpawns- });-- await updateTask(taskId, await stampLiExecutionVerdict({- status: 'blocked',- metadata: {- ...task.metadata,- orphanRetryCount: retryCount,- // See the requeue branch: the metadata spread carries prior keys forward.- interruptedByRestart: false,- blockedReason: reason,- blockedCategory: 'max-retries',- blockedAt: new Date().toISOString()- }- }, task, { success: false }), taskType);+ })+ : {};+ const outcome = buildOrphanOutcome({ decision, task, agentId, now: Date.now(), resumePatch });+ const taskUpdate = requeue ? outcome : await stampLiExecutionVerdict(outcome, task, { success: false });+ await updateTask(taskId, taskUpdate, task.taskType || 'user');+ if (decision.outcome === 'blocked') await fileOrphanInvestigation({ taskId, agentId, task, decision });+}- const description = `[Auto-Fix] Investigate repeated agent orphaning for task ${taskId}+async function fileOrphanInvestigation({ taskId, agentId, task, decision }) {+ const { retryCount, totalSpawns, reason } = decision;+ const description = `[Auto-Fix] Investigate repeated agent orphaning for task ${taskId}**Original Task**: ${(task.description || '').substring(0, 200)}**Orphan Retries**: ${retryCount}@@ -1791,20 +1699,19 @@ This task has been blocked after ${totalSpawns} total agent spawns. Investigate:Once the issue is resolved, task \`${taskId}\` is retried automatically — completing this task revives it.`;- // Filed through the shared producer so this participates in the same- // loop-aware approval policy, storm counter, and fingerprint dedup as every- // other investigation (#3714), and so `affectedTasks` closes the loop: the- // orphaned task comes back on its own when this one completes. Hardcoding- // `approvalRequired: true` here is what turned one repeatedly-orphaned task- // into a queue of approvals nobody asked for.- await fileInvestigationTask({- description,- priority: 'HIGH',- context: `Auto-generated from repeated orphan failures for task ${taskId}`,- fingerprint: buildInvestigationFingerprint(task, { category: 'max-retries' }),- affectedTasks: [taskId]- }).catch(err => {- emitLog('error', `Failed to create investigation task: ${err.message}`, { taskId, error: err.message });- });- }+ // Filed through the shared producer so this participates in the same+ // loop-aware approval policy, storm counter, and fingerprint dedup as every+ // other investigation (#3714), and so `affectedTasks` closes the loop: the+ // orphaned task comes back on its own when this one completes. Hardcoding+ // `approvalRequired: true` here is what turned one repeatedly-orphaned task+ // into a queue of approvals nobody asked for.+ await fileInvestigationTask({+ description,+ priority: 'HIGH',+ context: `Auto-generated from repeated orphan failures for task ${taskId}`,+ fingerprint: buildInvestigationFingerprint(task, { category: 'max-retries' }),+ affectedTasks: [taskId]+ }).catch(err => {+ emitLog('error', `Failed to create investigation task: ${err.message}`, { taskId, error: err.message });+ });}diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.jsindex 420cda8554..f667f48ccc 100644--- a/server/services/agentManagement.test.js+++ b/server/services/agentManagement.test.js@@ -759,6 +759,141 @@ describe('handleOrphanedTask — duplicate-investigation guard', () => {});});+// Pins the exact metadata handleOrphanedTask persists for each recovery outcome and+// the precedence between its guards (#12945). The retry budget is a documented+// contract (#3202 refund rules): a drift here retries a task forever or blocks it early.+describe('handleOrphanedTask — persisted outcome shapes and guard precedence (#12945)', () => {+ const NOW = Date.parse('2026-06-01T12:00:00.000Z');+ const iso = (ms) => new Date(ms).toISOString();+ const MIN = 60 * 1000;+ const taskWith = (metadata, status = 'in_progress') =>+ ({ id: 'task-foo', status, taskType: 'user', description: 'Original work', metadata });+ const run = (task, options) => handleOrphanedTask('task-foo', 'agent-dead', vi.fn().mockResolvedValue(task), options);+ const persisted = () => updateTask.mock.calls[0];++ beforeEach(() => {+ vi.clearAllMocks();+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(NOW);+ getAllTasks.mockResolvedValue({ user: { tasks: [] }, cos: { tasks: [] } });+ });+ afterEach(() => {+ vi.useRealTimers();+ });++ it('requeues a genuine orphan: charges a retry, stamps the orphan breadcrumb, clears interruptedByRestart', async () => {+ resolveTaskResumePatch.mockResolvedValueOnce({ resumeBranch: 'branch-a' });+ await run(taskWith({ keep: 'x', orphanRetryCount: 1, totalSpawnCount: 2, interruptedByRestart: true }));++ expect(persisted()).toEqual(['task-foo', {+ status: 'pending',+ metadata: {+ keep: 'x', orphanRetryCount: 2, totalSpawnCount: 2, interruptedByRestart: false,+ lastOrphanedAt: iso(NOW), lastOrphanedAgentId: 'agent-dead',+ resumeBranch: 'branch-a',+ },+ }, 'user']);+ });++ it('requeues a restart-interrupted run without charging a retry, refunding the spawn and leaving lastOrphanedAt alone', async () => {+ await run(taskWith({ orphanRetryCount: 2, totalSpawnCount: 3, lastOrphanedAt: iso(NOW - MIN) }), { interrupted: true });++ expect(persisted()).toEqual(['task-foo', {+ status: 'pending',+ metadata: {+ orphanRetryCount: 2, totalSpawnCount: 2, lastOrphanedAt: iso(NOW - MIN),+ lastInterruptedAt: iso(NOW), lastInterruptedAgentId: 'agent-dead', interruptedByRestart: true,+ },+ }, 'user']);+ });++ it('blocks a second genuine orphan inside the cooldown, with the remaining time and a cleared interruptedByRestart', async () => {+ await run(taskWith({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: iso(NOW - 10 * MIN), interruptedByRestart: true }));++ expect(persisted()).toEqual(['task-foo', {+ status: 'blocked',+ metadata: {+ orphanRetryCount: 2, totalSpawnCount: 2, lastOrphanedAt: iso(NOW), interruptedByRestart: false,+ lastOrphanedAgentId: 'agent-dead',+ blockedReason: 'Orphan retry cooldown (20m remaining)', blockedCategory: 'orphan-cooldown',+ blockedAt: iso(NOW), cooldownUntil: iso(NOW + 20 * MIN),+ },+ }, 'user']);+ expect(addTask).not.toHaveBeenCalled();+ });++ it('blocks as max-retries and files an investigation once the orphan retries are spent', async () => {+ await run(taskWith({ orphanRetryCount: 2, totalSpawnCount: 3, interruptedByRestart: true }));++ expect(persisted()).toEqual(['task-foo', {+ status: 'blocked',+ metadata: {+ orphanRetryCount: 3, totalSpawnCount: 3, interruptedByRestart: false,+ blockedReason: 'orphan retries exceeded (3/3)', blockedCategory: 'max-retries', blockedAt: iso(NOW),+ },+ }, 'user']);+ expect(addTask).toHaveBeenCalledTimes(1);+ expect(addTask.mock.calls[0][0].description).toContain('**Orphan Retries**: 3\n**Total Spawns**: 3');+ });++ it('blocks as max-retries on the spawn ceiling even for a restart-interrupted run, without charging a retry', async () => {+ // 6 spawns minus the refunded one is still at the MAX_TOTAL_SPAWNS ceiling of 5.+ await run(taskWith({ orphanRetryCount: 0, totalSpawnCount: 6 }), { interrupted: true });++ expect(persisted()).toEqual(['task-foo', {+ status: 'blocked',+ metadata: {+ orphanRetryCount: 0, totalSpawnCount: 6, interruptedByRestart: false,+ blockedReason: 'total spawns exceeded (5/5)', blockedCategory: 'max-retries', blockedAt: iso(NOW),+ },+ }, 'user']);+ expect(addTask).toHaveBeenCalledTimes(1);+ });++ describe('guard precedence', () => {+ const refusal = { executionInterruption: { category: 'content-refusal' } };++ it('terminal user-terminated outranks the private-security failure write', async () => {+ await run(taskWith({ analysisType: 'private-security-assessment', blockedCategory: 'user-terminated' }, 'blocked'));+ expect(updateTask).not.toHaveBeenCalled();+ });++ it('the private-security failure write outranks the already-handled skip', async () => {+ await run(taskWith({ analysisType: 'private-security-assessment', blockedCategory: 'max-retries' }, 'blocked'));+ expect(updateTask).toHaveBeenCalledTimes(1);+ expect(updateTask.mock.calls[0][1].metadata.blockedCategory).toBe('private-security-assessment-failed');+ });++ it('a completed private-security task is left alone', async () => {+ await run(taskWith({ analysisType: 'private-security-assessment' }, 'completed'));+ expect(updateTask).not.toHaveBeenCalled();+ });++ it('the already-handled skip outranks the content-refusal write', async () => {+ await run(taskWith({ blockedCategory: 'orphan-cooldown' }, 'blocked'), { agentMetadata: refusal });+ expect(updateTask).not.toHaveBeenCalled();+ });++ it('the completed skip outranks the content-refusal write', async () => {+ await run(taskWith({}, 'completed'), { agentMetadata: refusal });+ expect(updateTask).not.toHaveBeenCalled();+ });++ it('the content-refusal write outranks the retry-hold transition', async () => {+ await run(taskWith({ retryPendingCleanup: 'agent-dead' }), { agentMetadata: refusal });+ expect(updateTask).toHaveBeenCalledTimes(1);+ expect(updateTask.mock.calls[0][1].metadata.blockedCategory).toBe('content-refusal');+ });++ it('a retry-held task finishes its transition instead of being completed by a commit found in the run window', async () => {+ committedDuringRun.mockResolvedValueOnce(true);+ await run(taskWith({ retryPendingCleanup: 'agent-dead' }), { agentStartedAt: iso(NOW - MIN) });+ expect(committedDuringRun).not.toHaveBeenCalled();+ expect(updateTask.mock.calls[0][1].status).toBe('pending');+ });+ });+});+describe('pauseAgent', () => {beforeEach(() => {vi.clearAllMocks();diff --git a/server/services/agentOrphanRecovery.js b/server/services/agentOrphanRecovery.jsnew file mode 100644index 0000000000..a190231961--- /dev/null+++ b/server/services/agentOrphanRecovery.js@@ -0,0 +1,148 @@+/**+ * Orphan recovery decisions — the pure half of `handleOrphanedTask` (#12945).+ *+ * Everything here is a function of its arguments (no I/O, no clock reads), so the+ * guard precedence and the retry-budget contract can be pinned without a task+ * store. `agentManagement.js` owns the reads and writes around these decisions.+ */++import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';+import { MAX_TOTAL_SPAWNS } from '../lib/cosValidation.js';+import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';+import { isRetryHeld } from '../lib/taskRetryHold.js';++// Max genuine orphan retries before the task is blocked and an investigation filed.+export const MAX_ORPHAN_RETRIES = 3;+// Minimum gap between genuine orphan retries (30 minutes).+export const ORPHAN_RETRY_COOLDOWN_MS = 30 * 60 * 1000;++const REFUSAL_CATEGORIES = ['content-refusal', 'content-filtered'];+const TERMINAL_BLOCK_CATEGORIES = ['user-terminated', ...REFUSAL_CATEGORIES];+// A prior orphan in the same sweep already routed the task through recovery.+const ALREADY_HANDLED_CATEGORIES = ['max-retries', 'orphan-cooldown'];++const isBlockedWith = (task, categories) =>+ task.status === 'blocked' && categories.includes(task.metadata?.blockedCategory);++/**+ * Guards checked before any retry accounting, FIRST MATCH WINS — the array order+ * is the precedence. Each reason has a handler in `agentManagement.js`; some only+ * skip, others persist a terminal state (private-security, content-refusal,+ * deep-audit, retry-held). The interleaving is deliberate: a private-security task+ * is failed even when already `max-retries`-blocked, but a user-terminated block+ * outranks it, and `completed` outranks every write below it.+ */+export const ORPHAN_GUARDS = [+ // User stops and policy refusals remain terminal across recovery sweeps.+ { reason: 'terminal-block', matches: (task) => isBlockedWith(task, TERMINAL_BLOCK_CATEGORIES) },+ // Lost in-memory source scope cannot validate a recovered assessment.+ { reason: 'private-security', matches: (task, agentMetadata) => isPrivateSecurityTask(task) || isPrivateSecurityTask({ metadata: agentMetadata }) },+ // Otherwise each extra orphaned agent for the same task files a duplicate investigation.+ { reason: 'already-handled', matches: (task) => isBlockedWith(task, ALREADY_HANDLED_CATEGORIES) },+ { reason: 'completed', matches: (task) => task.status === 'completed' },+ // A crash after the refusal was saved must not authorize a retry or commit rescue.+ { reason: 'content-refusal', matches: (_task, agentMetadata) => REFUSAL_CATEGORIES.includes(agentMetadata?.executionInterruption?.category) },+ // A dead Deep invocation is a checkpoint, never permission to launch another provider call.+ { reason: 'deep-audit', matches: (task) => isLegacyDeepAudit(task.metadata) },+ // A retry whose process died mid-transition (#3373): finish it, don't charge orphan budget.+ { reason: 'retry-held', matches: (task) => isRetryHeld(task.metadata) },+];++/** The reason of the first guard that claims this orphan, or null when it needs a retry decision. */+export function classifyOrphanGuard(task, agentMetadata) {+ return ORPHAN_GUARDS.find(guard => guard.matches(task, agentMetadata))?.reason ?? null;+}++const counter = (value) => Number(value) || 0;++/** Ms left before the next genuine orphan retry may run; 0 when none was recorded (or it is unparseable). */+function remainingCooldownMs(lastOrphanedAt, now) {+ const lastMs = lastOrphanedAt ? new Date(lastOrphanedAt).getTime() : 0;+ return lastMs ? ORPHAN_RETRY_COOLDOWN_MS - (now - lastMs) : 0;+}++/**+ * The retry-budget decision. A genuine orphan costs one retry and arms the+ * cooldown; a run PortOS itself interrupted (host restart) costs neither and has+ * its spawn refunded (#3202), because the task did not kill its own agent.+ * `MAX_TOTAL_SPAWNS` still bounds both, so a restart loop cannot spawn without limit.+ *+ * @returns {{ outcome: 'requeue'|'cooldown'|'blocked', interrupted: boolean,+ * retryCount: number, totalSpawns: number, cooldownRemaining: number,+ * cooldownMinutes: number, reason: string|null }}+ * `reason` is the blockedReason for `cooldown` and `blocked`, null for `requeue`.+ */+export function decideOrphanRetry({ task, wasInterrupted, now }) {+ const priorRetries = counter(task.metadata?.orphanRetryCount);+ const retryCount = wasInterrupted ? priorRetries : priorRetries + 1;+ const priorSpawns = counter(task.metadata?.totalSpawnCount);+ const totalSpawns = wasInterrupted ? Math.max(0, priorSpawns - 1) : priorSpawns;++ const cooldownRemaining = remainingCooldownMs(task.metadata?.lastOrphanedAt, now);+ const cooldownMinutes = Math.ceil(cooldownRemaining / 60000);++ const totalExceeded = totalSpawns >= MAX_TOTAL_SPAWNS;+ // An interrupted run is exempt from the orphan ceiling for the same reason it is+ // exempt from the counter; `totalExceeded` still gates it.+ const withinOrphanBudget = wasInterrupted || retryCount < MAX_ORPHAN_RETRIES;+ const decision = { interrupted: wasInterrupted, retryCount, totalSpawns, cooldownRemaining, cooldownMinutes };++ if (totalExceeded || !withinOrphanBudget) {+ const reason = totalExceeded+ ? `total spawns exceeded (${totalSpawns}/${MAX_TOTAL_SPAWNS})`+ : `orphan retries exceeded (${retryCount}/${MAX_ORPHAN_RETRIES})`;+ return { ...decision, outcome: 'blocked', reason };+ }+ if (!wasInterrupted && cooldownRemaining > 0) {+ return { ...decision, outcome: 'cooldown', reason: `Orphan retry cooldown (${cooldownMinutes}m remaining)` };+ }+ return { ...decision, outcome: 'requeue', reason: null };+}++/**+ * The task update that persists a retry decision. One builder for the three+ * shapes so the `interruptedByRestart` handling is written once: the spread of+ * `task.metadata` carries every prior key forward, so any outcome other than a+ * restart-interrupted requeue must clear the flag or the task would read as+ * restart-interrupted through every later outcome.+ *+ * A restart-interrupted requeue records WHAT happened without touching the orphan+ * bookkeeping: `lastOrphanedAt` stays so a later genuine orphan's cooldown is+ * still measured from the last genuine orphan, not from our restart.+ *+ * @param {object} args+ * @param {number} args.now - epoch ms, the same instant the decision used+ * @param {object} [args.resumePatch] - merged last, requeue only+ * @returns {{ status: 'pending'|'blocked', metadata: object }}+ */+export function buildOrphanOutcome({ decision, task, agentId, now, resumePatch = {} }) {+ const { outcome, interrupted, retryCount, totalSpawns, reason, cooldownRemaining } = decision;+ const nowIso = new Date(now).toISOString();+ const restartInterrupted = outcome === 'requeue' && interrupted;+ const genuineOrphan = { orphanRetryCount: retryCount, lastOrphanedAt: nowIso, lastOrphanedAgentId: agentId };++ const outcomeFields = {+ requeue: restartInterrupted+ // The refunded count is persisted so the restart's spawn is genuinely given back.+ ? { totalSpawnCount: totalSpawns, lastInterruptedAt: nowIso, lastInterruptedAgentId: agentId }+ : genuineOrphan,+ cooldown: {+ ...genuineOrphan,+ blockedReason: reason,+ blockedCategory: 'orphan-cooldown',+ blockedAt: nowIso,+ cooldownUntil: new Date(now + cooldownRemaining).toISOString(),+ },+ blocked: { orphanRetryCount: retryCount, blockedReason: reason, blockedCategory: 'max-retries', blockedAt: nowIso },+ }[outcome];++ return {+ status: outcome === 'requeue' ? 'pending' : 'blocked',+ metadata: {+ ...task.metadata,+ ...outcomeFields,+ interruptedByRestart: restartInterrupted,+ ...(outcome === 'requeue' ? resumePatch : {}),+ },+ };+}diff --git a/server/services/agentOrphanRecovery.test.js b/server/services/agentOrphanRecovery.test.jsnew file mode 100644index 0000000000..8ad1f769f4--- /dev/null+++ b/server/services/agentOrphanRecovery.test.js@@ -0,0 +1,77 @@+import { describe, it, expect } from 'vitest';+import { MAX_TOTAL_SPAWNS } from '../lib/cosValidation.js';+import { MAX_ORPHAN_RETRIES, ORPHAN_GUARDS, ORPHAN_RETRY_COOLDOWN_MS, classifyOrphanGuard, decideOrphanRetry } from './agentOrphanRecovery.js';++const NOW = Date.parse('2026-06-01T12:00:00.000Z');+const MIN = 60 * 1000;+const decide = (metadata, wasInterrupted = false) =>+ decideOrphanRetry({ task: { metadata }, wasInterrupted, now: NOW });++// The #3202 budget contract: a genuine orphan pays a retry and arms the cooldown;+// a PortOS restart pays neither and gets its spawn refunded; the spawn ceiling+// bounds both.+describe('decideOrphanRetry', () => {+ it.each([+ ['first genuine orphan', { orphanRetryCount: 0, totalSpawnCount: 1 }, false, { outcome: 'requeue', retryCount: 1, totalSpawns: 1 }],+ ['last genuine retry still within budget', { orphanRetryCount: MAX_ORPHAN_RETRIES - 2, totalSpawnCount: 2 }, false, { outcome: 'requeue', retryCount: MAX_ORPHAN_RETRIES - 1, totalSpawns: 2 }],+ ['genuine orphan at the retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES - 1, totalSpawnCount: 3 }, false,+ { outcome: 'blocked', retryCount: MAX_ORPHAN_RETRIES, reason: `orphan retries exceeded (${MAX_ORPHAN_RETRIES}/${MAX_ORPHAN_RETRIES})` }],+ ['genuine orphan at the spawn ceiling', { orphanRetryCount: 0, totalSpawnCount: MAX_TOTAL_SPAWNS }, false,+ { outcome: 'blocked', reason: `total spawns exceeded (${MAX_TOTAL_SPAWNS}/${MAX_TOTAL_SPAWNS})` }],+ ['spawn ceiling is reported ahead of the retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES, totalSpawnCount: MAX_TOTAL_SPAWNS }, false,+ { outcome: 'blocked', reason: `total spawns exceeded (${MAX_TOTAL_SPAWNS}/${MAX_TOTAL_SPAWNS})` }],+ ['interrupted run is not charged and its spawn is refunded', { orphanRetryCount: 2, totalSpawnCount: 4 }, true,+ { outcome: 'requeue', retryCount: 2, totalSpawns: 3 }],+ ['interrupted run ignores the orphan retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES + 2, totalSpawnCount: 1 }, true,+ { outcome: 'requeue', retryCount: MAX_ORPHAN_RETRIES + 2, totalSpawns: 0 }],+ ['interrupted run is still bounded by the spawn ceiling after the refund', { orphanRetryCount: 0, totalSpawnCount: MAX_TOTAL_SPAWNS + 1 }, true,+ { outcome: 'blocked', retryCount: 0, totalSpawns: MAX_TOTAL_SPAWNS }],+ ['interrupted run never reads a negative spawn count', {}, true, { outcome: 'requeue', retryCount: 0, totalSpawns: 0 }],+ ])('%s', (_name, metadata, wasInterrupted, expected) => {+ expect(decide(metadata, wasInterrupted)).toMatchObject({ interrupted: wasInterrupted, ...expected });+ });++ describe('cooldown', () => {+ const recent = (minutesAgo) => new Date(NOW - minutesAgo * MIN).toISOString();++ it('holds a genuine orphan inside the window and reports the remaining minutes', () => {+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(10) })).toMatchObject({+ outcome: 'cooldown', cooldownRemaining: ORPHAN_RETRY_COOLDOWN_MS - 10 * MIN, cooldownMinutes: 20,+ reason: 'Orphan retry cooldown (20m remaining)',+ });+ });++ it('requeues once the window has elapsed', () => {+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(31) }).outcome).toBe('requeue');+ });++ it('does not cool down an interrupted run', () => {+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(1) }, true).outcome).toBe('requeue');+ });++ it('blocks rather than cools down when the budget is spent', () => {+ expect(decide({ orphanRetryCount: MAX_ORPHAN_RETRIES - 1, totalSpawnCount: 3, lastOrphanedAt: recent(1) }).outcome).toBe('blocked');+ });++ it('treats an unparseable lastOrphanedAt as never orphaned', () => {+ expect(decide({ orphanRetryCount: 0, lastOrphanedAt: 'not-a-date' })).toMatchObject({ outcome: 'requeue', cooldownRemaining: 0 });+ });+ });+});++describe('orphan guard order', () => {+ it('pins the precedence between the guards', () => {+ expect(ORPHAN_GUARDS.map(guard => guard.reason)).toEqual([+ 'terminal-block', 'private-security', 'already-handled', 'completed', 'content-refusal', 'deep-audit', 'retry-held',+ ]);+ });++ it('returns null for an ordinary orphan', () => {+ expect(classifyOrphanGuard({ status: 'in_progress', metadata: {} }, null)).toBeNull();+ });++ it('resolves a task matching several guards to the earliest one', () => {+ const task = { status: 'blocked', metadata: { blockedCategory: 'user-terminated', retryPendingCleanup: 'agent-dead' } };+ expect(classifyOrphanGuard(task, { executionInterruption: { category: 'content-refusal' } })).toBe('terminal-block');+ });+});