Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

refactor: orphan-task recovery decides retry budget and guard order in a pure module (#12945)

[._.]/ Adam Eivyaa850bd0b2c2
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index a1592545fe..d8bf7f6b0b 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -1,9 +1,9 @@
-import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';
import { hasClaimFlowContract } from '../lib/claimFlowTaskTypes.js';
import { isFalsyMeta, isTruthyMeta } from '../lib/metadataFlags.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
import { reconcileDeadOwnerReservations } from './deadOwnerReservations.js';
+import { MAX_ORPHAN_RETRIES, ORPHAN_GUARDS, buildOrphanOutcome, classifyOrphanGuard, decideOrphanRetry } from './agentOrphanRecovery.js';
/**
* Agent Management
*
@@ -54,11 +54,6 @@ import * as shellService from './shell.js';
const ROOT_DIR = PATHS.root;
-// Max retries before creating investigation task
-const MAX_ORPHAN_RETRIES = 3;
-// Minimum cooldown between orphan retries (30 minutes)
-const ORPHAN_RETRY_COOLDOWN_MS = 30 * 60 * 1000;
-
// Startup has two callers for this sweep: cos.start() awaits it before the
// first dequeue, while the spawner's delayed safety sweep runs shortly after
// its event wiring. Keep the whole sweep single-flight so both callers cannot
@@ -1490,292 +1485,205 @@ export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agent
// Without this, correctness depended on `cleanupOrphanedAgents` happening to
// run before `resetOrphanedTasks` in cos.js's boot sequence.
const wasInterrupted = interrupted ?? (agentMetadata?.interruptedBy === HOST_SHUTDOWN_REASON);
- // Consume the breadcrumb the moment it's honored, exactly as the marker is
- // consumed after the sweep. It is written once and would otherwise stay on the
- // agent record forever: `resetOrphanedTasks` keys on the most recent agent per
- // task, so a respawn that dies before creating its own agent record would keep
- // re-deriving `wasInterrupted` from this same stale breadcrumb — and, because
- // an interrupted run bypasses the cooldown, respawn on EVERY 15-minute sweep
- // instead of once per 30 minutes.
- if (wasInterrupted && agentMetadata?.interruptedBy === HOST_SHUTDOWN_REASON) {
- await updateAgent(agentId, { metadata: { interruptedBy: null } })
- .catch(err => emitLog('warn', `Could not clear interrupted breadcrumb on ${agentId}: ${err.message}`, { agentId }));
- }
- if (isPrivateSecurityTask({ metadata: agentMetadata })) {
- const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');
- await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });
- }
+ await consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted);
+ if (isPrivateSecurityTask({ metadata: agentMetadata })) await removePrivateSecurityScratch(agentId);
+
const task = await getTaskByIdFn(taskId).catch(() => null);
if (!task) {
emitLog('warn', `Could not find task ${taskId} for orphaned agent ${agentId}`, { taskId, agentId });
return;
}
- // User stops and policy refusals remain terminal across recovery sweeps.
- if (task.status === 'blocked' && ['user-terminated', 'content-refusal', 'content-filtered'].includes(task.metadata?.blockedCategory)) {
- emitLog('info', `⏭️ Skipping orphaned task ${taskId} — ${task.metadata.blockedCategory}`, { taskId, agentId });
- return;
- }
+ const ctx = { taskId, agentId, task, agentMetadata, agentStartedAt };
+ const guard = classifyOrphanGuard(task, agentMetadata);
+ if (guard) return ORPHAN_GUARD_HANDLERS[guard](ctx);
+ if (await completeIfCommitted(ctx)) return;
- // Lost in-memory source scope cannot validate a recovered assessment. Never
- // hand its transcript to an ordinary investigation or retry provider.
- if (isPrivateSecurityTask(task) || isPrivateSecurityTask({ metadata: agentMetadata })) {
- if (task.status === 'completed') return;
- const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');
- await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });
- const taskUpdate = await stampLiExecutionVerdict({ status: 'blocked', metadata: { ...task.metadata,
- blockedAt: new Date().toISOString(), blockedCategory: 'private-security-assessment-failed',
- blockedReason: 'Private assessment was interrupted. Inspect its local output and explicitly rerun it; no automatic investigation or provider fallback will run.',
- } }, task, { success: false });
- await updateTask(taskId, taskUpdate, task.taskType || 'user');
- return;
- }
+ const decision = decideOrphanRetry({ task, wasInterrupted, now: Date.now() });
+ await applyOrphanDecision(ctx, decision);
+}
- // If a prior orphan in this sweep already routed the task through this handler
- // (max-retries → investigation task created, or orphan-cooldown → blocked until later),
- // skip — otherwise each additional orphaned agent for the same task spawns a
- // duplicate investigation task and inflates orphanRetryCount past its ceiling.
- if (task.status === 'blocked' &&
- (task.metadata?.blockedCategory === 'max-retries' ||
- task.metadata?.blockedCategory === 'orphan-cooldown')) {
- emitLog('info', `⏭️ Skipping orphaned task ${taskId} — already handled (${task.metadata.blockedCategory})`, {
- taskId, agentId, blockedCategory: task.metadata.blockedCategory
- });
- return;
- }
+/**
+ * Consume the interrupted breadcrumb the moment it's honored, exactly as the
+ * marker is consumed after the sweep. It is written once and would otherwise stay
+ * on the agent record forever: `resetOrphanedTasks` keys on the most recent agent
+ * per task, so a respawn that dies before creating its own agent record would keep
+ * re-deriving `wasInterrupted` from this same stale breadcrumb — and, because an
+ * interrupted run bypasses the cooldown, respawn on EVERY 15-minute sweep instead
+ * of once per 30 minutes.
+ */
+async function consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted) {
+ if (!wasInterrupted || agentMetadata?.interruptedBy !== HOST_SHUTDOWN_REASON) return;
+ await updateAgent(agentId, { metadata: { interruptedBy: null } })
+ .catch(err => emitLog('warn', `Could not clear interrupted breadcrumb on ${agentId}: ${err.message}`, { agentId }));
+}
- // Skip tasks already completed
- if (task.status === 'completed') {
- emitLog('debug', `⏭️ Skipping orphaned task ${taskId} — already completed`, { taskId, agentId });
- return;
- }
+async function removePrivateSecurityScratch(agentId) {
+ const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');
+ await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });
+}
- // A crash/restart after the refusal was saved must not authorize a retry or
- // commit rescue. Do not let an older refused run block a newer invocation.
- const interruption = agentMetadata?.executionInterruption;
- if (['content-refusal', 'content-filtered'].includes(interruption?.category)) {
+function skipTerminalBlockedOrphan({ taskId, agentId, task }) {
+ emitLog('info', `⏭️ Skipping orphaned task ${taskId} — ${task.metadata.blockedCategory}`, { taskId, agentId });
+}
+
+function skipHandledOrphan({ taskId, agentId, task }) {
+ const { blockedCategory } = task.metadata;
+ emitLog('info', `⏭️ Skipping orphaned task ${taskId} — already handled (${blockedCategory})`, { taskId, agentId, blockedCategory });
+}
+
+function skipCompletedOrphan({ taskId, agentId }) {
+ emitLog('debug', `⏭️ Skipping orphaned task ${taskId} — already completed`, { taskId, agentId });
+}
+
+async function recoverPrivateSecurityOrphan({ taskId, agentId, task }) {
+ if (task.status === 'completed') return;
+ await removePrivateSecurityScratch(agentId);
+ const taskUpdate = await stampLiExecutionVerdict({ status: 'blocked', metadata: { ...task.metadata,
+ blockedAt: new Date().toISOString(), blockedCategory: 'private-security-assessment-failed',
+ blockedReason: 'Private assessment was interrupted. Inspect its local output and explicitly rerun it; no automatic investigation or provider fallback will run.',
+ } }, task, { success: false });
+ await updateTask(taskId, taskUpdate, task.taskType || 'user');
+}
+
+async function recoverRefusedOrphan({ taskId, task, agentMetadata, agentStartedAt }) {
+ const interruption = agentMetadata.executionInterruption;
+ // Do not let an older refused run block a newer invocation.
+ const startedAt = toEpochMs(agentStartedAt);
+ if (startedAt && toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]) > startedAt) return;
+ await updateTask(taskId, await stampLiExecutionVerdict({ status: 'blocked', metadata: {
+ ...task.metadata, ...clearedRetryHoldMetadata(), executionInterruption: interruption,
+ blockedAt: new Date().toISOString(), blockedCategory: interruption.category,
+ blockedReason: 'Provider safety refusal; automatic continuation, retries and provider fallback are disabled.',
+ } }, task, { success: false }), task.taskType || 'user');
+}
+
+async function recoverDeepAuditOrphan({ taskId, agentId, task, agentMetadata, agentStartedAt }) {
+ if (task.status !== 'in_progress') return;
+ const held = isRetryHeld(task.metadata);
+ if (held && task.metadata[RETRY_HOLD_KEY] !== agentId) return;
+ if (!held) {
+ const spawnedAt = toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]);
const startedAt = toEpochMs(agentStartedAt);
- if (startedAt && toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]) > startedAt) return;
- await updateTask(taskId, await stampLiExecutionVerdict({ status: 'blocked', metadata: {
- ...task.metadata, ...clearedRetryHoldMetadata(), executionInterruption: interruption,
- blockedAt: new Date().toISOString(), blockedCategory: interruption.category,
- blockedReason: 'Provider safety refusal; automatic continuation, retries and provider fallback are disabled.',
- } }, task, { success: false }), task.taskType || 'user');
- return;
+ // A legacy/unknown owner cannot authorize parking a newer invocation.
+ if (!spawnedAt || !startedAt || spawnedAt > startedAt) return;
+ const agents = await getAgents();
+ if (!Array.isArray(agents) || agents.some(entry => entry.taskId === taskId && entry.id !== agentId
+ && (entry.status === 'running' || toEpochMs(entry.startedAt) > startedAt))) return;
}
+ const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata });
+ await updateTask(taskId, { status: 'blocked', metadata: {
+ ...resumePatch, ...clearedRetryHoldMetadata(), blockedCategory: 'deep-audit-partial',
+ blockedReason: 'Interrupted Deep audit; inspect checkpoint and resume explicitly', blockedAt: new Date().toISOString(),
+ } }, task.taskType || 'user', { expectedStatus: 'in_progress',
+ expectedMetadata: held ? { [RETRY_HOLD_KEY]: agentId } : { [LAST_SPAWNED_AT_KEY]: task.metadata[LAST_SPAWNED_AT_KEY] },
+ });
+}
- // Recovery of a dead Deep invocation is a checkpoint, never permission to
- // auto-launch another provider call or to treat commits as completed coverage.
- if (isLegacyDeepAudit(task.metadata)) {
- if (task.status !== 'in_progress') return;
- const held = isRetryHeld(task.metadata);
- if (held && task.metadata[RETRY_HOLD_KEY] !== agentId) return;
- if (!held) {
- const spawnedAt = toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]);
- const startedAt = toEpochMs(agentStartedAt);
- // A legacy/unknown owner cannot authorize parking a newer invocation.
- if (!spawnedAt || !startedAt || spawnedAt > startedAt) return;
- const agents = await getAgents();
- if (!Array.isArray(agents) || agents.some(entry => entry.taskId === taskId && entry.id !== agentId
- && (entry.status === 'running' || toEpochMs(entry.startedAt) > startedAt))) return;
- }
- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata });
- await updateTask(taskId, { status: 'blocked', metadata: {
- ...resumePatch, ...clearedRetryHoldMetadata(), blockedCategory: 'deep-audit-partial',
- blockedReason: 'Interrupted Deep audit; inspect checkpoint and resume explicitly', blockedAt: new Date().toISOString(),
- } }, task.taskType || 'user', { expectedStatus: 'in_progress',
- expectedMetadata: held ? { [RETRY_HOLD_KEY]: agentId } : { [LAST_SPAWNED_AT_KEY]: task.metadata[LAST_SPAWNED_AT_KEY] },
- });
+/**
+ * The run's verdict is already persisted — it failed, and was budgeted a retry —
+ * so this is not a fresh orphan: finish the transition the dead process started
+ * (resolve the pointer, flip to `pending`, drop the marker) rather than charging
+ * it orphan budget for a decision already made. Precedes the commit check: a failed
+ * run that committed is exactly the case the resume pointer exists for, and
+ * completing the task on that evidence would discard the retry finalizeAgent granted.
+ */
+async function recoverRetryHeldOrphan({ taskId, agentId, task, agentMetadata }) {
+ const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
+ emitLog('warn', `Resume pointer for held task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });
+ return {};
+ });
+ const targetBranch = resolveTaskTargetBranch(resumePatch);
+ const result = await updateTask(taskId, {
+ status: 'pending',
+ metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }
+ }, task.taskType || 'user', {
+ expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },
+ });
+ if (result?.error) {
+ emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });
return;
}
+ if (result?.statusChanged) return;
+ emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {
+ taskId, agentId, branchName: targetBranch
+ });
+}
- // A retry hold whose process died mid-transition (#3373). The run's verdict is
- // already persisted — it failed, and was budgeted a retry — so this is not a
- // fresh orphan: finish the transition the dead process started (resolve the
- // pointer, flip to `pending`, drop the marker) rather than charging it orphan
- // budget for a decision that was already made. Runs BEFORE the commit check
- // below: a failed run that committed is exactly the case the resume pointer
- // exists for, and completing the task on that evidence would discard the retry
- // finalizeAgent already granted.
- if (isRetryHeld(task.metadata)) {
- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
- emitLog('warn', `Resume pointer for held task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });
- return {};
- });
- const targetBranch = resolveTaskTargetBranch(resumePatch);
- const result = await updateTask(taskId, {
- status: 'pending',
- metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }
- }, task.taskType || 'user', {
- expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },
- });
- if (result?.error) {
- emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });
- return;
- }
- if (result?.statusChanged) return;
- emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {
- taskId, agentId, branchName: targetBranch
- });
- return;
- }
+// One handler per `ORPHAN_GUARDS` reason; the precedence lives in that list.
+const ORPHAN_GUARD_HANDLERS = {
+ 'terminal-block': skipTerminalBlockedOrphan,
+ 'private-security': recoverPrivateSecurityOrphan,
+ 'already-handled': skipHandledOrphan,
+ 'completed': skipCompletedOrphan,
+ 'content-refusal': recoverRefusedOrphan,
+ 'deep-audit': recoverDeepAuditOrphan,
+ 'retry-held': recoverRetryHeldOrphan,
+};
+for (const { reason } of ORPHAN_GUARDS) {
+ if (!ORPHAN_GUARD_HANDLERS[reason]) throw new Error(`No orphan recovery handler for guard "${reason}"`);
+}
- // Check if the agent actually committed work before treating as orphaned.
- // Scoped to the dead run's OWN window (#3637): the retired task-id commit marker
- // was never emitted by anything, and an unbounded `git log` would credit this
- // task with any commit in the repo — including another agent's.
- // Both shapes the agent record can carry — the persisted ISO string and a raw
- // epoch-ms number (see toEpochMs for why a bare Date.parse drops the latter).
- const orphanRunStartedAt = toEpochMs(agentStartedAt);
- const commitFound = Number.isFinite(orphanRunStartedAt)
- && await committedDuringRun(agentMetadata?.workspacePath || ROOT_DIR, orphanRunStartedAt);
- if (commitFound) {
- emitLog('info', `✅ Orphaned agent ${agentId} actually completed work - commit found for task ${taskId}`, { taskId, agentId });
- const taskUpdate = await stampLiExecutionVerdict({ status: 'completed' }, task, { success: true });
- await updateTask(taskId, taskUpdate, task.taskType || 'user');
- return;
- }
+/**
+ * Did the agent actually commit work before it died? Scoped to the dead run's OWN
+ * window (#3637): the retired task-id commit marker was never emitted by anything,
+ * and an unbounded `git log` would credit this task with any commit in the repo —
+ * including another agent's. `toEpochMs` accepts both shapes the agent record can
+ * carry (the persisted ISO string and a raw epoch-ms number); without a start time
+ * the probe is skipped.
+ */
+async function completeIfCommitted({ taskId, agentId, task, agentMetadata, agentStartedAt }) {
+ const runStartedAt = toEpochMs(agentStartedAt);
+ const commitFound = Number.isFinite(runStartedAt)
+ && await committedDuringRun(agentMetadata?.workspacePath || ROOT_DIR, runStartedAt);
+ if (!commitFound) return false;
+ emitLog('info', `✅ Orphaned agent ${agentId} actually completed work - commit found for task ${taskId}`, { taskId, agentId });
+ const taskUpdate = await stampLiExecutionVerdict({ status: 'completed' }, task, { success: true });
+ await updateTask(taskId, taskUpdate, task.taskType || 'user');
+ return true;
+}
- // Get current retry count from task metadata.
- //
- // Retry budget and the cooldown exist to stop a task that keeps KILLING ITS OWN
- // agent from spawning forever. A PortOS host restart is not that: the agent was
- // healthy and PortOS took it down (#3202). Charging it a retry — and arming the
- // 30-minute cooldown — punishes the task for our maintenance, and three routine
- // restarts were enough to exhaust the budget and block the task outright. So an
- // interrupted run costs no retry and arms no cooldown; it just requeues with a
- // resume pointer. `MAX_TOTAL_SPAWNS` still applies, so nothing can spawn
- // unboundedly even under a restart loop.
- const priorOrphanRetries = Number(task.metadata?.orphanRetryCount) || 0;
- const retryCount = wasInterrupted ? priorOrphanRetries : priorOrphanRetries + 1;
- const taskType = task.taskType || 'user';
-
- // `totalSpawnCount` is incremented when a task goes `in_progress`, so the run a
- // restart destroyed already consumed a spawn. REFUND it: that spawn produced no
- // work and was ended by PortOS, not by the task. Without the refund the fix is
- // only half-done — the ceiling moves from 3 restarts to MAX_TOTAL_SPAWNS, after
- // which the task is still blocked `max-retries` AND an "[Auto-Fix] Investigate
- // repeated agent orphaning" task is filed blaming an agent that never failed.
- // The refund keeps a real ceiling on genuine failures (each still costs a
- // spawn) while making a restart cost nothing.
- const priorTotalSpawns = Number(task.metadata?.totalSpawnCount) || 0;
- const totalSpawns = wasInterrupted ? Math.max(0, priorTotalSpawns - 1) : priorTotalSpawns;
-
- // Block if total spawn count across all retry types is exhausted
- const totalExceeded = totalSpawns >= MAX_TOTAL_SPAWNS;
-
- // Enforce cooldown between orphan retries
- const lastOrphanedAt = task.metadata?.lastOrphanedAt ? new Date(task.metadata.lastOrphanedAt).getTime() : 0;
- const cooldownRemaining = lastOrphanedAt ? ORPHAN_RETRY_COOLDOWN_MS - (Date.now() - lastOrphanedAt) : 0;
- const inCooldown = !wasInterrupted && cooldownRemaining > 0;
-
- // An interrupted run is exempt from the orphan ceiling for the same reason it
- // is exempt from the counter: it isn't one of the failures the ceiling counts.
- // `totalExceeded` still gates it, so a restart loop can't spawn without bound.
- const withinOrphanBudget = wasInterrupted || retryCount < MAX_ORPHAN_RETRIES;
-
- if (withinOrphanBudget && !totalExceeded && !inCooldown) {
- emitLog('info', wasInterrupted
+function logOrphanDecision(taskId, decision) {
+ const { outcome, interrupted, retryCount, totalSpawns, cooldownMinutes, reason } = decision;
+ if (outcome === 'requeue') {
+ emitLog('info', interrupted
? `Resuming task ${taskId} interrupted by a PortOS restart (no retry charged; total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`
: `Resetting orphaned task ${taskId} for retry (attempt ${retryCount}/${MAX_ORPHAN_RETRIES}, total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`, {
- taskId,
- retryCount,
- totalSpawns,
- interrupted: wasInterrupted,
- maxRetries: MAX_ORPHAN_RETRIES
+ taskId, retryCount, totalSpawns, interrupted, maxRetries: MAX_ORPHAN_RETRIES
});
+ } else if (outcome === 'cooldown') {
+ emitLog('info', `⏳ Orphan retry for task ${taskId} in cooldown (${cooldownMinutes}m remaining)`, { taskId, cooldownMinutes, retryCount });
+ } else {
+ emitLog('warn', `Task ${taskId} blocked: ${reason}, creating investigation task`, { taskId, retryCount, totalSpawns });
+ }
+}
- // Point the retry at whatever the dead run left behind — its branch, or the
- // whole worktree when the process died mid-edit and the uncommitted work is
- // still sitting in it. Folded into THIS write rather than stamped separately
- // beforehand: the flip to `pending` emits `tasks:changed`, which can spawn the
- // retry immediately, so a pointer written afterwards could land too late.
- // Fails open to "start clean": requeueing the task matters far more than
- // resuming it, and a throw here would strand it in_progress until the next sweep.
- const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
+/** Persist the retry decision: requeue, orphan-cooldown block, or max-retries block + investigation. */
+async function applyOrphanDecision({ taskId, agentId, task, agentMetadata }, decision) {
+ logOrphanDecision(taskId, decision);
+ const requeue = decision.outcome === 'requeue';
+ // Point the retry at whatever the dead run left behind — its branch, or the
+ // whole worktree when the process died mid-edit and the uncommitted work is
+ // still sitting in it. Folded into THIS write rather than stamped separately
+ // beforehand: the flip to `pending` emits `tasks:changed`, which can spawn the
+ // retry immediately, so a pointer written afterwards could land too late.
+ // Fails open to "start clean": requeueing the task matters far more than
+ // resuming it, and a throw here would strand it in_progress until the next sweep.
+ const resumePatch = requeue
+ ? await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
emitLog('warn', `Resume pointer for task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });
return {};
- });
-
- // An interrupted run records WHAT happened without touching the orphan
- // bookkeeping: `lastOrphanedAt` stays as-is so a later genuine orphan's
- // cooldown is still measured from the last genuine orphan, not from our
- // restart. The genuine-orphan branch clears `interruptedByRestart` — the
- // metadata spread below carries every prior key forward, so leaving it set
- // would make a task that was once restarted look permanently restart-
- // interrupted, through this orphan and every later one.
- const now = new Date().toISOString();
- const recoveryMetadata = wasInterrupted
- ? {
- // Persist the refunded count (see the totalSpawns note above) so the
- // restart's spawn is genuinely given back rather than just ignored once.
- totalSpawnCount: totalSpawns,
- lastInterruptedAt: now,
- lastInterruptedAgentId: agentId,
- interruptedByRestart: true,
- }
- : {
- orphanRetryCount: retryCount,
- lastOrphanedAt: now,
- lastOrphanedAgentId: agentId,
- interruptedByRestart: false,
- };
-
- await updateTask(taskId, {
- status: 'pending',
- metadata: {
- ...task.metadata,
- ...recoveryMetadata,
- ...resumePatch
- }
- }, taskType);
- } else if (inCooldown && retryCount < MAX_ORPHAN_RETRIES && !totalExceeded) {
- const cooldownMinutes = Math.ceil(cooldownRemaining / 60000);
- emitLog('info', `⏳ Orphan retry for task ${taskId} in cooldown (${cooldownMinutes}m remaining)`, {
- taskId, cooldownMinutes, retryCount
- });
-
- await updateTask(taskId, await stampLiExecutionVerdict({
- status: 'blocked',
- metadata: {
- ...task.metadata,
- orphanRetryCount: retryCount,
- lastOrphanedAt: new Date().toISOString(),
- lastOrphanedAgentId: agentId,
- // Cleared for the same reason the requeue branch clears it: the metadata
- // spread carries every prior key forward, so a task interrupted once
- // would read as restart-interrupted through every later outcome.
- interruptedByRestart: false,
- blockedReason: `Orphan retry cooldown (${cooldownMinutes}m remaining)`,
- blockedCategory: 'orphan-cooldown',
- blockedAt: new Date().toISOString(),
- cooldownUntil: new Date(Date.now() + cooldownRemaining).toISOString()
- }
- }, task, { success: false }), taskType);
- } else {
- const reason = totalExceeded
- ? `total spawns exceeded (${totalSpawns}/${MAX_TOTAL_SPAWNS})`
- : `orphan retries exceeded (${retryCount}/${MAX_ORPHAN_RETRIES})`;
- emitLog('warn', `Task ${taskId} blocked: ${reason}, creating investigation task`, {
- taskId,
- retryCount,
- totalSpawns
- });
-
- await updateTask(taskId, await stampLiExecutionVerdict({
- status: 'blocked',
- metadata: {
- ...task.metadata,
- orphanRetryCount: retryCount,
- // See the requeue branch: the metadata spread carries prior keys forward.
- interruptedByRestart: false,
- blockedReason: reason,
- blockedCategory: 'max-retries',
- blockedAt: new Date().toISOString()
- }
- }, task, { success: false }), taskType);
+ })
+ : {};
+ const outcome = buildOrphanOutcome({ decision, task, agentId, now: Date.now(), resumePatch });
+ const taskUpdate = requeue ? outcome : await stampLiExecutionVerdict(outcome, task, { success: false });
+ await updateTask(taskId, taskUpdate, task.taskType || 'user');
+ if (decision.outcome === 'blocked') await fileOrphanInvestigation({ taskId, agentId, task, decision });
+}
- const description = `[Auto-Fix] Investigate repeated agent orphaning for task ${taskId}
+async function fileOrphanInvestigation({ taskId, agentId, task, decision }) {
+ const { retryCount, totalSpawns, reason } = decision;
+ const description = `[Auto-Fix] Investigate repeated agent orphaning for task ${taskId}
**Original Task**: ${(task.description || '').substring(0, 200)}
**Orphan Retries**: ${retryCount}
@@ -1791,20 +1699,19 @@ This task has been blocked after ${totalSpawns} total agent spawns. Investigate:
Once the issue is resolved, task \`${taskId}\` is retried automatically — completing this task revives it.`;
- // Filed through the shared producer so this participates in the same
- // loop-aware approval policy, storm counter, and fingerprint dedup as every
- // other investigation (#3714), and so `affectedTasks` closes the loop: the
- // orphaned task comes back on its own when this one completes. Hardcoding
- // `approvalRequired: true` here is what turned one repeatedly-orphaned task
- // into a queue of approvals nobody asked for.
- await fileInvestigationTask({
- description,
- priority: 'HIGH',
- context: `Auto-generated from repeated orphan failures for task ${taskId}`,
- fingerprint: buildInvestigationFingerprint(task, { category: 'max-retries' }),
- affectedTasks: [taskId]
- }).catch(err => {
- emitLog('error', `Failed to create investigation task: ${err.message}`, { taskId, error: err.message });
- });
- }
+ // Filed through the shared producer so this participates in the same
+ // loop-aware approval policy, storm counter, and fingerprint dedup as every
+ // other investigation (#3714), and so `affectedTasks` closes the loop: the
+ // orphaned task comes back on its own when this one completes. Hardcoding
+ // `approvalRequired: true` here is what turned one repeatedly-orphaned task
+ // into a queue of approvals nobody asked for.
+ await fileInvestigationTask({
+ description,
+ priority: 'HIGH',
+ context: `Auto-generated from repeated orphan failures for task ${taskId}`,
+ fingerprint: buildInvestigationFingerprint(task, { category: 'max-retries' }),
+ affectedTasks: [taskId]
+ }).catch(err => {
+ emitLog('error', `Failed to create investigation task: ${err.message}`, { taskId, error: err.message });
+ });
}
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index 420cda8554..f667f48ccc 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -759,6 +759,141 @@ describe('handleOrphanedTask — duplicate-investigation guard', () => {
});
});
+// Pins the exact metadata handleOrphanedTask persists for each recovery outcome and
+// the precedence between its guards (#12945). The retry budget is a documented
+// contract (#3202 refund rules): a drift here retries a task forever or blocks it early.
+describe('handleOrphanedTask — persisted outcome shapes and guard precedence (#12945)', () => {
+ const NOW = Date.parse('2026-06-01T12:00:00.000Z');
+ const iso = (ms) => new Date(ms).toISOString();
+ const MIN = 60 * 1000;
+ const taskWith = (metadata, status = 'in_progress') =>
+ ({ id: 'task-foo', status, taskType: 'user', description: 'Original work', metadata });
+ const run = (task, options) => handleOrphanedTask('task-foo', 'agent-dead', vi.fn().mockResolvedValue(task), options);
+ const persisted = () => updateTask.mock.calls[0];
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(NOW);
+ getAllTasks.mockResolvedValue({ user: { tasks: [] }, cos: { tasks: [] } });
+ });
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('requeues a genuine orphan: charges a retry, stamps the orphan breadcrumb, clears interruptedByRestart', async () => {
+ resolveTaskResumePatch.mockResolvedValueOnce({ resumeBranch: 'branch-a' });
+ await run(taskWith({ keep: 'x', orphanRetryCount: 1, totalSpawnCount: 2, interruptedByRestart: true }));
+
+ expect(persisted()).toEqual(['task-foo', {
+ status: 'pending',
+ metadata: {
+ keep: 'x', orphanRetryCount: 2, totalSpawnCount: 2, interruptedByRestart: false,
+ lastOrphanedAt: iso(NOW), lastOrphanedAgentId: 'agent-dead',
+ resumeBranch: 'branch-a',
+ },
+ }, 'user']);
+ });
+
+ it('requeues a restart-interrupted run without charging a retry, refunding the spawn and leaving lastOrphanedAt alone', async () => {
+ await run(taskWith({ orphanRetryCount: 2, totalSpawnCount: 3, lastOrphanedAt: iso(NOW - MIN) }), { interrupted: true });
+
+ expect(persisted()).toEqual(['task-foo', {
+ status: 'pending',
+ metadata: {
+ orphanRetryCount: 2, totalSpawnCount: 2, lastOrphanedAt: iso(NOW - MIN),
+ lastInterruptedAt: iso(NOW), lastInterruptedAgentId: 'agent-dead', interruptedByRestart: true,
+ },
+ }, 'user']);
+ });
+
+ it('blocks a second genuine orphan inside the cooldown, with the remaining time and a cleared interruptedByRestart', async () => {
+ await run(taskWith({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: iso(NOW - 10 * MIN), interruptedByRestart: true }));
+
+ expect(persisted()).toEqual(['task-foo', {
+ status: 'blocked',
+ metadata: {
+ orphanRetryCount: 2, totalSpawnCount: 2, lastOrphanedAt: iso(NOW), interruptedByRestart: false,
+ lastOrphanedAgentId: 'agent-dead',
+ blockedReason: 'Orphan retry cooldown (20m remaining)', blockedCategory: 'orphan-cooldown',
+ blockedAt: iso(NOW), cooldownUntil: iso(NOW + 20 * MIN),
+ },
+ }, 'user']);
+ expect(addTask).not.toHaveBeenCalled();
+ });
+
+ it('blocks as max-retries and files an investigation once the orphan retries are spent', async () => {
+ await run(taskWith({ orphanRetryCount: 2, totalSpawnCount: 3, interruptedByRestart: true }));
+
+ expect(persisted()).toEqual(['task-foo', {
+ status: 'blocked',
+ metadata: {
+ orphanRetryCount: 3, totalSpawnCount: 3, interruptedByRestart: false,
+ blockedReason: 'orphan retries exceeded (3/3)', blockedCategory: 'max-retries', blockedAt: iso(NOW),
+ },
+ }, 'user']);
+ expect(addTask).toHaveBeenCalledTimes(1);
+ expect(addTask.mock.calls[0][0].description).toContain('**Orphan Retries**: 3\n**Total Spawns**: 3');
+ });
+
+ it('blocks as max-retries on the spawn ceiling even for a restart-interrupted run, without charging a retry', async () => {
+ // 6 spawns minus the refunded one is still at the MAX_TOTAL_SPAWNS ceiling of 5.
+ await run(taskWith({ orphanRetryCount: 0, totalSpawnCount: 6 }), { interrupted: true });
+
+ expect(persisted()).toEqual(['task-foo', {
+ status: 'blocked',
+ metadata: {
+ orphanRetryCount: 0, totalSpawnCount: 6, interruptedByRestart: false,
+ blockedReason: 'total spawns exceeded (5/5)', blockedCategory: 'max-retries', blockedAt: iso(NOW),
+ },
+ }, 'user']);
+ expect(addTask).toHaveBeenCalledTimes(1);
+ });
+
+ describe('guard precedence', () => {
+ const refusal = { executionInterruption: { category: 'content-refusal' } };
+
+ it('terminal user-terminated outranks the private-security failure write', async () => {
+ await run(taskWith({ analysisType: 'private-security-assessment', blockedCategory: 'user-terminated' }, 'blocked'));
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
+ it('the private-security failure write outranks the already-handled skip', async () => {
+ await run(taskWith({ analysisType: 'private-security-assessment', blockedCategory: 'max-retries' }, 'blocked'));
+ expect(updateTask).toHaveBeenCalledTimes(1);
+ expect(updateTask.mock.calls[0][1].metadata.blockedCategory).toBe('private-security-assessment-failed');
+ });
+
+ it('a completed private-security task is left alone', async () => {
+ await run(taskWith({ analysisType: 'private-security-assessment' }, 'completed'));
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
+ it('the already-handled skip outranks the content-refusal write', async () => {
+ await run(taskWith({ blockedCategory: 'orphan-cooldown' }, 'blocked'), { agentMetadata: refusal });
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
+ it('the completed skip outranks the content-refusal write', async () => {
+ await run(taskWith({}, 'completed'), { agentMetadata: refusal });
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
+ it('the content-refusal write outranks the retry-hold transition', async () => {
+ await run(taskWith({ retryPendingCleanup: 'agent-dead' }), { agentMetadata: refusal });
+ expect(updateTask).toHaveBeenCalledTimes(1);
+ expect(updateTask.mock.calls[0][1].metadata.blockedCategory).toBe('content-refusal');
+ });
+
+ it('a retry-held task finishes its transition instead of being completed by a commit found in the run window', async () => {
+ committedDuringRun.mockResolvedValueOnce(true);
+ await run(taskWith({ retryPendingCleanup: 'agent-dead' }), { agentStartedAt: iso(NOW - MIN) });
+ expect(committedDuringRun).not.toHaveBeenCalled();
+ expect(updateTask.mock.calls[0][1].status).toBe('pending');
+ });
+ });
+});
+
describe('pauseAgent', () => {
beforeEach(() => {
vi.clearAllMocks();
diff --git a/server/services/agentOrphanRecovery.js b/server/services/agentOrphanRecovery.js
new file mode 100644
index 0000000000..a190231961
--- /dev/null
+++ b/server/services/agentOrphanRecovery.js
@@ -0,0 +1,148 @@
+/**
+ * Orphan recovery decisions — the pure half of `handleOrphanedTask` (#12945).
+ *
+ * Everything here is a function of its arguments (no I/O, no clock reads), so the
+ * guard precedence and the retry-budget contract can be pinned without a task
+ * store. `agentManagement.js` owns the reads and writes around these decisions.
+ */
+
+import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';
+import { MAX_TOTAL_SPAWNS } from '../lib/cosValidation.js';
+import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';
+import { isRetryHeld } from '../lib/taskRetryHold.js';
+
+// Max genuine orphan retries before the task is blocked and an investigation filed.
+export const MAX_ORPHAN_RETRIES = 3;
+// Minimum gap between genuine orphan retries (30 minutes).
+export const ORPHAN_RETRY_COOLDOWN_MS = 30 * 60 * 1000;
+
+const REFUSAL_CATEGORIES = ['content-refusal', 'content-filtered'];
+const TERMINAL_BLOCK_CATEGORIES = ['user-terminated', ...REFUSAL_CATEGORIES];
+// A prior orphan in the same sweep already routed the task through recovery.
+const ALREADY_HANDLED_CATEGORIES = ['max-retries', 'orphan-cooldown'];
+
+const isBlockedWith = (task, categories) =>
+ task.status === 'blocked' && categories.includes(task.metadata?.blockedCategory);
+
+/**
+ * Guards checked before any retry accounting, FIRST MATCH WINS — the array order
+ * is the precedence. Each reason has a handler in `agentManagement.js`; some only
+ * skip, others persist a terminal state (private-security, content-refusal,
+ * deep-audit, retry-held). The interleaving is deliberate: a private-security task
+ * is failed even when already `max-retries`-blocked, but a user-terminated block
+ * outranks it, and `completed` outranks every write below it.
+ */
+export const ORPHAN_GUARDS = [
+ // User stops and policy refusals remain terminal across recovery sweeps.
+ { reason: 'terminal-block', matches: (task) => isBlockedWith(task, TERMINAL_BLOCK_CATEGORIES) },
+ // Lost in-memory source scope cannot validate a recovered assessment.
+ { reason: 'private-security', matches: (task, agentMetadata) => isPrivateSecurityTask(task) || isPrivateSecurityTask({ metadata: agentMetadata }) },
+ // Otherwise each extra orphaned agent for the same task files a duplicate investigation.
+ { reason: 'already-handled', matches: (task) => isBlockedWith(task, ALREADY_HANDLED_CATEGORIES) },
+ { reason: 'completed', matches: (task) => task.status === 'completed' },
+ // A crash after the refusal was saved must not authorize a retry or commit rescue.
+ { reason: 'content-refusal', matches: (_task, agentMetadata) => REFUSAL_CATEGORIES.includes(agentMetadata?.executionInterruption?.category) },
+ // A dead Deep invocation is a checkpoint, never permission to launch another provider call.
+ { reason: 'deep-audit', matches: (task) => isLegacyDeepAudit(task.metadata) },
+ // A retry whose process died mid-transition (#3373): finish it, don't charge orphan budget.
+ { reason: 'retry-held', matches: (task) => isRetryHeld(task.metadata) },
+];
+
+/** The reason of the first guard that claims this orphan, or null when it needs a retry decision. */
+export function classifyOrphanGuard(task, agentMetadata) {
+ return ORPHAN_GUARDS.find(guard => guard.matches(task, agentMetadata))?.reason ?? null;
+}
+
+const counter = (value) => Number(value) || 0;
+
+/** Ms left before the next genuine orphan retry may run; 0 when none was recorded (or it is unparseable). */
+function remainingCooldownMs(lastOrphanedAt, now) {
+ const lastMs = lastOrphanedAt ? new Date(lastOrphanedAt).getTime() : 0;
+ return lastMs ? ORPHAN_RETRY_COOLDOWN_MS - (now - lastMs) : 0;
+}
+
+/**
+ * The retry-budget decision. A genuine orphan costs one retry and arms the
+ * cooldown; a run PortOS itself interrupted (host restart) costs neither and has
+ * its spawn refunded (#3202), because the task did not kill its own agent.
+ * `MAX_TOTAL_SPAWNS` still bounds both, so a restart loop cannot spawn without limit.
+ *
+ * @returns {{ outcome: 'requeue'|'cooldown'|'blocked', interrupted: boolean,
+ * retryCount: number, totalSpawns: number, cooldownRemaining: number,
+ * cooldownMinutes: number, reason: string|null }}
+ * `reason` is the blockedReason for `cooldown` and `blocked`, null for `requeue`.
+ */
+export function decideOrphanRetry({ task, wasInterrupted, now }) {
+ const priorRetries = counter(task.metadata?.orphanRetryCount);
+ const retryCount = wasInterrupted ? priorRetries : priorRetries + 1;
+ const priorSpawns = counter(task.metadata?.totalSpawnCount);
+ const totalSpawns = wasInterrupted ? Math.max(0, priorSpawns - 1) : priorSpawns;
+
+ const cooldownRemaining = remainingCooldownMs(task.metadata?.lastOrphanedAt, now);
+ const cooldownMinutes = Math.ceil(cooldownRemaining / 60000);
+
+ const totalExceeded = totalSpawns >= MAX_TOTAL_SPAWNS;
+ // An interrupted run is exempt from the orphan ceiling for the same reason it is
+ // exempt from the counter; `totalExceeded` still gates it.
+ const withinOrphanBudget = wasInterrupted || retryCount < MAX_ORPHAN_RETRIES;
+ const decision = { interrupted: wasInterrupted, retryCount, totalSpawns, cooldownRemaining, cooldownMinutes };
+
+ if (totalExceeded || !withinOrphanBudget) {
+ const reason = totalExceeded
+ ? `total spawns exceeded (${totalSpawns}/${MAX_TOTAL_SPAWNS})`
+ : `orphan retries exceeded (${retryCount}/${MAX_ORPHAN_RETRIES})`;
+ return { ...decision, outcome: 'blocked', reason };
+ }
+ if (!wasInterrupted && cooldownRemaining > 0) {
+ return { ...decision, outcome: 'cooldown', reason: `Orphan retry cooldown (${cooldownMinutes}m remaining)` };
+ }
+ return { ...decision, outcome: 'requeue', reason: null };
+}
+
+/**
+ * The task update that persists a retry decision. One builder for the three
+ * shapes so the `interruptedByRestart` handling is written once: the spread of
+ * `task.metadata` carries every prior key forward, so any outcome other than a
+ * restart-interrupted requeue must clear the flag or the task would read as
+ * restart-interrupted through every later outcome.
+ *
+ * A restart-interrupted requeue records WHAT happened without touching the orphan
+ * bookkeeping: `lastOrphanedAt` stays so a later genuine orphan's cooldown is
+ * still measured from the last genuine orphan, not from our restart.
+ *
+ * @param {object} args
+ * @param {number} args.now - epoch ms, the same instant the decision used
+ * @param {object} [args.resumePatch] - merged last, requeue only
+ * @returns {{ status: 'pending'|'blocked', metadata: object }}
+ */
+export function buildOrphanOutcome({ decision, task, agentId, now, resumePatch = {} }) {
+ const { outcome, interrupted, retryCount, totalSpawns, reason, cooldownRemaining } = decision;
+ const nowIso = new Date(now).toISOString();
+ const restartInterrupted = outcome === 'requeue' && interrupted;
+ const genuineOrphan = { orphanRetryCount: retryCount, lastOrphanedAt: nowIso, lastOrphanedAgentId: agentId };
+
+ const outcomeFields = {
+ requeue: restartInterrupted
+ // The refunded count is persisted so the restart's spawn is genuinely given back.
+ ? { totalSpawnCount: totalSpawns, lastInterruptedAt: nowIso, lastInterruptedAgentId: agentId }
+ : genuineOrphan,
+ cooldown: {
+ ...genuineOrphan,
+ blockedReason: reason,
+ blockedCategory: 'orphan-cooldown',
+ blockedAt: nowIso,
+ cooldownUntil: new Date(now + cooldownRemaining).toISOString(),
+ },
+ blocked: { orphanRetryCount: retryCount, blockedReason: reason, blockedCategory: 'max-retries', blockedAt: nowIso },
+ }[outcome];
+
+ return {
+ status: outcome === 'requeue' ? 'pending' : 'blocked',
+ metadata: {
+ ...task.metadata,
+ ...outcomeFields,
+ interruptedByRestart: restartInterrupted,
+ ...(outcome === 'requeue' ? resumePatch : {}),
+ },
+ };
+}
diff --git a/server/services/agentOrphanRecovery.test.js b/server/services/agentOrphanRecovery.test.js
new file mode 100644
index 0000000000..8ad1f769f4
--- /dev/null
+++ b/server/services/agentOrphanRecovery.test.js
@@ -0,0 +1,77 @@
+import { describe, it, expect } from 'vitest';
+import { MAX_TOTAL_SPAWNS } from '../lib/cosValidation.js';
+import { MAX_ORPHAN_RETRIES, ORPHAN_GUARDS, ORPHAN_RETRY_COOLDOWN_MS, classifyOrphanGuard, decideOrphanRetry } from './agentOrphanRecovery.js';
+
+const NOW = Date.parse('2026-06-01T12:00:00.000Z');
+const MIN = 60 * 1000;
+const decide = (metadata, wasInterrupted = false) =>
+ decideOrphanRetry({ task: { metadata }, wasInterrupted, now: NOW });
+
+// The #3202 budget contract: a genuine orphan pays a retry and arms the cooldown;
+// a PortOS restart pays neither and gets its spawn refunded; the spawn ceiling
+// bounds both.
+describe('decideOrphanRetry', () => {
+ it.each([
+ ['first genuine orphan', { orphanRetryCount: 0, totalSpawnCount: 1 }, false, { outcome: 'requeue', retryCount: 1, totalSpawns: 1 }],
+ ['last genuine retry still within budget', { orphanRetryCount: MAX_ORPHAN_RETRIES - 2, totalSpawnCount: 2 }, false, { outcome: 'requeue', retryCount: MAX_ORPHAN_RETRIES - 1, totalSpawns: 2 }],
+ ['genuine orphan at the retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES - 1, totalSpawnCount: 3 }, false,
+ { outcome: 'blocked', retryCount: MAX_ORPHAN_RETRIES, reason: `orphan retries exceeded (${MAX_ORPHAN_RETRIES}/${MAX_ORPHAN_RETRIES})` }],
+ ['genuine orphan at the spawn ceiling', { orphanRetryCount: 0, totalSpawnCount: MAX_TOTAL_SPAWNS }, false,
+ { outcome: 'blocked', reason: `total spawns exceeded (${MAX_TOTAL_SPAWNS}/${MAX_TOTAL_SPAWNS})` }],
+ ['spawn ceiling is reported ahead of the retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES, totalSpawnCount: MAX_TOTAL_SPAWNS }, false,
+ { outcome: 'blocked', reason: `total spawns exceeded (${MAX_TOTAL_SPAWNS}/${MAX_TOTAL_SPAWNS})` }],
+ ['interrupted run is not charged and its spawn is refunded', { orphanRetryCount: 2, totalSpawnCount: 4 }, true,
+ { outcome: 'requeue', retryCount: 2, totalSpawns: 3 }],
+ ['interrupted run ignores the orphan retry ceiling', { orphanRetryCount: MAX_ORPHAN_RETRIES + 2, totalSpawnCount: 1 }, true,
+ { outcome: 'requeue', retryCount: MAX_ORPHAN_RETRIES + 2, totalSpawns: 0 }],
+ ['interrupted run is still bounded by the spawn ceiling after the refund', { orphanRetryCount: 0, totalSpawnCount: MAX_TOTAL_SPAWNS + 1 }, true,
+ { outcome: 'blocked', retryCount: 0, totalSpawns: MAX_TOTAL_SPAWNS }],
+ ['interrupted run never reads a negative spawn count', {}, true, { outcome: 'requeue', retryCount: 0, totalSpawns: 0 }],
+ ])('%s', (_name, metadata, wasInterrupted, expected) => {
+ expect(decide(metadata, wasInterrupted)).toMatchObject({ interrupted: wasInterrupted, ...expected });
+ });
+
+ describe('cooldown', () => {
+ const recent = (minutesAgo) => new Date(NOW - minutesAgo * MIN).toISOString();
+
+ it('holds a genuine orphan inside the window and reports the remaining minutes', () => {
+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(10) })).toMatchObject({
+ outcome: 'cooldown', cooldownRemaining: ORPHAN_RETRY_COOLDOWN_MS - 10 * MIN, cooldownMinutes: 20,
+ reason: 'Orphan retry cooldown (20m remaining)',
+ });
+ });
+
+ it('requeues once the window has elapsed', () => {
+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(31) }).outcome).toBe('requeue');
+ });
+
+ it('does not cool down an interrupted run', () => {
+ expect(decide({ orphanRetryCount: 1, totalSpawnCount: 2, lastOrphanedAt: recent(1) }, true).outcome).toBe('requeue');
+ });
+
+ it('blocks rather than cools down when the budget is spent', () => {
+ expect(decide({ orphanRetryCount: MAX_ORPHAN_RETRIES - 1, totalSpawnCount: 3, lastOrphanedAt: recent(1) }).outcome).toBe('blocked');
+ });
+
+ it('treats an unparseable lastOrphanedAt as never orphaned', () => {
+ expect(decide({ orphanRetryCount: 0, lastOrphanedAt: 'not-a-date' })).toMatchObject({ outcome: 'requeue', cooldownRemaining: 0 });
+ });
+ });
+});
+
+describe('orphan guard order', () => {
+ it('pins the precedence between the guards', () => {
+ expect(ORPHAN_GUARDS.map(guard => guard.reason)).toEqual([
+ 'terminal-block', 'private-security', 'already-handled', 'completed', 'content-refusal', 'deep-audit', 'retry-held',
+ ]);
+ });
+
+ it('returns null for an ordinary orphan', () => {
+ expect(classifyOrphanGuard({ status: 'in_progress', metadata: {} }, null)).toBeNull();
+ });
+
+ it('resolves a task matching several guards to the earliest one', () => {
+ const task = { status: 'blocked', metadata: { blockedCategory: 'user-terminated', retryPendingCleanup: 'agent-dead' } };
+ expect(classifyOrphanGuard(task, { executionInterruption: { category: 'content-refusal' } })).toBe('terminal-block');
+ });
+});