Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

85.8 KB
Raw Blame
import { reconcileAssignmentForTask } from '../lib/claimContinuation.js';
import { hasClaimFlowContract } from '../lib/claimFlowTaskTypes.js';
import { isFalsyMeta, isTruthyMeta } from '../lib/metadataFlags.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
import { reconcileDeadOwnerReservations } from './deadOwnerReservations.js';
import { MAX_ORPHAN_RETRIES, ORPHAN_GUARDS, buildOrphanOutcome, classifyOrphanGuard, decideOrphanRetry } from './agentOrphanRecovery.js';
/**
 * Agent Management
 *
 * Handles agent termination, process stats, kill-all, orphan cleanup,
 * and orphaned task retry logic.
 */

import { rm } from 'node:fs/promises';
import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';
import { ServerError } from '../lib/errorHandler.js';
import { emitLog } from './cosEvents.js';
// The DEFINING module, not a barrel (#3450). This module is one
// of the three `agentOrchestrator.js` imports, so it can never reach a transition
// through the facade — an import back from here closes the loop the facade exists
// to open. Inside the closure the single address for a transition is the module
// that declares it; a barrel would be a third answer to "where does completeAgent
// live", which is the thing this sequencing keeps removing.
import { completeAgent, updateAgent, getAgents, getAgentRecord, AGENT_RECORD_UNREADABLE, isLiveAgentRecord, readAgentRecordOrUnreadable } from './cosAgentLifecycle.js';
import { updateTask, addTask, getTaskById, reviveBlockedTask } from './cosTaskStore.js';
import { evaluateTasks } from './cosTaskGenerator.js';
import { forceSpawnTask } from './cos.js';
import { AGENT_PAUSED_CATEGORY, PAUSE_METADATA_KEYS, pauseMetadata, isAgentPausedTask, isResumablePausedTask, registerPauseReleaseAdapter } from '../lib/taskPauseHold.js';
import { terminateAgentViaRunner, killAgentViaRunner, pauseAgentViaRunner, getAgentStatsFromRunner, getActiveAgentsFromRunner } from './cosRunnerClient.js';
import { runnerEntryShieldsRunningRecord } from '../lib/runnerAgentLiveness.js';
import { MAX_TOTAL_SPAWNS } from '../lib/validation.js';
import { isInternalTaskId } from '../lib/taskParser.js';
import { activeAgents, runnerAgents, completingAgents, spawningTasks, userTerminatedAgents, pausedAgents, whenPausedAgentExits, useRunner, unregisterSpawnedAgent } from './agentState.js';
// Both were extracted out of agentLifecycle.js (issue #2837) so this module no
// longer depends on the lifecycle orchestrator — which depends on THIS module
// for handleOrphanedTask. Importing them from their own leaf modules is what
// lets that edge be a plain static import instead of a dynamic-import dodge.
import { cleanupAgentWorktree, resolveTaskResumePatch } from './agentWorktreeCleanup.js';
import { RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY, isRetryHeld, clearedRetryHoldMetadata, isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';
import { CLAIM_METADATA_KEYS } from './cosTaskClaim.js';
import { REQUEUED_AT_KEY, LAST_SPAWNED_AT_KEY } from '../lib/taskRequeue.js';
import { resolveTaskTargetBranch } from '../lib/taskTargetBranch.js';
import { syncRunnerAgents } from './agentRunnerSync.js';
import { flushRunnerOutputBatcher } from './agentRunnerOutputBatchers.js';
import { appendRunEvent } from './agentRunEventLog.js';
import { committedDuringRun, toEpochMs } from '../lib/gitCommitProbe.js';
import { retireDeadAgent, stampLiExecutionVerdict } from './agentFinalization.js';
import { fileInvestigationTask } from './investigationTaskProducer.js';
import { buildInvestigationFingerprint } from '../lib/investigationTasks.js';
import { PATHS } from '../lib/fileUtils.js';
import { readHostShutdownMarker, clearHostShutdownMarker, HOST_SHUTDOWN_REASON } from '../lib/hostShutdown.js';
import { killProcessTree } from '../lib/bufferedSpawn.js';
import { release } from './executionLanes.js';
import { completeExecution, errorExecution } from './toolStateMachine.js';
import * as shellService from './shell.js';

const ROOT_DIR = PATHS.root;

// Startup has two callers for this sweep: cos.start() awaits it before the
// first dequeue, while the spawner's delayed safety sweep runs shortly after
// its event wiring. Keep the whole sweep single-flight so both callers cannot
// read the same running agent and each requeue it (which would emit duplicate
// wakeups and dispatch the same claim task twice).
let orphanCleanupPromise = null;

/**
 * Map a failed runner-op result (`{ error?, status? }`) to a ServerError.
 * A genuine runner 404 — the agent is gone / the runner restarted out of sync
 * with `runnerAgents` — preserves the missing-agent contract (404 NOT_FOUND);
 * any other runner/RPC failure is a 500 operational error under `failCode`.
 */
function runnerFailureError(result, fallbackMessage, failCode) {
  const notFound = result?.status === 404;
  return new ServerError(result?.error || fallbackMessage, {
    status: notFound ? 404 : 500,
    code: notFound ? 'NOT_FOUND' : failCode,
  });
}

/**
 * Shared runner-mode termination path for terminateAgent and killAgent.
 * Calls runnerFn, marks the task blocked, and cleans up. Returns the runner
 * result shape (`{ success, error?, status? }`) — callers decide whether a
 * failure throws (killAgent, whose route surfaces status) or is returned for
 * internal orchestration to inspect (terminateAgent, whose route is
 * fire-and-forget). The runner's HTTP `status` is preserved on failure so
 * killAgent can distinguish a genuine 404 from a 5xx infra error.
 */
async function terminateRunnerAgent(agentId, runnerFn, errorMessage, blockedReason) {
  const agentInfo = runnerAgents.get(agentId);
  if (agentInfo?.initializationTimeout) clearTimeout(agentInfo.initializationTimeout);
  const result = await runnerFn(agentId).catch(err => ({ success: false, error: err.message, status: err.status }));
  if (result.success) {
    // Drain + drop this agent's runner output batcher before the terminal
    // record so pending ~250ms-batched output lands first, and the Map entry
    // doesn't leak if the runner never emits a later completion event.
    // flushRunnerOutputBatcher is a no-op if no batcher exists.
    await flushRunnerOutputBatcher(agentId);
    await completeAgent(agentId, { success: false, error: errorMessage });
    await releaseAgentReservation(agentId);
    const task = agentInfo?.task;
    if (task) {
      await updateTask(task.id, {
        status: 'blocked',
        metadata: {
          ...task.metadata,
          blockedReason,
          blockedCategory: 'user-terminated',
          blockedAt: new Date().toISOString()
        }
      }, task.taskType || 'user');
    }
    runnerAgents.delete(agentId);
  }
  return result;
}

// The agent-addressed half of the pause release (#3730). `updateTask` drops the
// pause bookkeeping whenever a task leaves `blocked`, but the resumed run also has
// to be POINTED at the branch/worktree the paused run left behind — and resolving
// that needs the paused agent's record. `cosTaskStore.js` cannot import this module
// (static cycle, see `agentImportCycles.test.js`), so it reaches these two steps
// through the registration below, the way `sharing/recordEvents.js` registers its
// subscription adapter. That is what makes the Tasks-tab status toggle and
// `reviveBlockedTask` resume in place instead of spawning clean, not just the
// Resume dialog.
registerPauseReleaseAdapter({
  /**
   * Same `resolveTaskResumePatch` every other dead-run path uses, addressed by the
   * agent the task's own pause bookkeeping names.
   */
  resolvePausedTaskResume: async (task) => {
    const agentId = task?.metadata?.pausedAgentId;
    if (!agentId) return {};
    const agent = await getAgentRecord(agentId).catch(() => null);
    return resolveTaskResumePatch({ task, agentId, agentMetadata: agent?.metadata || null });
  },

  /**
   * Retire the paused record the moment its task is running again, rather than
   * leaving the card stranded until the next `retireStrandedPausedAgents` sweep.
   * Guarded on the record still being `paused`, and it writes the SAME verdict
   * `resumeAgent` would — so whichever of the two fires first, the record reads
   * identically and the other is a no-op through `completeAgent`'s idempotence.
   */
  retirePausedAgent: async (agentId, taskId, branchName) => {
    const agent = await getAgentRecord(agentId).catch(() => null);
    if (agent?.status !== 'paused') return;
    pausedAgents.delete(agentId);
    await completeAgent(agentId, {
      success: false,
      resumed: true,
      resumedTaskId: taskId,
      error: resumeSummary(agentId, { taskId, mode: 'requeued', branchName })
    });
  }
});

async function markPausedTask(agentInfo, agentId, pausedAt, reason) {
  const task = agentInfo?.task || (agentInfo?.taskId ? await getTaskById(agentInfo.taskId).catch(() => null) : null);
  if (!task) return;
  await updateTask(task.id, {
    status: 'blocked',
    metadata: {
      ...task.metadata,
      blockedReason: reason ? `Paused by user: ${reason}` : 'Paused by user',
      blockedCategory: AGENT_PAUSED_CATEGORY,
      blockedAt: pausedAt,
      ...pauseMetadata({ agentId, pausedAt, workspacePath: agentInfo?.workspacePath, runId: agentInfo?.runId })
    }
  }, task.taskType || 'user');
}

/**
 * Record a stop REQUEST against a live run (#4540).
 *
 * The request is the fact, not the exit it causes: a run still reading
 * `interrupted` with no `run.finalized` after it is a kill that never took —
 * the process ignored SIGTERM, or the runner RPC failed — and that discrepancy
 * is invisible in the mutable record, which only ever shows the last state
 * something managed to write.
 *
 * Ids come from whichever ownership map holds the agent: neither is
 * authoritative alone (a runner-backed TUI registers in `activeAgents` while a
 * runner-spawned CLI lives in `runnerAgents`), so reading only one would
 * silently drop half the runs. An agent in NEITHER map has nothing to interrupt
 * — its caller is about to 404 or return not-found — so no event is written
 * rather than minting a projection for a run that isn't there.
 *
 * No explicit idempotency key: every termination request is a distinct fact,
 * including a second one aimed at an agent that ignored the first.
 */
async function recordInterrupt(agentId, data) {
  const info = activeAgents.get(agentId) ?? runnerAgents.get(agentId);
  if (!info) return;
  await appendRunEvent({
    kind: 'run.interrupted',
    runId: info.runId ?? null,
    agentId,
    taskId: info.taskId ?? null,
    data: { ...data, mode: runnerAgents.has(agentId) ? 'runner' : 'direct' },
  });
}

async function markAgentPaused(agentId, agentInfo, pausedAt, reason) {
  await updateAgent(agentId, {
    status: 'paused',
    pausedAt,
    metadata: {
      phase: 'paused',
      pauseReason: reason || null,
      pausedAt,
      resumeWorkspacePath: agentInfo?.workspacePath || null
    }
  });
  await markPausedTask(agentInfo, agentId, pausedAt, reason);
  // Recorded AFTER the persist, not before (#4540): both pause arms funnel
  // through here, and both roll their in-memory flag back when this function
  // throws. An event written ahead of the writes would leave the projection
  // reading `paused` for a pause that never stuck — the exact class of lie the
  // ledger exists to catch. `at: pausedAt` is the same instant the agent record
  // carries, so the content-derived id is stable and a retried persist files one
  // paused event rather than two.
  await appendRunEvent({
    kind: 'run.paused',
    runId: agentInfo?.runId ?? null,
    agentId,
    taskId: agentInfo?.taskId ?? null,
    at: pausedAt,
    data: { reason: reason || null, workspacePath: agentInfo?.workspacePath ?? null },
  });
  release(agentId);
  if (agentInfo?.executionId) {
    errorExecution(agentInfo.executionId, { message: reason ? `Agent paused by user: ${reason}` : 'Agent paused by user', code: 'agent-paused' });
    completeExecution(agentInfo.executionId, { success: false, paused: true });
  }
}

/**
 * Pause an agent without finalizing its task or cleaning up its worktree.
 * The underlying process is stopped, but the persisted agent remains paused
 * so a later resume task can use the same workspace/change context.
 */
export async function pauseAgent(agentId, reason = null) {
  return maintenance.withResource('agent', agentId, () =>
    maintenance.continueSettlement(() => pauseAdmittedAgent(agentId, reason)));
}

async function pauseAdmittedAgent(agentId, reason) {
  const pausedAt = new Date().toISOString();

  if (runnerAgents.has(agentId)) {
    const agentInfo = runnerAgents.get(agentId);
    if (agentInfo?.initializationTimeout) clearTimeout(agentInfo.initializationTimeout);
    pausedAgents.set(agentId, { pausedAt, reason });
    const result = await pauseAgentViaRunner(agentId, reason).catch(err => ({ success: false, error: err.message, status: err.status }));
    if (!result.success) {
      pausedAgents.delete(agentId);
      throw runnerFailureError(result, 'Failed to pause runner agent', 'AGENT_PAUSE_FAILED');
    }
    // Persist failure must roll back the in-memory flag too, or the maps drift
    // (pausedAgents set, runnerAgents never deleted) until the next restart.
    const persistedRunner = await markAgentPaused(agentId, agentInfo, pausedAt, reason).then(() => true, (err) => {
      pausedAgents.delete(agentId);
      emitLog('error', `❌ Failed to persist pause for runner agent ${agentId}: ${err.message}`, { agentId });
      return false;
    });
    if (!persistedRunner) {
      throw new ServerError('Failed to persist paused state', { status: 500, code: 'AGENT_PAUSE_FAILED' });
    }
    runnerAgents.delete(agentId);
    emitLog('info', `⏸️ Paused runner agent ${agentId}${reason ? `: ${reason}` : ''}`, { agentId, reason });
    return { success: true, agentId, pausedAt, mode: 'runner' };
  }

  const agent = activeAgents.get(agentId);
  if (!agent) {
    throw new ServerError('Agent not found or not running', { status: 404, code: 'NOT_FOUND' });
  }

  pausedAgents.set(agentId, { pausedAt, reason });
  // Persist BEFORE killing the process, and roll back the in-memory flag on a
  // persist failure — otherwise a throw here leaves the agent flagged paused
  // in-memory while its process keeps running (the kill below never executes).
  const persisted = await markAgentPaused(agentId, agent, pausedAt, reason).then(() => true, (err) => {
    pausedAgents.delete(agentId);
    emitLog('error', `❌ Failed to persist pause for agent ${agentId}: ${err.message}`, { agentId });
    return false;
  });
  if (!persisted) {
    throw new ServerError('Failed to persist paused state', { status: 500, code: 'AGENT_PAUSE_FAILED' });
  }

  if (agent.tuiSessionId) {
    shellService.writeToSession(agent.tuiSessionId, '\x1b');
    setTimeout(() => {
      if (activeAgents.has(agentId)) shellService.killSession(agent.tuiSessionId);
    }, 250);
  } else {
    // killProcessTree so a Windows cmd.exe-wrapped CLI shim's real child is
    // taken down, not orphaned (#2243). `processGroup` (set at spawn time by
    // agentCliSpawning) additionally reaches the harness behind a credential-
    // bootstrap wrapper on POSIX; it is false for every other agent, so their
    // teardown is unchanged (#7496).
    killProcessTree(agent.process, 'SIGTERM', { processGroup: agent.processGroup });
    const killTimer = setTimeout(() => {
      if (activeAgents.has(agentId)) {
        killProcessTree(agent.process, 'SIGKILL', { processGroup: agent.processGroup });
      }
    }, 5000);
    const agentEntry = activeAgents.get(agentId);
    if (agentEntry) agentEntry.killTimer = killTimer;
  }

  emitLog('info', `⏸️ Paused agent ${agentId}${reason ? `: ${reason}` : ''}`, { agentId, reason });
  return { success: true, agentId, pausedAt, mode: agent.tuiSessionId ? 'tui' : 'direct' };
}

/**
 * What resuming `agentId` should actually DO, given the current state of the task
 * its pause parked. Four outcomes, and only one of them creates anything:
 *
 * - `requeued` — the task is still BLOCKED and is ours to restart. That covers the
 *   pause itself and any later block the task fell into (a failed retry, a cooldown,
 *   a config block): the user asking to resume is an explicit dispatch, and
 *   `reviveBlockedTask` resets the spawn/orphan budgets so it can actually run.
 * - `superseded` — a DIFFERENT agent has since paused it. That pause is live and
 *   belongs to someone else; ours is spent, so we retire our record and touch nothing.
 * - `already-active` — the task is `pending` or `in_progress`: some other path
 *   (a dedupe revive, an autopilot re-dispatch, a cooldown expiry, a human unblocking
 *   it) already put it back in flight. Queueing anything here is what spawned the
 *   SECOND agent users kept seeing — the work is already running, so we only retire
 *   the stale paused record.
 * - `new-task` — the task is gone or `completed`. Nothing to restart, so the fresh
 *   task the dialog described is the honest answer.
 */
function classifyResume(task, agentId) {
  if (!task) return 'new-task';
  if (task.status === 'pending' || task.status === 'in_progress') return 'already-active';
  if (task.status === 'blocked') {
    return isAgentPausedTask(task) && !isResumablePausedTask(task, agentId) ? 'superseded' : 'requeued';
  }
  return 'new-task';
}

/**
 * Resume a PAUSED agent: requeue the agent's OWN task, pointed at the branch and
 * worktree its paused run left behind, and retire the paused agent record.
 *
 * Before this, "resume" was a client-side illusion — the UI queued a brand-new
 * `[Resume] <description>` task carrying the old run's context as prose. That
 * produced three wrong outcomes at once: a second agent spawned on a clean
 * worktree off the default branch (so the paused run's work was redone, or lost),
 * the original task stayed `blocked` as `agent-paused` forever, and the paused
 * agent record sat in `paused` with nothing left to resume it.
 *
 * The requeue reuses the retry machinery: `resolveTaskResumePatch` decides what
 * the paused run actually left behind (adopt its surviving worktree, attach to
 * its branch, or nothing) and `agentWorkspacePrep` already honors the resulting
 * `existingBranch`/`resumeWorktreePath` pointer — so resuming needs no new spawn
 * plumbing, and `resumedFromAgentId` gets the prompt's resume banner for free.
 *
 * `overrides` carries the resume dialog's edits: extra `context` (appended to the
 * task's, never replacing it), and provider/model/effort/app changes for the new
 * run. A falsy value means "unchanged" — the dialog seeds its selects from the
 * paused run, so blank is absence, not an intentional clear.
 *
 * `classifyResume` decides which of the four outcomes applies; only `new-task`
 * creates anything. The agent record is retired on ALL of them, so a resume never
 * leaves a stale `paused` row next to a task that has moved on — that stale row is
 * what made the next resume click read its own pause as spent and queue a duplicate.
 *
 * The retirement carries `resumed: true`, which is what keeps a pause from reading
 * as a failure downstream: `completeAgent` skips the `stats.errors` bump and the
 * learning listener (taskLearning/lifecycle.js) skips the record entirely. Both
 * matter because the run is a CONTINUATION — the resumed run records the real
 * verdict, so counting this one charges a phantom failure and double-counts the task.
 *
 * @param {string} agentId
 * @param {{context?: string, description?: string, provider?: string, model?: string, effort?: string, app?: string, screenshots?: string[]}} overrides
 * @returns {Promise<{success: true, agentId: string, taskId: string|null, mode: 'requeued'|'already-active'|'superseded'|'new-task', created: boolean, branchName: string|null}>}
 */
export async function resumeAgent(agentId, overrides = {}) {
  maintenance.assertOpen();
  const agent = await getAgentRecord(agentId);
  if (!agent) {
    throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });
  }
  if (agent.status !== 'paused') {
    throw new ServerError(`Agent ${agentId} is ${agent.status}, not paused`, {
      status: 409, code: 'AGENT_NOT_PAUSED'
    });
  }

  const taskId = agent.taskId || agent.metadata?.taskId || null;
  let task = taskId ? await getTaskById(taskId).catch(() => null) : null;
  const taskType = task?.taskType || agent.metadata?.taskType || 'user';

  const mode = classifyResume(task, agentId);
  if (mode === 'requeued' || mode === 'new-task') {
    const priorClaim = isTruthyMeta(agent.metadata?.configClaimFlow) || hasClaimFlowContract({ ...agent, taskType: agent.metadata?.taskType });
    // Missing task metadata cannot be reconstructed from caller prose. Only a
    // positively generic registration may use the historical description fallback.
    if ((!task && (priorClaim || !isFalsyMeta(agent.metadata?.configClaimFlow)))
      || (task && priorClaim && !hasClaimFlowContract(task))) {
      throw new ServerError('Resume blocked: the original task configuration is unavailable, so claim authority cannot be verified. Restore the original task or start a new task from its configured workflow.', {
        status: 409, code: 'AGENT_RESUME_CONTRACT_MISSING'
      });
    }
    if (hasClaimFlowContract(task)) {
      task = { ...task, metadata: { ...task.metadata, claimFlow: true } };
    }
  }
  let resumed;
  switch (mode) {
    case 'requeued':
      resumed = await requeuePausedTask({ task, taskType, overrides });
      break;
    case 'new-task':
      resumed = await replacePausedTask({ agentId, agent, task, taskType, overrides });
      break;
    default:
      // `already-active` / `superseded` — the task needs nothing done to it.
      resumed = { taskId: task.id, mode, branchName: null };
  }
  const summary = resumeSummary(agentId, resumed);

  // Retire the paused record LAST: `completeAgent` emits `agent:completed`, whose
  // handler dequeues — so the task must already be `pending` and pointed at the
  // resume branch before the spawn it triggers can pick it up. Paused records are
  // explicitly completable (see the idempotence guard in `completeAgent`).
  pausedAgents.delete(agentId);
  // A paused run was released back to the queue (#4540). Only the modes that
  // actually CONTINUE the work qualify: 'already-active' and 'superseded' retire
  // the paused record without queueing anything, so recording them as a resume
  // would show a run going back to `running` that nothing is running. `mode` is
  // the part the mutable record still loses — 'requeued' and 'new-task' both
  // read as a completed paused agent afterwards, and only one left a fresh task.
  if (mode === 'requeued' || mode === 'new-task') {
    await appendRunEvent({
      kind: 'run.resumed',
      runId: agent.metadata?.runId ?? null,
      agentId,
      taskId: resumed.taskId ?? taskId,
      data: { mode: resumed.mode ?? mode, branchName: resumed.branchName ?? null },
    });
  }
  await completeAgent(agentId, {
    success: false,
    resumed: true,
    resumedTaskId: resumed.taskId,
    error: summary
  });

  emitLog('info', `▶️ ${summary}`, {
    agentId, taskId: resumed.taskId, branchName: resumed.branchName, mode: resumed.mode
  });

  // Dispatched AFTER the retirement above, for the same reason that retirement is
  // last: the task has to be `pending` and pointed at its resume branch before
  // anything can spawn it.
  const dispatch = await dispatchResumedTask(resumed);

  // `created` says whether anything was queued, so a caller doesn't have to keep its
  // own copy of the mode enum to know — a client that guesses would go on announcing
  // "created a resume task" for a future non-creating mode, which is the exact false
  // claim this change exists to delete.
  return { success: true, agentId, created: mode === 'new-task', ...resumed, ...dispatch };
}

/**
 * Start the resumed task NOW, when the agent pool has room for it.
 *
 * The requeue alone only makes the task ELIGIBLE. What actually spawns it is the
 * automatic dequeue `completeAgent` schedules, and that path admits pending USER
 * tasks plus auto-approved system tasks under CoS auto-run in `execute` mode — so
 * a resumed internal task on an install with auto-run off, or one still awaiting
 * approval, sat `pending` until the user opened the task list and pressed Run.
 * That turns the one-click Resume/Relaunch into a two-step, on the exact screen
 * the user just acted from.
 *
 * This sits in `resumeAgent` rather than in `relaunchAgent` because relaunch is
 * defined as a pause plus a resume — it inherits the dispatch through that
 * composition, so the rule has one address. It does NOT sit deeper, in
 * `reviveBlockedTask` or the `tasks:changed` unblock listener, because those are
 * shared with the autonomous revival paths (investigation retry, orphan cooldown,
 * completion cleanup), and force-spawning there would strip the auto-run gate that
 * deliberately withholds unattended spawns. Both doors into here are a human
 * clicking a button, which is what makes the explicit dispatch correct.
 *
 * It goes through `forceSpawnTask`, the same door "Run now" uses, and inherits its
 * refusals rather than restating them. Best-effort by design: a stopped/paused
 * daemon, an unreachable runner, a task needing approval, and a full agent pool all
 * mean "leave it queued", which is the pre-existing behavior. The refusal is
 * reported as `spawnHold` so the dialog can say which, instead of toasting a resume
 * that silently didn't start.
 */
async function dispatchResumedTask({ taskId, mode }) {
  // `already-active` and `superseded` queue nothing on purpose — the task is
  // either already in flight or owned by a later pause, and dispatching it here
  // is exactly the duplicate spawn `classifyResume` exists to prevent.
  if (!taskId || (mode !== 'requeued' && mode !== 'new-task')) return { spawned: false, spawnHold: null };

  const result = await forceSpawnTask(taskId).catch(err => ({ error: err?.message || String(err) }));
  if (result?.success) return { spawned: true, spawnHold: null };

  // The dequeue `completeAgent` schedules races this call, so a refusal can mean
  // "already claimed" rather than "held". Read the state back and treat a real
  // claim as started — string-matching the refusal would drift the moment
  // forceSpawnTask reworded one of them.
  //
  // BOTH halves are load-bearing. A spawn registers its agent as `running`
  // BEFORE it flips the task off `pending` (the window `forceSpawnTask`'s own
  // holder guard exists for), so mid-spawn the task still reads `pending` while
  // the run is genuinely under way — and the refusal that lands there is that
  // holder guard, so checking only the task status would report a hold for a
  // task that is already running. The holder half is only reachable while that
  // guard is refusing, i.e. while the holder is young enough to be mid-spawn:
  // an older, zombie holder is superseded by `forceSpawnTask` instead, so this
  // can't read a dead agent's record as a live run.
  const [task, agents] = await Promise.all([
    getTaskById(taskId).catch(() => null),
    getAgents().catch(() => []),
  ]);
  const heldByRunningAgent = agents.some(agent =>
    agent.status === 'running' && (agent.taskId === taskId || agent.metadata?.taskId === taskId));
  if (task?.status === 'in_progress' || heldByRunningAgent) return { spawned: true, spawnHold: null };

  const spawnHold = result?.error || 'Task could not be spawned';
  emitLog('info', `⏳ Resumed task ${taskId} stays queued — ${spawnHold}`, { taskId });
  return { spawned: false, spawnHold };
}

/**
 * How long a relaunch waits for a LOCAL paused run's process to actually exit.
 * The pause path escalates SIGTERM → SIGKILL after 5s, so the close handler has
 * landed well inside this window in every observed case; the cap only exists so
 * a wedged child can't hold the request open forever.
 */
const RELAUNCH_EXIT_TIMEOUT_MS = 15000;

/**
 * Relaunch a RUNNING agent's task on a different provider/model/effort.
 *
 * The motivating case is a run that is alive but going nowhere — a CLI sitting on
 * a provider usage limit. Pausing and resuming already does exactly the right
 * thing (stop the process, keep the worktree, requeue the same task with new
 * provider/model/effort overrides), but as two clicks it is neither discoverable
 * nor safe to do quickly. This composes the two so one click switches providers.
 *
 * Composition, not a third code path: `pauseAgent` owns stopping the process and
 * preserving the worktree, `resumeAgent` owns the requeue and the four resume
 * modes. The only thing between them is waiting for the stopped process to be
 * gone — a human clicking Pause then Resume takes seconds, but collapsing the two
 * into one request does not, and resuming first is what loses the worktree
 * (`whenPausedAgentExits` in agentState.js has the mechanism).
 *
 * @param {string} agentId
 * @param {{context?: string, provider?: string, model?: string, effort?: string, app?: string, reason?: string}} overrides
 */
export async function relaunchAgent(agentId, overrides = {}) {
  maintenance.assertOpen();
  const agent = await getAgentRecord(agentId);
  if (!agent) {
    throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });
  }
  if (agent.status !== 'running') {
    throw new ServerError(`Agent ${agentId} is ${agent.status}, not running`, {
      status: 409, code: 'AGENT_NOT_RUNNING'
    });
  }

  const { reason, ...resumeOverrides } = overrides;
  const pauseReason = reason || relaunchReason(resumeOverrides);
  const paused = await pauseAgent(agentId, pauseReason);

  if (!await whenPausedAgentExits(agentId, RELAUNCH_EXIT_TIMEOUT_MS)) {
    // Keep the pause flag until the actual exit: retiring it early makes a late
    // exit finalize the old run and clean up the worktree the retry needs.
    throw new ServerError('The previous agent has not exited yet. Its task and worktree remain paused; resume after it stops.', {
      status: 409, code: 'AGENT_EXIT_PENDING'
    });
  }

  // `resumeAgent` already force-spawned the requeued task where it could
  // (`dispatchResumedTask`), so `spawned`/`spawnHold` ride along in `resumed` —
  // relaunch inherits the immediate dispatch through the composition rather than
  // repeating it.
  const resumed = await resumeAgent(agentId, resumeOverrides);
  emitLog('info', `🔁 Relaunched agent ${agentId} — ${pauseReason}`, {
    agentId, taskId: resumed.taskId, mode: resumed.mode
  });
  return { ...resumed, relaunched: true, pausedAt: paused.pausedAt };
}

/** The pause reason a relaunch records, naming what the user actually changed. */
function relaunchReason({ provider, model, effort }) {
  const target = [provider, model, effort].filter(Boolean).join(' / ');
  return `Relaunched by user${target ? ` on ${target}` : ''}`;
}

/** One line naming what the resume actually did — used for the log and the record. */
function resumeSummary(agentId, { taskId, mode, branchName }) {
  switch (mode) {
    case 'requeued':
      return `Resumed agent ${agentId} — task ${taskId} requeued${branchName ? ` on ${branchName}` : ' from a clean workspace'}`;
    case 'already-active':
      return `Resumed agent ${agentId} — task ${taskId} is already queued or running, nothing to restart`;
    case 'superseded':
      return `Resumed agent ${agentId} — task ${taskId} is paused by a later agent, leaving that pause intact`;
    default:
      return `Resumed agent ${agentId} — its task was no longer resumable, queued ${taskId} instead`;
  }
}

/**
 * The in-place resume: flip the paused task back to `pending` carrying the resume
 * pointer and the dialog's overrides, in ONE write — the task must never be
 * spawnable before it is pointed, or the dequeue `completeAgent` triggers could
 * grab it mid-transition and start clean.
 *
 * Since #3730 this is a thin agent-addressed wrapper contributing ONLY the dialog's
 * overrides: the pointer, the pause release, and the record's retirement belong to
 * `cosTaskStore.updateTask`, which performs them for every caller of the flip.
 *
 * Through `reviveBlockedTask`, the shared address for "an explicit dispatch path
 * un-blocks a blocked task": on top of updateTask's blocked-transition clear it
 * resets the spawn/orphan retry budgets, so a task paused near the MAX_TOTAL_SPAWNS
 * ceiling doesn't resume straight into `max-spawns`.
 */
async function requeuePausedTask({ task, taskType, overrides }) {
  // Only the dialog's edits. The resume pointer and the pause release belong to
  // `updateTask` (#3730) so every OTHER door into this flip gets them too — resolving
  // them here as well would run the same git probe twice per Resume click and put a
  // second copy of the rule where it can drift.
  //
  // `pending` is non-terminal, so the pointer that write lands survives it
  // (updateTask only strips a resume pointer on a terminal status).
  const result = await reviveBlockedTask(task.id, {
    metadata: {
      ...resumeOverrideMetadata(overrides, task.metadata),
      ...(hasClaimFlowContract(task) ? { claimFlow: true } : {})
    }
  }, taskType);
  if (result?.error) {
    throw new ServerError(`Failed to requeue task ${task.id}: ${result.error}`, {
      status: 500, code: 'AGENT_RESUME_FAILED'
    });
  }
  return { taskId: task.id, mode: 'requeued', branchName: resolveTaskTargetBranch(result?.metadata) };
}

// A replacement is a fresh task, but its task-type payload/configuration still
// defines what the run must do. Preserve that durable contract while dropping
// state owned by the spent task/run so the replacement gets a new retry budget,
// lease, workspace pointer, and output-hook dispatch.
const REPLACEMENT_RUNTIME_METADATA_KEYS = new Set([
  'blockedReason', 'blockedCategory', 'blockedAt', 'blocker', 'failureCount',
  'lastErrorCategory', 'lastFailureAt', 'cooldownUntil', 'totalSpawnCount',
  'orphanRetryCount', 'lastOrphanedAt', 'lastOrphanedAgentId', 'worktreeBusyAttempts',
  LAST_SPAWNED_AT_KEY, 'interruptedByRestart', 'lastInterruptedAt', 'lastInterruptedAgentId',
  'outputHookDispatchedAt', 'existingBranch', 'resumedFromAgentId', 'resumeWorktreePath', 'claimResumeInPlace',
  'autoRetryCount', 'autoRetriedByInvestigation', 'autoRetriedAt', 'autoRetryExhaustedAt',
  'resolution', 'autoExpiredReason', 'autoExpiredAt',
  REQUEUED_AT_KEY, RETRY_HOLD_KEY, RETRY_HOLD_SINCE_KEY,
  ...PAUSE_METADATA_KEYS, ...CLAIM_METADATA_KEYS,
]);

/**
 * A replacement queued because the original task is gone still has to be the same
 * kind of run. Registration projects `analysisType` onto the agent as
 * `taskAnalysisType` and does not copy it back, so a missing task became a generic
 * commit task: the live checkout was probed, a correct no-op failed delivery, and
 * the retry budget spawned more agents. Fill only keys the task record did not
 * already carry. The assignment is accepted only when it still validates as this
 * task's own snapshot — resume prose cannot supply one.
 */
function inheritPausedCoordinatorContract(metadata, agentMeta, { app, taskType }) {
  if (!agentMeta) return;
  if ((metadata.analysisType == null || metadata.analysisType === '')
    && typeof agentMeta.taskAnalysisType === 'string' && agentMeta.taskAnalysisType.trim()) {
    metadata.analysisType = agentMeta.taskAnalysisType;
  }
  // `false` and the TASKS.md string `"false"` are the load-bearing values.
  if (metadata.worktreeChangesExpected == null && agentMeta.worktreeChangesExpected != null) {
    metadata.worktreeChangesExpected = agentMeta.worktreeChangesExpected;
  }
  // Strict boolean: a string `"true"` is not the on-demand lane marker.
  if (metadata.onDemand == null && agentMeta.taskOnDemand === true) metadata.onDemand = true;
  if (metadata.analysisType === 'branch-reconcile' && metadata.reconcileAssignment == null) {
    const validated = reconcileAssignmentForTask({
      taskType,
      metadata: {
        // Top-level `app` overwrites a metadata seed at intake, so a dialog
        // switch has to win here too. Otherwise the previous app's snapshot
        // would be stored on the task the replacement actually runs.
        app: app || metadata.app,
        analysisType: 'branch-reconcile',
        reconcileAssignment: agentMeta.reconcileAssignment,
      },
    });
    if (validated) metadata.reconcileAssignment = validated;
  }
}

/**
 * The fallback: the paused run's task is gone or completed, so queue the fresh task
 * the caller described instead. Same outcome the pre-fix client always took, kept
 * only for the case where it is actually correct.
 *
 * Everything the original task carried that SHAPES the run — its app, its
 * provider/model/effort, its context — is INHERITED as the base, with the dialog's
 * edits layered on top through the same `resumeOverrideMetadata` the requeue path
 * uses, so both resumes merge by one rule. A bare description is not a runnable
 * substitute for a task that was scoped to a managed app or pinned to a provider:
 * dropping those is how the replacement ended up running against the wrong repo, or
 * against a default model the user had deliberately moved off.
 */
async function replacePausedTask({ agentId, agent, task, taskType, overrides }) {
  const description = overrides.description
    || task?.description
    || agent?.metadata?.taskDescription
    || `Resume ${agentId}`;
  // `prompt` rides along with `context` (#4153): on a task written by current
  // code the agent-facing payload lives there, and a replacement queued without
  // it would run with nothing but the one-line description.
  const taskMeta = task?.metadata || {};
  const agentMeta = agent?.metadata || {};
  const context = taskMeta.context ?? agentMeta.context;
  const prompt = taskMeta.prompt ?? agentMeta.prompt;
  const provider = taskMeta.provider ?? agentMeta.providerId ?? agentMeta.provider;
  const model = taskMeta.model ?? agentMeta.model;
  const effort = taskMeta.effort ?? agentMeta.effort;
  const app = taskMeta.app ?? agentMeta.taskApp ?? agentMeta.app;
  const inheritedMetadata = { ...taskMeta };
  if (!task && agent?.metadata) {
    if (agent.metadata.taskDescription) inheritedMetadata.taskDescription = agent.metadata.taskDescription;
    if (agent.metadata.jobId) inheritedMetadata.jobId = agent.metadata.jobId;
    if (agent.metadata.taskApp) inheritedMetadata.taskApp = agent.metadata.taskApp;
  }
  for (const key of REPLACEMENT_RUNTIME_METADATA_KEYS) delete inheritedMetadata[key];
  if (agentId) inheritedMetadata.resumedFromAgentId = agentId;
  // Validate against the app the replacement will actually run, including a
  // dialog switch, so a snapshot for the previous app is not attached as a grant.
  const overridesPatch = resumeOverrideMetadata(overrides, task?.metadata || agent?.metadata);
  inheritPausedCoordinatorContract(inheritedMetadata, agent?.metadata, {
    app: overridesPatch.app || app, taskType,
  });
  const created = await addTask({
    description,
    context, prompt, provider, model, effort, app,
    metadata: inheritedMetadata,
    ...overridesPatch
  }, taskType);
  if (created?.error) {
    throw new ServerError(`Failed to queue resume task: ${created.error}`, {
      status: 500, code: 'AGENT_RESUME_FAILED'
    });
  }
  return { taskId: created.id, mode: 'new-task', branchName: null };
}

/**
 * The dialog's edits as a task-metadata patch. Every field is "unchanged unless
 * supplied" — the dialog seeds its selects from the paused run, so a blank value
 * is absence rather than an intentional clear (AGENTS.md's absent-vs-empty rule).
 * `context` APPENDS to `existingContext`: the task's own note is what the
 * original run was given, and dropping it would resume with less information than
 * the run that paused. It never touches `metadata.prompt` — the dialog edits the
 * NOTE, and folding a note into the agent-facing payload would make the two
 * indistinguishable again (#4153).
 */
function resumeOverrideMetadata({ context, provider, model, effort, app, screenshots }, priorMetadata = {}) {
  const patch = {};
  if (context) {
    patch.context = [priorMetadata?.context, context].filter(Boolean).join('\n\n');
  }
  if (provider) patch.provider = provider;
  if (model) patch.model = model;
  if (effort) patch.effort = effort;
  if (app) patch.app = app;
  if (screenshots?.length) patch.screenshots = screenshots;
  // The one place blank does NOT mean unchanged: a provider SWITCH invalidates a
  // model or effort pinned to the provider being left behind. `selectModelForTask`
  // returns `metadata.model` verbatim as the user's choice, so carrying
  // `claude-opus-5` across to codex hands that CLI a model it does not have and the
  // requeued run dies on its first spawn. Both dialogs clear their model select when
  // the provider changes, so blank here is the user seeing "Default model" — clear the
  // stale pin and let the new provider resolve its own.
  if (provider && provider !== priorMetadata?.provider) {
    if (!model) patch.model = '';
    if (!effort) patch.effort = '';
  }
  return patch;
}

/**
 * Terminate an agent (graceful SIGTERM with SIGKILL fallback).
 */
export async function terminateAgent(agentId) {
  // Recorded ahead of both arms, so the request lands whichever one runs.
  await recordInterrupt(agentId, { reason: 'terminated-by-user', signal: 'SIGTERM' });

  // Check if agent is in runner mode
  if (runnerAgents.has(agentId)) {
    return terminateRunnerAgent(agentId, terminateAgentViaRunner, 'Agent terminated by user', 'Terminated by user');
  }

  // Direct mode
  const agent = activeAgents.get(agentId);

  // terminateAgent stays result-shaped (not a throw): its route path is
  // fire-and-forget (cosAgentLifecycle emits `agent:terminate` and returns
  // `{ success: true }` before termination runs), and its other callers are
  // internal orchestration (including the event handler) that
  // inspect the result rather than a thrown ServerError. Only pauseAgent and
  // killAgent — whose routes surface an HTTP status — throw (issue #2534).
  if (!agent) {
    return { success: false, error: 'Agent not found or not running' };
  }

  // Track as user-terminated so the close handler doesn't re-queue
  userTerminatedAgents.add(agentId);

  // Drain pending batched stdout/stderr before the terminal record so it
  // doesn't land after completion (the close handler also drains; idempotent).
  await agent.flushOutput?.();

  // Mark agent as completed immediately with termination status
  await completeAgent(agentId, { success: false, error: 'Agent terminated by user' });

  // Block task immediately (don't defer to close handler — prevents requeue on server restart)
  if (agent.taskId) {
    const task = await getTaskById(agent.taskId).catch(() => null);
    if (task) {
      await updateTask(agent.taskId, {
        status: 'blocked',
        metadata: {
          ...task.metadata,
          blockedReason: 'Terminated by user',
          blockedCategory: 'user-terminated',
          blockedAt: new Date().toISOString()
        }
      }, task.taskType || 'user');
    }
  }

  // Kill the process — killProcessTree so a Windows cmd.exe-wrapped shim's
  // real child isn't orphaned (#2243), and `processGroup` so a credential-
  // bootstrap wrapper's harness goes down with it rather than outliving the
  // terminated run (#7496).
  killProcessTree(agent.process, 'SIGTERM', { processGroup: agent.processGroup });

  // Give it a moment, then force kill if still running
  const killTimer = setTimeout(() => {
    if (activeAgents.has(agentId)) {
      killProcessTree(agent.process, 'SIGKILL', { processGroup: agent.processGroup });
      unregisterSpawnedAgent(agent.pid);
      activeAgents.delete(agentId);
    }
  }, 5000);

  // Store the timer so the close handler can clear it when the process exits cleanly
  const agentEntry = activeAgents.get(agentId);
  if (agentEntry) agentEntry.killTimer = killTimer;

  return { success: true, agentId };
}

/**
 * Get list of active agents.
 */
export function getActiveAgents() {
  const agents = [];

  // Direct mode agents
  for (const [agentId, agent] of activeAgents) {
    agents.push({
      id: agentId,
      taskId: agent.taskId,
      startedAt: agent.startedAt,
      runningTime: Date.now() - agent.startedAt,
      mode: 'direct'
    });
  }

  // Runner mode agents
  for (const [agentId, agent] of runnerAgents) {
    agents.push({
      id: agentId,
      taskId: agent.taskId,
      startedAt: agent.startedAt,
      runningTime: Date.now() - agent.startedAt,
      mode: 'runner'
    });
  }

  return agents;
}

/**
 * Force kill an agent immediately with SIGKILL (no graceful shutdown).
 */
export async function killAgent(agentId) {
  await recordInterrupt(agentId, { reason: 'killed-by-user', signal: 'SIGKILL' });

  // Check if agent is in runner mode
  if (runnerAgents.has(agentId)) {
    const result = await terminateRunnerAgent(agentId, killAgentViaRunner, 'Agent force killed by user (SIGKILL)', 'Force killed by user');
    // Surface a runner failure to the kill route as a ServerError instead of
    // the old result-shape 404 string-match (issue #2534): a genuine runner
    // 404 (agent gone / runner out of sync) stays NOT_FOUND, any other
    // runner-RPC failure is a 500 AGENT_KILL_FAILED.
    if (!result.success) {
      throw runnerFailureError(result, 'Failed to kill runner agent', 'AGENT_KILL_FAILED');
    }
    return result;
  }

  // Direct mode
  const agent = activeAgents.get(agentId);

  if (!agent) {
    throw new ServerError('Agent not found or not running', { status: 404, code: 'NOT_FOUND' });
  }

  // Track as user-terminated so the close handler doesn't re-queue
  userTerminatedAgents.add(agentId);

  // Drain pending batched stdout/stderr before the terminal record so it
  // doesn't land after completion (the close handler also drains; idempotent).
  await agent.flushOutput?.();

  // Mark agent as completed immediately with kill status
  await completeAgent(agentId, { success: false, error: 'Agent force killed by user (SIGKILL)' });

  // Block task immediately
  if (agent.taskId) {
    const task = await getTaskById(agent.taskId).catch(() => null);
    if (task) {
      await updateTask(agent.taskId, {
        status: 'blocked',
        metadata: {
          ...task.metadata,
          blockedReason: 'Force killed by user',
          blockedCategory: 'user-terminated',
          blockedAt: new Date().toISOString()
        }
      }, task.taskType || 'user');
    }
  }

  // Kill the process immediately with SIGKILL — killProcessTree so a Windows
  // cmd.exe-wrapped shim's real child isn't orphaned (#2243), and
  // `processGroup` so a credential-bootstrap wrapper's harness dies too (#7496).
  killProcessTree(agent.process, 'SIGKILL', { processGroup: agent.processGroup });

  unregisterSpawnedAgent(agent.pid);
  activeAgents.delete(agentId);

  return { success: true, agentId, pid: agent.pid, signal: 'SIGKILL' };
}

/**
 * Parse a single CSV row as emitted by `tasklist /FO CSV /NH`.
 * Handles quoted fields (quotes are stripped; commas inside quotes are not splits).
 * Returns an array of unquoted field strings.
 */
function parseTasklistCsvRow(line) {
  const fields = [];
  let cur = '';
  let inQuotes = false;
  for (let i = 0; i < line.length; i++) {
    const ch = line[i];
    if (ch === '"') { inQuotes = !inQuotes; continue; }
    if (ch === ',' && !inQuotes) { fields.push(cur); cur = ''; continue; }
    cur += ch;
  }
  fields.push(cur);
  return fields;
}

/**
 * Get process stats for an agent (CPU, memory usage).
 */
export async function getAgentProcessStats(agentId) {
  const agent = activeAgents.get(agentId);
  if (agent) {
    // TUI agents may have a null pid until the PTY child is fully attached;
    // ps/tasklist with a non-numeric pid produces misleading "dead" output.
    if (!Number.isFinite(agent.pid)) {
      return { active: true, agentId, pid: null, cpu: 0, memoryKb: 0, memoryMb: 0, state: 'unknown' };
    }

    const { exec } = await import('../lib/childProcess.js');
    const { promisify } = await import('util');
    const execAsync = promisify(exec);

    const psCmd = process.platform === 'win32'
      ? `tasklist /FI "PID eq ${agent.pid}" /FO CSV /NH`
      : `ps -p ${agent.pid} -o pid=,pcpu=,rss=,state=`;
    const result = await execAsync(psCmd).catch(() => ({ stdout: '' }));
    const line = result.stdout.trim();

    if (!line) {
      return { active: false, pid: agent.pid, cpu: 0, memoryKb: 0, memoryMb: 0, state: 'dead' };
    }

    if (process.platform === 'win32') {
      // tasklist /FO CSV /NH columns: Image Name, PID, Session Name, Session#, Memory Usage
      // Memory Usage looks like: 82,156 K (comma as thousands separator, space before K)
      // CPU is not available from basic tasklist; use 0 as an honest default.
      const fields = parseTasklistCsvRow(line);
      if (fields.length >= 5) {
        const pid = parseInt(fields[1], 10);
        const memoryKb = parseInt(fields[4].replace(/,/g, '').replace(/\s*K$/i, '').trim(), 10) || 0;
        return {
          active: true,
          agentId,
          pid,
          cpu: 0,
          memoryKb,
          memoryMb: Math.round(memoryKb / 1024 * 10) / 10,
          state: 'running'
        };
      }
    } else {
      const parts = line.split(/\s+/).filter(Boolean);
      if (parts.length >= 3) {
        return {
          active: true,
          agentId,
          pid: parseInt(parts[0], 10),
          cpu: parseFloat(parts[1]) || 0,
          memoryKb: parseInt(parts[2], 10) || 0,
          memoryMb: Math.round((parseInt(parts[2], 10) || 0) / 1024 * 10) / 10,
          state: parts[3] || 'unknown'
        };
      }
    }

    return { active: true, agentId, pid: agent.pid, cpu: 0, memoryKb: 0, memoryMb: 0, state: 'unknown' };
  }

  if (runnerAgents.has(agentId) || useRunner) {
    return await getAgentStatsFromRunner(agentId);
  }

  return null;
}

/**
 * Check if a process is running by PID.
 */
export async function isPidAlive(pid) {
  if (!pid) return false;
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
}

/**
 * Clean up orphaned agents on startup.
 * Agents marked as "running" in state but not tracked anywhere are orphaned.
 *
 * Must check:
 * 1. Local activeAgents map (direct-spawned)
 * 2. Local runnerAgents map (recently spawned via runner)
 * 3. CoS Runner service (may have agents from before server restart)
 *
 * After cleanup:
 * - Resets associated tasks to pending for auto-retry
 * - Creates investigation task after max retries exceeded
 * - Triggers evaluation to spawn new agents
 */

/**
 * Settle a Creative Director run tied to an orphaned/reaped agent (issue #2705).
 *
 * A CD plan/treatment agent that dies without issuing its completion PATCH leaves
 * its `runs[]` row stuck `running`. `maybeEnqueuePlan`'s guard then treats that
 * non-terminal run as "a worker is already on it" and refuses to re-dispatch — so
 * the project wedges in `planning` until the NEXT server restart runs boot
 * recovery (`recoverInFlightProjects`). This helper reproduces recovery's per-project
 * reap so the project recovers without a restart. It does THREE things, in the
 * same order and shape recovery does — this is boot-ordering-critical:
 *
 *   1. **Fail the stuck run** so `maybeEnqueuePlan`'s non-terminal-run guard clears.
 *   2. **Retire the CoS task** (`status:'completed'`), NOT leave it running for
 *      `handleOrphanedTask` to requeue. This is load-bearing for boot ordering:
 *      `cleanupOrphanedAgents` runs BEFORE `recoverInFlightProjects` awaits its gate
 *      (`cos.js` start sequence), and recovery only reaps runs still marked
 *      `running` — so once we've failed the run in step 1, recovery would no longer
 *      retire this task. If we left it, `handleOrphanedTask` would requeue a stale
 *      agent that races recovery's fresh dispatch. Retiring it here makes
 *      `handleOrphanedTask` skip it (it skips `completed` tasks) — matching exactly
 *      what recovery's own `updateTask(status:'completed')` reap does. (#2705 review)
 *   3. **Re-dispatch via the deduped advance loop** (`advanceAfterPlanStepSettled` for
 *      directive projects, `advanceAfterSceneSettled` otherwise) — the SAME functions
 *      recovery calls, so the project retries on its own in steady state, and at boot
 *      the call is idempotent with recovery's advance (both go through the in-memory
 *      inflight-dedup guards, so no double dispatch). This deliberately does NOT use
 *      `handleOrphanedTask`/`resetOrphanedTasks`' raw task respawn, which the
 *      `cdRecoveryDone` gate exists to keep from racing recovery.
 *
 * No-op for non-CD tasks; the CD-module imports are deferred so non-CD orphans pay
 * nothing. Exported for unit testing.
 *
 * @param {object|null} task - the orphaned agent's task record.
 * @returns {Promise<boolean>} true if a CD run was settled, false for a non-CD task.
 */
export async function settleOrphanedCreativeDirectorRun(task) {
  const cd = task?.metadata?.creativeDirector;
  if (!cd?.projectId || !cd?.runId) return false;
  const now = new Date().toISOString();
  const { updateRun, getProject } = await import('./creativeDirector/local.js');
  // 1. Fail the stuck run (clears the maybeEnqueuePlan non-terminal-run guard).
  await updateRun(cd.projectId, cd.runId, {
    status: 'failed',
    completedAt: now,
    failureReason: 'agent process terminated unexpectedly (orphaned)',
  }).catch((err) => console.log(`⚠️ CD orphan settle: run ${cd.runId?.slice(0, 8)} of ${cd.projectId} failed: ${err.message}`));
  // 2. Retire the task the same way recovery does, so handleOrphanedTask (which
  //    skips `completed` tasks) can't requeue a stale agent to race recovery.
  //    CD tasks are internal (agentBridge#persistAndEmit adds them as 'internal').
  if (task.id) {
    await updateTask(task.id, {
      status: 'completed',
      metadata: { ...task.metadata, orphanedRunSettledAt: now },
    }, 'internal').catch((err) => console.log(`⚠️ CD orphan settle: retire task ${task.id} failed: ${err.message}`));
  }
  // 3. Re-dispatch via the deduped advance loop (fire-and-forget, like recovery).
  const project = await getProject(cd.projectId).catch(() => null);
  if (project && project.status !== 'paused' && project.status !== 'failed') {
    if (project.directive) {
      const { advanceAfterPlanStepSettled } = await import('./creativeDirector/planAdvance.js');
      advanceAfterPlanStepSettled(cd.projectId)
        .catch((e) => console.log(`⚠️ CD orphan settle: plan advance for ${cd.projectId} failed: ${e.message}`));
    } else {
      const { advanceAfterSceneSettled } = await import('./creativeDirector/completionHook.js');
      advanceAfterSceneSettled(cd.projectId)
        .catch((e) => console.log(`⚠️ CD orphan settle: advance for ${cd.projectId} failed: ${e.message}`));
    }
  }
  return true;
}

/**
 * How long after a pause the sweep leaves a paused record alone regardless of what
 * its task says. `markAgentPaused` writes the agent record BEFORE the task's pause
 * hold, so for a moment a genuinely-paused agent has a task that doesn't know it —
 * and if that second write fails outright, the sweep is the recovery, just not
 * within the same minute.
 */
const PAUSE_SETTLE_GRACE_MS = 60 * 1000;

/**
 * Retire `paused` agent records whose task has moved on without them.
 *
 * A pause is a two-part state: the agent record says `paused`, and its task holds
 * the matching pause (see lib/taskPauseHold.js). Plenty of paths legitimately take
 * the task half back — a dedupe revive, an autopilot re-dispatch, a cooldown expiry,
 * a human unblocking it, another agent pausing it later. None of them know about the
 * agent record, so it used to sit in `paused` forever: visible in the paused list,
 * and — because a stale pause reads as spent — turning the next resume click into a
 * duplicate task on a second agent.
 *
 * Only the record is retired. The worktree stays on disk: whoever picked the task up
 * may be running on that very branch, and the scheduled `agent-data-cleanup` job is
 * what reaps worktrees with no live agent.
 */
async function retireStrandedPausedAgents(agents) {
  const now = Date.now();
  for (const agent of agents) {
    if (agent.status !== 'paused') continue;
    // No task to compare against — nothing proves the pause is spent, so leave it.
    const taskId = agent.taskId || agent.metadata?.taskId || null;
    if (!taskId) continue;
    const pausedAt = Date.parse(agent.pausedAt ?? agent.metadata?.pausedAt ?? '');
    if (Number.isFinite(pausedAt) && now - pausedAt < PAUSE_SETTLE_GRACE_MS) continue;

    const task = await getTaskById(taskId).catch(() => null);
    if (isResumablePausedTask(task, agent.id)) continue;

    const reason = !task
      ? `its task ${taskId} no longer exists`
      : `task ${taskId} is ${task.status} and no longer holds this pause`;
    pausedAgents.delete(agent.id);
    // `resumed: true` for the same reason `resumeAgent` uses it: this run produced no
    // verdict of its own, so charging it an error would invent a failure the task
    // never had — whoever ran the task records the real outcome.
    await completeAgent(agent.id, {
      success: false,
      resumed: true,
      error: `Pause retired — ${reason}`
    });
    emitLog('info', `⏹️ Retired stranded paused agent ${agent.id} — ${reason}`, { agentId: agent.id, taskId });
  }
}

// A running record with no pid this young may still be mid-spawn.
const SPAWN_GRACE_MS = 30000;

export function cleanupOrphanedAgents() {
  if (!orphanCleanupPromise) {
    orphanCleanupPromise = runCleanupOrphanedAgents().finally(() => {
      orphanCleanupPromise = null;
    });
  }
  return orphanCleanupPromise;
}

async function runCleanupOrphanedAgents() {
  const agents = await getAgents();
  let cleanedCount = 0;
  const orphanedTaskIds = [];
  // Agents retired by this sweep, with or without a task: each still holds the
  // maintenance reservation its original spawn took, and retireDeadAgent does
  // not release it.
  const retiredAgentIds = [];

  // Agents the PREVIOUS process owned when it was signalled down (#3202). Their
  // processes died because PortOS restarted, not because they failed — so they
  // are *interrupted*, and must not be charged orphan-retry budget or held in the
  // 30-minute orphan cooldown. A missing/garbled marker yields an empty set,
  // which is exactly the pre-existing behavior (everything is an ordinary orphan).
  const shutdownMarker = await readHostShutdownMarker();
  const interruptedByRestart = new Set(shutdownMarker?.agentIds || []);
  if (interruptedByRestart.size > 0) {
    emitLog('info', `🛑 Host restart marker: ${interruptedByRestart.size} agent(s) were interrupted by a PortOS restart`, {
      count: interruptedByRestart.size,
      shutdownAt: shutdownMarker?.at ?? null,
    });
  }

  // Get list of agents actively running in the CoS Runner
  const runnerProbe = await getActiveAgentsFromRunner().then(
    (agents) => Array.isArray(agents)
      ? { available: true, agents }
      : { available: false, agents: [] },
    (err) => {
      emitLog('debug', `Runner agent recovery probe unavailable: ${err.message}`);
      return { available: false, agents: [] };
    },
  );
  const runnerById = new Map();
  const runnerAgentsList = runnerProbe.agents;
  for (const row of runnerAgentsList) {
    if (row?.id) runnerById.set(row.id, row);
  }

  // Also sync runner agents to our local map for event handling
  if (runnerAgentsList.length > 0) {
    const synced = await syncRunnerAgents();
    if (synced > 0) {
      console.log(`🔄 Synced ${synced} agents from CoS Runner`);
    }
  }

  // The reverse pass: drop `runnerAgents` entries with nothing behind them.
  // The sweep below walks durable RECORDS, so it can never reach a map entry
  // that has no record — and nothing else prunes the map either, so a phantom
  // adopted from the runner survived until the process restarted. It then
  // over-protected worktrees from the cleanup job, and counted against the
  // update gate that blocks a restart, which is the one thing that would have
  // cleared it.
  //
  // Scoped to `runnerAgents` on purpose. `activeAgents` entries hold live
  // PTY/child handles owned by this process's own spawn closures; dropping one
  // would flip `isAgentOwnedLocally` to false and re-open the #4540
  // double-finalize that `agentRunnerSync`'s ownership check exists to prevent.
  // A runner-side entry is only ever a mirror, so losing it costs nothing but a
  // re-adoption on the next sweep if the runner still advertises it.
  for (const agentId of [...runnerAgents.keys()]) {
    const read = await readAgentRecordOrUnreadable(agentId);
    if (read === AGENT_RECORD_UNREADABLE || isLiveAgentRecord(read)) continue;
    runnerAgents.delete(agentId);
    emitLog('info', `🧹 Dropped stale runner-agent tracking for ${agentId} — no live record behind it`, { agentId });
  }

  for (const agent of agents) {
    if (agent.status === 'running') {
      if (completingAgents.has(agent.id)) continue;
      const inRemoteRunner = await runnerEntryShieldsRunningRecord(
        runnerById.get(agent.id),
        isPidAlive,
      );

      // A runner-owned agent may still be alive while the runner is booting or
      // reconnecting. Its absence from a failed probe is not evidence that the
      // process died. Leave the durable record in_progress so the next
      // connection can re-adopt it; requeueing here would race a second claim
      // agent onto the same work. Old records use `useRunner`, while newer
      // records carry the more precise executionMode.
      const executionMode = agent.metadata?.executionMode;
      const runnerOwned = agent.metadata?.useRunner === true
        || agent.metadata?.useRunner === 'true'
        || executionMode === 'runner'
        || executionMode === 'runner-tui';
      if (!runnerProbe.available && runnerOwned) {
        emitLog('debug', `Skipping orphan cleanup for runner-owned agent ${agent.id} — runner is unavailable`, { agentId: agent.id });
        continue;
      }

      // Direct-spawn handles in this process are live. Leftover runnerAgents
      // ownership from an earlier adopt is not — a stale listing must not keep
      // the durable record running forever.
      if (activeAgents.has(agent.id)) continue;
      if (inRemoteRunner) continue;
      // The record is written before the spawn finishes (prompt build, task claim,
      // PTY open), so a fresh one has no pid and no handle yet. Retiring it here
      // completes a run that goes on to launch — its claim ownership bind then
      // fails with owner-not-running. Mirrors the zombie sweep's guards.
      if (agent.taskId && spawningTasks.has(agent.taskId)) continue;
      // A shutdown-marked dispatch cannot still be initializing in this process.
      // Retire it now; otherwise the grace skips it and we consume its marker,
      // so the next sweep charges a genuine host interruption as an orphan.
      if (!agent.pid && !interruptedByRestart.has(agent.id)) {
        const startedAtMs = agent.startedAt ? Date.parse(agent.startedAt) : 0;
        if (Date.now() - startedAtMs < SPAWN_GRACE_MS) continue;
      }
      if (runnerAgents.has(agent.id)) runnerAgents.delete(agent.id);

      // Before marking as orphaned, check if the process is actually still running
      if (agent.pid) {
        const stillAlive = await isPidAlive(agent.pid);
        if (stillAlive) {
          console.log(`🔄 Agent ${agent.id} (PID ${agent.pid}) still running, re-syncing to runner tracking`);
          const inferredType = isInternalTaskId(agent.taskId) ? 'internal' : 'user';
          runnerAgents.set(agent.id, {
            id: agent.id, pid: agent.pid, taskId: agent.taskId,
            task: { id: agent.taskId, taskType: inferredType, description: 'Re-synced from PID check' }
          });
          continue;
        }
      }

      // Recheck after the awaited pid probes: completion owns its terminal write.
      if (completingAgents.has(agent.id)) continue;

      const interrupted = interruptedByRestart.has(agent.id);
      const errorMessage = interrupted
        ? 'Agent was interrupted by a PortOS server restart'
        : 'Agent process terminated unexpectedly';
      console.log(interrupted
        ? `🛑 Recovering agent ${agent.id} interrupted by a PortOS restart (PID ${agent.pid || 'unknown'} not running)`
        : `🧹 Cleaning up orphaned agent ${agent.id} (PID ${agent.pid || 'unknown'} not running)`);
      // Record the reap in the lifecycle ledger BEFORE the run is closed
      // (#4540), so the ordered stream reads "orphaned, then finalized" and a
      // replay can tell an agent that died from one that exited. Emitted even
      // when the agent record carries no `runId` — a survivor whose run id
      // never landed is precisely the case the mutable records cannot explain,
      // so it keys off the agent instead of disappearing.
      await appendRunEvent({
        kind: 'run.orphan-recovered',
        // Explicit natural key rather than the content-derived default: this
        // sweep runs every 15 minutes and can re-observe the same dead agent
        // if the `completeAgent` write below ever fails to land, and the
        // content hash covers the wall-clock `at`, so a retry would otherwise
        // mint a second "this agent died" fact. An agent dies once per life,
        // and `id + startedAt` names exactly that.
        eventId: `orphan:${agent.id}:${agent.startedAt || 'unknown'}`,
        runId: agent.metadata?.runId,
        agentId: agent.id,
        taskId: agent.taskId,
        data: {
          interruptedByRestart: interrupted,
          pid: agent.pid ?? null,
          hasRunId: Boolean(agent.metadata?.runId),
          startedAt: agent.startedAt ?? null,
        },
      });
      const task = agent.taskId ? await getTaskById(agent.taskId).catch(() => null) : null;
      const startedAt = Date.parse(agent.startedAt);
      // Same shared step list the post-restart recovery in agentLifecycle.js uses
      // (issue #8440) — output hook, sentinel removal, run-record close, agent
      // completion, in that order. A hard kill (`pm2 restart`, reboot) lands here,
      // so this is the path that leaves dirt in a managed app repo if any of it
      // is skipped.
      const retirement = await retireDeadAgent({
        agent,
        task,
        success: false,
        exitCode: interrupted ? 143 : 1,
        duration: Number.isFinite(startedAt) ? Math.max(0, Date.now() - startedAt) : 0,
        errorMessage,
        category: interrupted ? 'interrupted' : 'orphaned',
        // Post-mortem telemetry on the agent record (nothing reads it yet — the
        // human-visible distinction is the `error` string above). Worth persisting
        // because an infrastructure interruption and a real agent fault are
        // indistinguishable from the process's point of view once the record is
        // written. Kept off the shared `retireDeadAgent` shape since the
        // post-restart path has no equivalent fact to report.
        agentResultExtra: { interruptedByRestart: interrupted },
      });
      cleanedCount++;
      retiredAgentIds.push(agent.id);

      if (agent.taskId && !retirement.deliveryHeld) {
        // Carry the agent's metadata forward rather than re-reading the record
        // later: `getAgent` on a completed agent still hits the disk to hydrate
        // its output.txt tail (capped since #3498, but not free), and the
        // worktree fields the resume pointer needs are stamped once at
        // registerAgent and never mutated.
        // `startedAt` rides along too — it's the window the commit probe in
        // handleOrphanedTask needs to tell this run's commits from the repo's.
        orphanedTaskIds.push({ taskId: agent.taskId, agentId: agent.id, agentMetadata: agent.metadata, agentStartedAt: agent.startedAt });
      }
    }
  }

  // Clean up worktrees for the specific agents detected as orphaned this cycle.
  // The blanket "reap every worktree dir with no live agent" sweep lives in the
  // scheduled `agent-data-cleanup` job (autonomousJobs.js), NOT here — running it
  // on the 15-min health-check hot path gave it a wide window to remove a human's
  // in-flight `/claim` worktree mid-review. Targeted cleanup of a known-dead
  // agent's own worktree is safe and stays.
  for (const { agentId } of orphanedTaskIds) {
    await cleanupAgentWorktree(agentId, false);
  }

  // Settle any Creative Director run tied to an orphaned agent (issue #2705)
  // BEFORE the retry below, so the project can advance without a server restart.
  for (const { taskId } of orphanedTaskIds) {
    const task = await getTaskById(taskId).catch(() => null);
    await settleOrphanedCreativeDirectorRun(task);
  }

  // Handle orphaned tasks - reset for retry or create investigation task.
  // `agentMetadata` rides along so the retry can resume what this run left behind
  // (see handleOrphanedTask). Runs AFTER cleanupAgentWorktree above so the resume
  // pointer reflects what actually survived — a dirty tree aborts removal, leaving
  // the whole worktree in place.
  for (const { taskId, agentId, agentMetadata, agentStartedAt } of orphanedTaskIds) {
    await handleOrphanedTask(taskId, agentId, getTaskById, {
      agentMetadata,
      agentStartedAt,
      // `|| null`, not a bare boolean: a plain `false` would hard-override the
      // per-agent breadcrumb fallback, leaving this — the path that handles
      // essentially all boot recovery — with the marker as its ONLY signal. The
      // marker is best-effort (a stalled disk can blow the 1.5s shutdown grace,
      // or the write can simply fail), and when it's the only signal its loss
      // silently reinstates the original bug. `null` means "no verdict from me,
      // check the breadcrumb"; `true` still short-circuits it.
      interrupted: interruptedByRestart.has(agentId) || null,
    });
  }

  // Same release order as a normal completion (agentLifecycle): only after the
  // run is retired, its worktree cleaned and its task settled. Without it the
  // reservation outlives the process and every later status read parses it.
  // finishResource reads the journal synchronously and throws while it needs
  // recovery; that must not abort the marker clear and stranded-agent sweep below.
  for (const agentId of retiredAgentIds) await releaseAgentReservation(agentId);
  // Backstop for reservations a previous process leaked on any other exit path.
  // Sweep-start snapshot plus what this sweep retired: `completed` is terminal and a
  // dead owner pid cannot belong to an agent spawned meanwhile, so no re-read is needed.
  const retired = new Set(retiredAgentIds);
  await reconcileStaleAgentReservations({ agents: agents.map(a => retired.has(a.id) ? { ...a, status: 'completed' } : a) });
  // Same sweep, different kinds: publication residue is stamped, and only a
  // cli-run or mind-turn whose owning record proves completion is released.
  reconcileDeadOwnerReservations();

  // Consume the marker. Doing it AFTER the retry pass (not before) means a crash
  // mid-sweep leaves it in place and the next boot still classifies these agents
  // correctly; re-reading a stale marker is harmless because the sweep only ever
  // looks at agents still marked `running`.
  //
  // Gated on the marker EXISTING, not on it naming anyone: a truncated or
  // malformed marker parses to zero agent ids, and gating on the id count would
  // leave that file on disk to be re-read and re-parsed on every boot and every
  // 15-minute sweep, forever.
  if (shutdownMarker) await clearHostShutdownMarker();

  // Paused records whose task moved on without them. Not orphans (nothing died) and
  // not counted as such — but this is the sweep that already holds the agent list,
  // and leaving them stranded is what turns a later resume click into a duplicate.
  await retireStrandedPausedAgents(agents);

  // Trigger evaluation to spawn new agents for retried tasks
  if (cleanedCount > 0) {
    emitLog('info', `Cleaned up ${cleanedCount} orphaned agents, triggering evaluation`, { cleanedCount });
    setTimeout(() => {
      evaluateTasks().catch(err => {
        console.error(`❌ Failed to evaluate tasks after orphan cleanup: ${err.message}`);
      });
    }, 1000);
  }

  return cleanedCount;
}

/**
 * Handle an orphaned task - retry or create investigation.
 *
 * Every path that retires a run whose process vanished funnels through here —
 * the boot/health-check orphan sweep, `resetOrphanedTasks`, and the post-restart
 * completion recovery — so the retry's resume pointer is resolved HERE rather than
 * at each call site. Callers that know which agent died pass its `agentMetadata`;
 * without it the retry simply starts clean, which is the pre-existing behavior.
 *
 * @param {object} [options]
 * @param {object} [options.agentMetadata] - the dead agent's registered metadata
 *   (`isWorktree` / `sourceWorkspace` / `worktreeBranch` / `workspacePath`), used
 *   to work out whether its branch or worktree is worth resuming.
 * @param {string|number|null} [options.agentStartedAt] - when the dead run began. It is
 *   the window for the commit probe below; without it the probe is skipped rather than
 *   run unbounded, which would credit this task with any commit already in the repo (#3637).
 * @param {boolean|null} [options.interrupted] - the run died because PortOS itself
 *   was restarted, not because the agent failed. Such a run is requeued immediately
 *   WITHOUT charging orphan-retry budget or arming the orphan cooldown — see the
 *   retry-budget note below (#3202). Pass it when the caller knows (the orphan sweep
 *   reads the host-shutdown marker); leave it null to derive from `agentMetadata`.
 */
export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agentMetadata = null, agentStartedAt = null, interrupted = null } = {}) {
  // Callers that watched the agent die (the orphan sweep, which reads the
  // host-shutdown marker) say so explicitly. The ones that didn't —
  // `resetOrphanedTasks`, post-restart completion recovery — fall back to the
  // breadcrumb the abandon path stamped on the agent itself, so an interrupted
  // run is still recognized no matter which recovery path reaches it first.
  // Without this, correctness depended on `cleanupOrphanedAgents` happening to
  // run before `resetOrphanedTasks` in cos.js's boot sequence.
  const wasInterrupted = interrupted ?? (agentMetadata?.interruptedBy === HOST_SHUTDOWN_REASON);
  await consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted);
  if (isPrivateSecurityTask({ metadata: agentMetadata })) await removePrivateSecurityScratch(agentId);

  const task = await getTaskByIdFn(taskId).catch(() => null);
  if (!task) {
    emitLog('warn', `Could not find task ${taskId} for orphaned agent ${agentId}`, { taskId, agentId });
    return;
  }

  if (isDeliveryCleanupHeld(task.metadata)) return;
  const ctx = { taskId, agentId, task, agentMetadata, agentStartedAt };
  const guard = classifyOrphanGuard(task, agentMetadata);
  if (guard) return ORPHAN_GUARD_HANDLERS[guard](ctx);
  if (await completeIfCommitted(ctx)) return;

  const decision = decideOrphanRetry({ task, wasInterrupted, now: Date.now() });
  await applyOrphanDecision(ctx, decision);
}

/**
 * Consume the interrupted breadcrumb the moment it's honored, exactly as the
 * marker is consumed after the sweep. It is written once and would otherwise stay
 * on the agent record forever: `resetOrphanedTasks` keys on the most recent agent
 * per task, so a respawn that dies before creating its own agent record would keep
 * re-deriving `wasInterrupted` from this same stale breadcrumb — and, because an
 * interrupted run bypasses the cooldown, respawn on EVERY 15-minute sweep instead
 * of once per 30 minutes.
 */
async function consumeInterruptedBreadcrumb(agentId, agentMetadata, wasInterrupted) {
  if (!wasInterrupted || agentMetadata?.interruptedBy !== HOST_SHUTDOWN_REASON) return;
  await updateAgent(agentId, { metadata: { interruptedBy: null } })
    .catch(err => emitLog('warn', `Could not clear interrupted breadcrumb on ${agentId}: ${err.message}`, { agentId }));
}

async function removePrivateSecurityScratch(agentId) {
  const { privateSecurityScratchCwd } = await import('../lib/privateSecuritySandbox.js');
  await rm(privateSecurityScratchCwd(agentId), { recursive: true, force: true });
}

function skipTerminalBlockedOrphan({ taskId, agentId, task }) {
  emitLog('info', `⏭️ Skipping orphaned task ${taskId} — ${task.metadata.blockedCategory}`, { taskId, agentId });
}

function skipHandledOrphan({ taskId, agentId, task }) {
  const { blockedCategory } = task.metadata;
  emitLog('info', `⏭️ Skipping orphaned task ${taskId} — already handled (${blockedCategory})`, { taskId, agentId, blockedCategory });
}

function skipCompletedOrphan({ taskId, agentId }) {
  emitLog('debug', `⏭️ Skipping orphaned task ${taskId} — already completed`, { taskId, agentId });
}

async function recoverPrivateSecurityOrphan({ taskId, agentId, task }) {
  if (task.status === 'completed') return;
  await removePrivateSecurityScratch(agentId);
  const taskUpdate = await stampLiExecutionVerdict({ status: 'blocked', metadata: { ...task.metadata,
    blockedAt: new Date().toISOString(), blockedCategory: 'private-security-assessment-failed',
    blockedReason: 'Private assessment was interrupted. Inspect its local output and explicitly rerun it; no automatic investigation or provider fallback will run.',
  } }, task, { success: false });
  await updateTask(taskId, taskUpdate, task.taskType || 'user');
}

async function recoverRefusedOrphan({ taskId, task, agentMetadata, agentStartedAt }) {
  const interruption = agentMetadata.executionInterruption;
  // Do not let an older refused run block a newer invocation.
  const startedAt = toEpochMs(agentStartedAt);
  if (startedAt && toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]) > startedAt) return;
  await updateTask(taskId, await stampLiExecutionVerdict({ status: 'blocked', metadata: {
    ...task.metadata, ...clearedRetryHoldMetadata(), executionInterruption: interruption,
    blockedAt: new Date().toISOString(), blockedCategory: interruption.category,
    blockedReason: 'Provider safety refusal; automatic continuation, retries and provider fallback are disabled.',
  } }, task, { success: false }), task.taskType || 'user');
}

async function recoverDeepAuditOrphan({ taskId, agentId, task, agentMetadata, agentStartedAt }) {
  if (task.status !== 'in_progress') return;
  const held = isRetryHeld(task.metadata);
  if (held && task.metadata[RETRY_HOLD_KEY] !== agentId) return;
  if (!held) {
    const spawnedAt = toEpochMs(task.metadata?.[LAST_SPAWNED_AT_KEY]);
    const startedAt = toEpochMs(agentStartedAt);
    // A legacy/unknown owner cannot authorize parking a newer invocation.
    if (!spawnedAt || !startedAt || spawnedAt > startedAt) return;
    const agents = await getAgents();
    if (!Array.isArray(agents) || agents.some(entry => entry.taskId === taskId && entry.id !== agentId
      && (entry.status === 'running' || toEpochMs(entry.startedAt) > startedAt))) return;
  }
  const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata });
  await updateTask(taskId, { status: 'blocked', metadata: {
    ...resumePatch, ...clearedRetryHoldMetadata(), blockedCategory: 'deep-audit-partial',
    blockedReason: 'Interrupted Deep audit; inspect checkpoint and resume explicitly', blockedAt: new Date().toISOString(),
  } }, task.taskType || 'user', { expectedStatus: 'in_progress',
    expectedMetadata: held ? { [RETRY_HOLD_KEY]: agentId } : { [LAST_SPAWNED_AT_KEY]: task.metadata[LAST_SPAWNED_AT_KEY] },
  });
}

/**
 * The run's verdict is already persisted — it failed, and was budgeted a retry —
 * so this is not a fresh orphan: finish the transition the dead process started
 * (resolve the pointer, flip to `pending`, drop the marker) rather than charging
 * it orphan budget for a decision already made. Precedes the commit check: a failed
 * run that committed is exactly the case the resume pointer exists for, and
 * completing the task on that evidence would discard the retry finalizeAgent granted.
 */
async function recoverRetryHeldOrphan({ taskId, agentId, task, agentMetadata }) {
  const resumePatch = await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
    emitLog('warn', `Resume pointer for held task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });
    return {};
  });
  const targetBranch = resolveTaskTargetBranch(resumePatch);
  const result = await updateTask(taskId, {
    status: 'pending',
    metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }
  }, task.taskType || 'user', {
    expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },
  });
  if (result?.error) {
    emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });
    return;
  }
  if (result?.statusChanged) return;
  emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {
    taskId, agentId, branchName: targetBranch
  });
}

// One handler per `ORPHAN_GUARDS` reason; the precedence lives in that list.
const ORPHAN_GUARD_HANDLERS = {
  'terminal-block': skipTerminalBlockedOrphan,
  'private-security': recoverPrivateSecurityOrphan,
  'already-handled': skipHandledOrphan,
  'completed': skipCompletedOrphan,
  'content-refusal': recoverRefusedOrphan,
  'deep-audit': recoverDeepAuditOrphan,
  'retry-held': recoverRetryHeldOrphan,
};
for (const { reason } of ORPHAN_GUARDS) {
  if (!ORPHAN_GUARD_HANDLERS[reason]) throw new Error(`No orphan recovery handler for guard "${reason}"`);
}

/**
 * Did the agent actually commit work before it died? Scoped to the dead run's OWN
 * window (#3637): the retired task-id commit marker was never emitted by anything,
 * and an unbounded `git log` would credit this task with any commit in the repo —
 * including another agent's. `toEpochMs` accepts both shapes the agent record can
 * carry (the persisted ISO string and a raw epoch-ms number); without a start time
 * the probe is skipped.
 */
async function completeIfCommitted({ taskId, agentId, task, agentMetadata, agentStartedAt }) {
  const runStartedAt = toEpochMs(agentStartedAt);
  const commitFound = Number.isFinite(runStartedAt)
    && await committedDuringRun(agentMetadata?.workspacePath || ROOT_DIR, runStartedAt);
  if (!commitFound) return false;
  emitLog('info', `✅ Orphaned agent ${agentId} actually completed work - commit found for task ${taskId}`, { taskId, agentId });
  const taskUpdate = await stampLiExecutionVerdict({ status: 'completed' }, task, { success: true });
  await updateTask(taskId, taskUpdate, task.taskType || 'user');
  return true;
}

function logOrphanDecision(taskId, decision) {
  const { outcome, interrupted, retryCount, totalSpawns, cooldownMinutes, reason } = decision;
  if (outcome === 'requeue') {
    emitLog('info', interrupted
      ? `Resuming task ${taskId} interrupted by a PortOS restart (no retry charged; total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`
      : `Resetting orphaned task ${taskId} for retry (attempt ${retryCount}/${MAX_ORPHAN_RETRIES}, total spawns ${totalSpawns}/${MAX_TOTAL_SPAWNS})`, {
      taskId, retryCount, totalSpawns, interrupted, maxRetries: MAX_ORPHAN_RETRIES
    });
  } else if (outcome === 'cooldown') {
    emitLog('info', `⏳ Orphan retry for task ${taskId} in cooldown (${cooldownMinutes}m remaining)`, { taskId, cooldownMinutes, retryCount });
  } else {
    emitLog('warn', `Task ${taskId} blocked: ${reason}, creating investigation task`, { taskId, retryCount, totalSpawns });
  }
}

/** Persist the retry decision: requeue, orphan-cooldown block, or max-retries block + investigation. */
async function applyOrphanDecision({ taskId, agentId, task, agentMetadata }, decision) {
  logOrphanDecision(taskId, decision);
  const requeue = decision.outcome === 'requeue';
  // Point the retry at whatever the dead run left behind — its branch, or the
  // whole worktree when the process died mid-edit and the uncommitted work is
  // still sitting in it. Folded into THIS write rather than stamped separately
  // beforehand: the flip to `pending` emits `tasks:changed`, which can spawn the
  // retry immediately, so a pointer written afterwards could land too late.
  // Fails open to "start clean": requeueing the task matters far more than
  // resuming it, and a throw here would strand it in_progress until the next sweep.
  const resumePatch = requeue
    ? await resolveTaskResumePatch({ task, agentId, agentMetadata }).catch(err => {
      emitLog('warn', `Resume pointer for task ${taskId} could not be resolved: ${err.message}`, { taskId, agentId });
      return {};
    })
    : {};
  const outcome = buildOrphanOutcome({ decision, task, agentId, now: Date.now(), resumePatch });
  const taskUpdate = requeue ? outcome : await stampLiExecutionVerdict(outcome, task, { success: false });
  await updateTask(taskId, taskUpdate, task.taskType || 'user');
  if (decision.outcome === 'blocked') await fileOrphanInvestigation({ taskId, agentId, task, decision });
}

async function fileOrphanInvestigation({ taskId, agentId, task, decision }) {
  const { retryCount, totalSpawns, reason } = decision;
  const description = `[Auto-Fix] Investigate repeated agent orphaning for task ${taskId}

**Original Task**: ${(task.description || '').substring(0, 200)}
**Orphan Retries**: ${retryCount}
**Total Spawns**: ${totalSpawns}
**Last Orphaned Agent**: ${agentId}
**Blocked Reason**: ${reason}

This task has been blocked after ${totalSpawns} total agent spawns. Investigate:
1. Check CoS Runner logs for errors
2. Verify process spawning is working correctly
3. Look for resource constraints (memory, CPU)
4. Check for network/connection issues between services

Once the issue is resolved, task \`${taskId}\` is retried automatically — completing this task revives it.`;

  // Filed through the shared producer so this participates in the same
  // loop-aware approval policy, storm counter, and fingerprint dedup as every
  // other investigation (#3714), and so `affectedTasks` closes the loop: the
  // orphaned task comes back on its own when this one completes. Hardcoding
  // `approvalRequired: true` here is what turned one repeatedly-orphaned task
  // into a queue of approvals nobody asked for.
  await fileInvestigationTask({
    description,
    priority: 'HIGH',
    context: `Auto-generated from repeated orphan failures for task ${taskId}`,
    fingerprint: buildInvestigationFingerprint(task, { category: 'max-retries' }),
    affectedTasks: [taskId]
  }).catch(err => {
    emitLog('error', `Failed to create investigation task: ${err.message}`, { taskId, error: err.message });
  });
}