Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, feature] Daemon incident forensics: prime-agent incident CLI and agents-view notices (#2406)

Seth Kartenf0223369ac85
diff --git a/packages/coding-agent/.changes/agents-view-incident-notice.md b/packages/coding-agent/.changes/agents-view-incident-notice.md
new file mode 100644
index 0000000000..e5dd4a1df0
--- /dev/null
+++ b/packages/coding-agent/.changes/agents-view-incident-notice.md
@@ -0,0 +1 @@
+- Added daemon incident notices to the agents view: recent worker crashes, command-timeout bursts, and update restarts surface as a dismissible status line pointing at `prime-agent incident`.
diff --git a/packages/coding-agent/.changes/daemon-incident-cli.md b/packages/coding-agent/.changes/daemon-incident-cli.md
new file mode 100644
index 0000000000..9c956c2eeb
--- /dev/null
+++ b/packages/coding-agent/.changes/daemon-incident-cli.md
@@ -0,0 +1 @@
+- Added `prime-agent incident` to reconstruct daemon incidents from logs into an operator timeline (supervisor events, session anomalies, recovery actions) with `--since`/`--until` window and `--session` filters.
diff --git a/packages/coding-agent/src/cli/command-registry.ts b/packages/coding-agent/src/cli/command-registry.ts
index f160929ae9..b3cdbebcb8 100644
--- a/packages/coding-agent/src/cli/command-registry.ts
+++ b/packages/coding-agent/src/cli/command-registry.ts
@@ -86,6 +86,19 @@ export const COMMAND_SPECS: readonly CommandSpec[] = [
summary: "Inspect and safely clean up background services",
options: ["--fix Remove stale sockets and stop idle orphaned services", "--json Print JSON"],
},
+ {
+ path: ["incident"],
+ usage: "incident [--since <time>] [--until <time>] [--session <id>]",
+ summary: "Reconstruct a daemon incident from its logs",
+ description:
+ "Summarizes the daemon logs for a time window into an operator timeline: supervisor and worker events, session anomalies, and recovery actions. Times without a timezone are read as UTC, matching the log; the default window is the last 24 hours.",
+ options: [
+ "--since <time> Window start (ISO date/time, date, or HH:MM today; default: 24h ago)",
+ "--until <time> Window end (default: now)",
+ "--session <id> Only events naming this session id, worker id, or session name (prefix match)",
+ ],
+ examples: ['incident --since "2026-09-16T20:02" --until "2026-09-16T20:21"', "incident --session 2339fb7da605"],
+ },
{
path: ["shutdown"],
usage: "shutdown [--force] [--json]",
diff --git a/packages/coding-agent/src/cli/incident.ts b/packages/coding-agent/src/cli/incident.ts
new file mode 100644
index 0000000000..dbd9a87c67
--- /dev/null
+++ b/packages/coding-agent/src/cli/incident.ts
@@ -0,0 +1,1279 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import chalk from "chalk";
+import { getAgentLogPath, getLogsDir } from "../config.js";
+
+/**
+ * `prime-agent incident` reconstructs what the daemon did during a time window
+ * from its diagnostic logs, so an operator does not have to grep hundreds of
+ * raw log lines by hand. The primary source is the shared structured log
+ * (~/.prime/agent/logs/agent.jsonl, one JSON object per line, written by the
+ * coding-agent.daemon-supervisor, coding-agent.daemon, and ai.provider
+ * components). When that file is missing or unreadable, the newest per-daemon
+ * log file (~/.prime/agent/logs/<socket>.<hash>.log, plain-text lines) is used
+ * as a fallback.
+ *
+ * Message shapes below match the strings the supervisor and session worker
+ * actually emit (the DaemonSupervisor.log / DaemonMode.log call sites).
+ * Unknown warning lines fall through to a generic per-line summary so new log
+ * messages degrade to a readable timeline instead of disappearing.
+ */
+
+export type IncidentSeverity = "critical" | "error" | "warn" | "info";
+
+export type IncidentCategory = "supervisor" | "anomaly" | "recovery";
+
+export interface IncidentLogEntry {
+ timeMs: number;
+ level: string;
+ component: string;
+ msg: string;
+ socketPath?: string;
+ pid?: number;
+ /** Raw log-record fields, used for provider failure details (kind/status). */
+ fields: Record<string, unknown>;
+}
+
+export interface IncidentEvent {
+ timeMs: number;
+ severity: IncidentSeverity;
+ category: IncidentCategory;
+ eventClass: string;
+ subject: string;
+ summary: string;
+ tokens: string[];
+}
+
+export interface IncidentCommandOptions {
+ since?: string;
+ until?: string;
+ session?: string;
+}
+
+export interface IncidentReportOptions {
+ sinceMs: number;
+ untilMs: number;
+ session?: string;
+ source?: string;
+ scannedCount?: number;
+ skippedCount?: number;
+}
+
+export interface IncidentReport {
+ text: string;
+}
+
+export interface IncidentLogSource {
+ entries: IncidentLogEntry[];
+ scannedCount: number;
+ skippedCount: number;
+ source: string;
+}
+
+export class IncidentUsageError extends Error {}
+
+const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000;
+const STALL_GAP_MS = 10 * 60 * 1000;
+const ERROR_BURST_THRESHOLD = 3;
+/** A burst is several warnings/errors close together; isolated failures far apart are not one incident. */
+const ERROR_BURST_WINDOW_MS = 10 * 60 * 1000;
+/**
+ * A stall is repeated command timeouts close together, like the burst window
+ * above; timeouts further apart than this never merge into one stall, whatever
+ * the report window is. The agents-view dismissal horizon derives from the
+ * same bound via latestIncidentStallTimeoutBySubject.
+ */
+const TIMEOUT_STALL_WINDOW_MS = 30 * 60 * 1000;
+const SUMMARY_TRUNCATION = 120;
+const RECOVERY_BREAKDOWN_LIMIT = 4;
+const WORKER_SOCKET_PATTERN = /^(?:prime-agent-)?worker-[0-9a-f]+-([0-9a-f]{12})(?:\.sock)?$/;
+// Per-daemon logs are named `<socket basename>.<hash8>.log` (config.ts
+// getDaemonLogPath); the socket basename itself may lack `.sock` for custom
+// sockets and Windows named pipes, so key on the hash suffix instead.
+const DAEMON_LOG_FILE_PATTERN = /\.[0-9a-f]{8}\.log$/;
+const BURST_CLASSES = new Set(["command-failure", "auth", "diagnostic"]);
+const LIFECYCLE_CLASSES = new Set(["worker-start", "worker-stop", "worker-crash", "worker-passivation"]);
+
+// ---------------------------------------------------------------------------
+// Time parsing
+// ---------------------------------------------------------------------------
+
+const FULL_TIME_PATTERN =
+ /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.(\d{1,3}))?)?(Z|[+-]\d{2}(?::?\d{2})?)?$/;
+const TIME_ONLY_PATTERN = /^(\d{2}):(\d{2})$/;
+
+/**
+ * Parse a --since/--until bound. Times without a timezone are read as UTC,
+ * matching the timestamps the daemon logs; "20:02" means today at 20:02 UTC.
+ */
+export function parseIncidentTimeBound(value: string, now: Date, flag: string): number {
+ const raw = value.trim();
+ if (!raw) {
+ throw new IncidentUsageError(`${flag} requires a time.`);
+ }
+ const invalid = () =>
+ new IncidentUsageError(
+ `Invalid time for ${flag}: "${value}". Use "2026-09-16T20:02", "2026-09-16", or "20:02" (today, UTC).`,
+ );
+ const timeOnly = TIME_ONLY_PATTERN.exec(raw);
+ if (timeOnly) {
+ const hour = Number(timeOnly[1]);
+ const minute = Number(timeOnly[2]);
+ if (hour > 23 || minute > 59) {
+ throw invalid();
+ }
+ return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, minute);
+ }
+ const full = FULL_TIME_PATTERN.exec(raw);
+ if (!full) {
+ throw invalid();
+ }
+ const [, year, month, day, hour, minute, second, fraction, zone] = full;
+ const baseMs = Date.UTC(
+ Number(year),
+ Number(month) - 1,
+ Number(day),
+ hour !== undefined ? Number(hour) : 0,
+ minute !== undefined ? Number(minute) : 0,
+ second !== undefined ? Number(second) : 0,
+ fraction !== undefined ? Number(fraction.padEnd(3, "0")) : 0,
+ );
+ const base = new Date(baseMs);
+ if (
+ base.getUTCFullYear() !== Number(year) ||
+ base.getUTCMonth() !== Number(month) - 1 ||
+ base.getUTCDate() !== Number(day) ||
+ (hour !== undefined && base.getUTCHours() !== Number(hour)) ||
+ (minute !== undefined && base.getUTCMinutes() !== Number(minute))
+ ) {
+ throw invalid();
+ }
+ if (!zone || zone === "Z") {
+ return baseMs;
+ }
+ const sign = zone[0] === "-" ? -1 : 1;
+ const digits = zone.slice(1).replace(":", "");
+ const offsetMinute = Number(digits.slice(2) || 0);
+ const offsetMinutes = Number(digits.slice(0, 2)) * 60 + offsetMinute;
+ if (!Number.isFinite(offsetMinutes) || offsetMinutes >= 24 * 60 || offsetMinute >= 60) {
+ throw invalid();
+ }
+ return baseMs - sign * offsetMinutes * 60_000;
+}
+
+// ---------------------------------------------------------------------------
+// Log line parsing
+// ---------------------------------------------------------------------------
+
+function timestampToMs(ts: string): number {
+ const ms = Date.parse(ts);
+ return Number.isNaN(ms) ? Number.NaN : ms;
+}
+
+/** Parse one agent.jsonl line; malformed lines return undefined. */
+export function parseIncidentLogLine(line: string): IncidentLogEntry | undefined {
+ const trimmed = line.trim();
+ if (!trimmed) {
+ return undefined;
+ }
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(trimmed);
+ } catch {
+ return undefined;
+ }
+ if (typeof parsed !== "object" || parsed === null) {
+ return undefined;
+ }
+ const record = parsed as Record<string, unknown>;
+ if (typeof record.ts !== "string" || typeof record.msg !== "string") {
+ return undefined;
+ }
+ const timeMs = timestampToMs(record.ts);
+ if (!Number.isFinite(timeMs)) {
+ return undefined;
+ }
+ return {
+ timeMs,
+ level: typeof record.level === "string" ? record.level : "warn",
+ component: typeof record.component === "string" ? record.component : "unknown",
+ msg: record.msg,
+ ...(typeof record.socketPath === "string" ? { socketPath: record.socketPath } : {}),
+ ...(typeof record.pid === "number" ? { pid: record.pid } : {}),
+ fields: record,
+ };
+}
+
+/** Parse one per-daemon log line: `[<ISO>] supervisor: <msg>` or `[<ISO>] <msg>`. */
+export function parseIncidentDaemonLogLine(line: string): IncidentLogEntry | undefined {
+ const match = /^\[([^\]]+)\]\s*(.*)$/.exec(line.trim());
+ if (!match) {
+ return undefined;
+ }
+ const [, ts, rest] = match;
+ if (!ts || rest === undefined || !rest.trim()) {
+ return undefined;
+ }
+ const timeMs = timestampToMs(ts);
+ if (!Number.isFinite(timeMs)) {
+ return undefined;
+ }
+ const supervisorLine = rest.startsWith("supervisor:");
+ const msg = supervisorLine ? rest.slice("supervisor:".length).trim() : rest;
+ if (!msg) {
+ return undefined;
+ }
+ return {
+ timeMs,
+ level: "warn",
+ component: supervisorLine ? "coding-agent.daemon-supervisor" : "coding-agent.daemon",
+ msg,
+ fields: {},
+ };
+}
+
+// ---------------------------------------------------------------------------
+// Classification
+// ---------------------------------------------------------------------------
+
+function firstLine(text: string): string {
+ return text.split("\n")[0]!.trim();
+}
+
+function truncateText(text: string, max: number): string {
+ const trimmed = text.trim();
+ return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max)}...`;
+}
+
+function errorMessage(message: string): string {
+ return message.replace(/^Error:\s*/, "");
+}
+
+function workerIdFromSocketPath(socketPath: string | undefined): string | undefined {
+ if (!socketPath) {
+ return undefined;
+ }
+ // Split on both separators so Windows named-pipe paths (`\\.\pipe\...`)
+ // resolve to their last segment on any platform, not only on win32.
+ const name = socketPath.split(/[\\/]/).pop()!;
+ return WORKER_SOCKET_PATTERN.exec(name)?.[1];
+}
+
+/** One log sighting of a worker id owning a pid at a timestamp. */
+export interface WorkerPidSighting {
+ timeMs: number;
+ workerId: string;
+}
+
+/**
+ * pid -> worker-id sightings, ordered by time. A pid can be reused by a later
+ * worker, so attribution picks the owner at the event time, not the union of
+ * every id ever seen on the pid.
+ */
+export type WorkerPidMap = ReadonlyMap<number, ReadonlyArray<WorkerPidSighting>>;
+
+/** Map pid -> worker-id sightings from entries whose socket path names a worker socket. */
+export function collectWorkerPidMap(entries: readonly IncidentLogEntry[]): WorkerPidMap {
+ const map = new Map<number, WorkerPidSighting[]>();
+ for (const entry of entries) {
+ const workerId = workerIdFromSocketPath(entry.socketPath);
+ if (workerId === undefined || entry.pid === undefined) {
+ continue;
+ }
+ const sightings = map.get(entry.pid) ?? [];
+ sightings.push({ timeMs: entry.timeMs, workerId });
+ map.set(entry.pid, sightings);
+ }
+ for (const sightings of map.values()) {
+ sightings.sort((a, b) => a.timeMs - b.timeMs);
+ }
+ return map;
+}
+
+/** The worker that owned `pid` at `timeMs`: the latest sighting at or before it. */
+function workerIdForPid(workerPids: WorkerPidMap, pid: number | undefined, timeMs: number): string | undefined {
+ if (pid === undefined) {
+ return undefined;
+ }
+ const sightings = workerPids.get(pid);
+ if (sightings === undefined || sightings.length === 0) {
+ return undefined;
+ }
+ let workerId: string | undefined;
+ for (const sighting of sightings) {
+ if (sighting.timeMs <= timeMs) {
+ workerId = sighting.workerId;
+ } else {
+ break;
+ }
+ }
+ return workerId;
+}
+
+function classifyCommandFailure(
+ command: string,
+ error: string,
+): {
+ severity: IncidentSeverity;
+ eventClass: string;
+ summary: string;
+ tokens: string[];
+} {
+ const err = firstLine(error);
+ if (err.includes("Timed out waiting for daemon worker response")) {
+ return {
+ severity: "error",
+ eventClass: "timeout",
+ summary: `command ${command} failed: timed out waiting for worker response`,
+ tokens: [],
+ };
+ }
+ if (/authentication failed/i.test(err)) {
+ return {
+ severity: "error",
+ eventClass: "auth",
+ summary: `command ${command} failed: worker authentication failed`,
+ tokens: [],
+ };
+ }
+ if (err.includes("Session worker is starting")) {
+ return {
+ severity: "warn",
+ eventClass: "command-failure",
+ summary: `command ${command} failed: session worker is starting`,
+ tokens: [],
+ };
+ }
+ if (err.includes("preparing an update")) {
+ return {
+ severity: "info",
+ eventClass: "command-failure",
+ summary: `command ${command} failed: update restart in preparation`,
+ tokens: [],
+ };
+ }
+ if (err.includes("Session worker is recovering")) {
+ return {
+ severity: "warn",
+ eventClass: "command-failure",
+ summary: `command ${command} failed: session worker is recovering`,
+ tokens: [],
+ };
+ }
+ const unknownSession = /Unknown active session: (\S+)/.exec(err);
+ if (unknownSession) {
+ const sessionId = unknownSession[1]!;
+ return {
+ severity: "warn",
+ eventClass: "command-failure",
+ summary: `command ${command} failed: unknown active session ${sessionId}`,
+ tokens: [sessionId],
+ };
+ }
+ // Quoted strings can carry session names (e.g. Agent name "Faerie").
+ const quotedNames = [...err.matchAll(/"([^"\n]+)"/g)].map((match) => match[1]!);
+ return {
+ severity: "warn",
+ eventClass: "command-failure",
+ summary: `command ${command} failed: ${truncateText(errorMessage(err), 100)}`,
+ tokens: quotedNames,
+ };
+}
+
+/** Classify the body of a `Session worker <id> stderr: <body>` log line. */
+function classifyWorkerStderrBody(
+ workerId: string,
+ body: string,
+ includeGeneric = true,
+): Omit<IncidentEvent, "timeMs" | "category"> | undefined {
+ const tokens = [workerId];
+ if (!body.trim() || /^\s*at\s/.test(body)) {
+ // Stack frames and blank lines belong to the previous stderr event.
+ return undefined;
+ }
+ const line = firstLine(body);
+ if (/^Prime Agent daemon listening on \S+$/.test(line)) {
+ return {
+ severity: "info",
+ eventClass: "worker-start",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} started`,
+ tokens,
+ };
+ }
+ const crash = /^(?:uncaught exception|unhandled rejection): (.+)$/.exec(line);
+ if (crash) {
+ return {
+ severity: "critical",
+ eventClass: "worker-crash",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} crashed: ${truncateText(line, 100)}`,
+ tokens,
+ };
+ }
+ const shutdown = /^shutting down \(exit (\d+)\); closing (\d+) active session\(s\)$/.exec(line);
+ if (shutdown) {
+ return {
+ severity: "info",
+ eventClass: "worker-stop",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} stopped (exit ${shutdown[1]}, ${shutdown[2]} session(s) closed)`,
+ tokens,
+ };
+ }
+ const signal = /^received (\S+); shutting down$/.exec(line);
+ if (signal) {
+ return {
+ severity: "info",
+ eventClass: "worker-stop",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} received ${signal[1]}; shutting down`,
+ tokens,
+ };
+ }
+ const stopRequested = /^shutdown command received over socket; (\d+) active session\(s\) will be closed$/.exec(line);
+ if (stopRequested) {
+ return {
+ severity: "info",
+ eventClass: "worker-stop",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} stop requested (${stopRequested[1]} active session(s))`,
+ tokens,
+ };
+ }
+ const passivated = /^Passivated idle child sessionId=(\S+) name=("[^"]*"|\S+) idleMinutes=(\d+)$/.exec(line);
+ if (passivated) {
+ // A missing session name logs as `name=""`; that empty token must not
+ // become a filter key.
+ const name = passivated[2].replace(/^"|"$/g, "");
+ return {
+ severity: "info",
+ eventClass: "worker-passivation",
+ subject: `session ${passivated[1]}`,
+ summary: `passivated idle child session ${passivated[1]} (idle ${passivated[3]}m)`,
+ tokens: name ? [workerId, passivated[1]!, name] : [workerId, passivated[1]!],
+ };
+ }
+ if (!includeGeneric) {
+ return undefined;
+ }
+ return {
+ severity: "warn",
+ eventClass: "worker-stderr",
+ subject: `worker ${workerId}`,
+ summary: `worker ${workerId} stderr: ${truncateText(line, 100)}`,
+ tokens,
+ };
+}
+
+function recoveryBreakdown(operations: string): { count: number; breakdown: string } {
+ const list = operations
+ .split(",")
+ .map((operation) => operation.trim())
+ .filter(Boolean);
+ if (list.length === 0) {
+ return { count: 0, breakdown: "none listed" };
+ }
+ const counts = new Map<string, number>();
+ for (const operation of list) {
+ counts.set(operation, (counts.get(operation) ?? 0) + 1);
+ }
+ const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+ const top = ranked.slice(0, RECOVERY_BREAKDOWN_LIMIT);
+ const shownCount = top.reduce((sum, [, count]) => sum + count, 0);
+ const rest = ranked.length > RECOVERY_BREAKDOWN_LIMIT ? `, +${list.length - shownCount} more` : "";
+ const breakdown = top.map(([operation, count]) => `${operation} x${count}`).join(", ");
+ return { count: list.length, breakdown: `${breakdown}${rest}` };
+}
+
+function event(
+ entry: IncidentLogEntry,
+ severity: IncidentSeverity,
+ category: IncidentCategory,
+ eventClass: string,
+ subject: string,
+ summary: string,
+ tokens: string[] = [],
+): IncidentEvent {
+ return { timeMs: entry.timeMs, severity, category, eventClass, subject, summary, tokens };
+}
+
+/**
+ * Key events on the entry's socket path so anomalies and aggregation never
+ * mix entries from different daemons sharing one agent.jsonl; entries without
+ * a socket field (the per-daemon fallback log) key on `daemon`.
+ */
+function daemonSubject(entry: IncidentLogEntry): string {
+ return entry.socketPath ?? "daemon";
+}
+
+/** Classify a provider-failure entry into an aggregate-friendly anomaly event. */
+function providerFailureEvent(entry: IncidentLogEntry, workerPids: WorkerPidMap): IncidentEvent | undefined {
+ const workerId = workerIdForPid(workerPids, entry.pid, entry.timeMs);
+ const subject = workerId ? `worker ${workerId}` : entry.pid !== undefined ? `pid ${entry.pid}` : "provider";
+ if (entry.msg !== "provider stream failure") {
+ return event(
+ entry,
+ "error",
+ "anomaly",
+ "provider",
+ subject,
+ truncateText(firstLine(entry.msg), SUMMARY_TRUNCATION),
+ [],
+ );
+ }
+ const kind = typeof entry.fields.kind === "string" ? entry.fields.kind : "unknown";
+ const statusText = typeof entry.fields.status === "number" ? ` ${entry.fields.status}` : "";
+ return {
+ timeMs: entry.timeMs,
+ severity: "error",
+ category: "anomaly",
+ eventClass: "provider",
+ subject,
+ summary: `provider stream failure (${kind}${statusText}) for ${subject}`,
+ tokens: workerId ? [workerId] : [],
+ };
+}
+
+/** Classify one log entry into an incident event; undefined when the entry is noise. */
+export function classifyIncidentEntry(entry: IncidentLogEntry, workerPids: WorkerPidMap): IncidentEvent | undefined {
+ const msg = firstLine(entry.msg);
+ const workerId = workerIdFromSocketPath(entry.socketPath);
+
+ if (entry.component === "ai.provider") {
+ return providerFailureEvent(entry, workerPids);
+ }
+
+ const stderr = /^Session worker ([0-9a-f]{12}) stderr: ?([\s\S]*)$/.exec(entry.msg);
+ if (stderr) {
+ const body = classifyWorkerStderrBody(stderr[1]!, stderr[2] ?? "");
+ if (!body) {
+ return undefined;
+ }
+ return { ...body, timeMs: entry.timeMs, category: "supervisor" };
+ }
+
+ if (/^Prime Agent daemon supervisor \S+ listening on \S+$/.test(msg)) {
+ return event(
+ entry,
+ "info",
+ "supervisor",
+ "supervisor-start",
+ daemonSubject(entry),
+ "daemon supervisor listening",
+ );
+ }
+ const startupFailed = /^Daemon supervisor startup failed: (.+)$/.exec(msg);
+ if (startupFailed) {
+ const err = startupFailed[1]!;
+ if (/lock file is already being held/i.test(err)) {
+ return event(
+ entry,
+ "warn",
+ "supervisor",
+ "supervisor-start",
+ daemonSubject(entry),
+ "supervisor startup blocked: another daemon holds the lock",
+ );
+ }
+ return event(
+ entry,
+ "error",
+ "supervisor",
+ "supervisor-start",
+ daemonSubject(entry),
+ `supervisor startup failed: ${truncateText(errorMessage(firstLine(err)), 100)}`,
+ );
+ }
+ const supervisorCommand = /^Supervisor command (\S+) failed: (.+)$/.exec(msg);
+ if (supervisorCommand) {
+ const classified = classifyCommandFailure(supervisorCommand[1]!, supervisorCommand[2]!);
+ return event(
+ entry,
+ classified.severity,
+ "supervisor",
+ classified.eventClass,
+ daemonSubject(entry),
+ classified.summary,
+ classified.tokens,
+ );
+ }
+ const daemonCommand = /^daemon command "([^"]+)" failed: (.+)$/.exec(msg);
+ if (daemonCommand) {
+ const classified = classifyCommandFailure(daemonCommand[1]!, daemonCommand[2]!);
+ return event(
+ entry,
+ classified.severity,
+ "supervisor",
+ classified.eventClass,
+ workerId ? `worker ${workerId}` : "worker",
+ classified.summary,
+ // Keep the session ids/names classified out of the error, not just the worker id.
+ workerId ? [workerId, ...classified.tokens] : classified.tokens,
+ );
+ }
+ const catchUp = /^(?:Failed|could not)(?: to)? catch up (?:snapshot )?client \S+(?: for (\S+))?: (.+)$/.exec(msg);
+ if (catchUp) {
+ const sessionId = catchUp[1];
+ const err = catchUp[2]!;
+ const timedOut = err.includes("Timed out waiting for daemon worker response");
+ return event(
+ entry,
+ timedOut ? "error" : "warn",
+ "supervisor",
+ timedOut ? "timeout" : "command-failure",
+ sessionId ? `session ${sessionId}` : daemonSubject(entry),
+ `client catch-up failed${sessionId ? ` for session ${sessionId}` : ""}: ${truncateText(errorMessage(firstLine(err)), 100)}`,
+ sessionId ? [sessionId] : [],
+ );
+ }
+ const heartbeats = /^Could not list heartbeats from a worker: (.+)$/.exec(msg);
+ if (heartbeats) {
+ const timedOut = heartbeats[1]!.includes("Timed out waiting for daemon worker response");
+ return event(
+ entry,
+ timedOut ? "error" : "warn",
+ "supervisor",
+ timedOut ? "timeout" : "command-failure",
+ daemonSubject(entry),
+ `worker heartbeat list failed: ${truncateText(errorMessage(firstLine(heartbeats[1]!)), 100)}`,
+ );
+ }
+
+ const recovered = /^Recovered worker (\S+) without replaying uncertain operations: (.+)$/.exec(msg);
+ if (recovered) {
+ const worker = recovered[1]!;
+ const { count, breakdown } = recoveryBreakdown(recovered[2]!);
+ return event(
+ entry,
+ "warn",
+ "recovery",
+ "recovery-replay",
+ `worker ${worker}`,
+ `worker ${worker} recovered; ${count} uncertain operation${count === 1 ? "" : "s"} not replayed (${breakdown})`,
+ [worker],
+ );
+ }
+ const recoveredPlain = /^Recovered worker (\S+)$/.exec(msg);
+ if (recoveredPlain) {
+ const worker = recoveredPlain[1]!;
+ return event(entry, "info", "recovery", "recovery-replay", `worker ${worker}`, `worker ${worker} recovered`, [
+ worker,
+ ]);
+ }
+ const adoptFailed = /^Could not adopt worker (\S+): (.+)$/.exec(msg);
+ if (adoptFailed) {
+ const worker = adoptFailed[1]!;
+ return event(
+ entry,
+ "warn",
+ "recovery",
+ "recovery-failure",
+ `worker ${worker}`,
+ `could not adopt worker ${worker}: ${truncateText(errorMessage(firstLine(adoptFailed[2]!)), 100)}`,
+ [worker],
+ );
+ }
+ const recoverFailed = /^Could not recover worker (\S+): (.+)$/.exec(msg);
+ if (recoverFailed) {
+ const worker = recoverFailed[1]!;
+ return event(
+ entry,
+ "error",
+ "recovery",
+ "recovery-failure",
+ `worker ${worker}`,
+ `could not recover worker ${worker}: ${truncateText(errorMessage(firstLine(recoverFailed[2]!)), 100)}`,
+ [worker],
+ );
+ }
+ const failedAfterRetries = /^Worker (\S+) failed after three recovery attempts$/.exec(msg);
+ if (failedAfterRetries) {
+ const worker = failedAfterRetries[1]!;
+ return event(
+ entry,
+ "error",
+ "recovery",
+ "recovery-failure",
+ `worker ${worker}`,
+ `worker ${worker} failed after three recovery attempts`,
+ [worker],
+ );
+ }
+ const unresponsive = /^Worker (\S+) is unresponsive; parked failed after \d+ probe rounds$/.exec(msg);
+ if (unresponsive) {
+ const worker = unresponsive[1]!;
+ return event(entry, "error", "recovery", "recovery-failure", `worker ${worker}`, msg, [worker]);
+ }
+ const reclaimed = /^Reclaimed stale registration for stopped worker (\S+)$/.exec(msg);
+ if (reclaimed) {
+ const worker = reclaimed[1]!;
+ return event(
+ entry,
+ "info",
+ "recovery",
+ "recovery-action",
+ `worker ${worker}`,
+ `reclaimed stale registration for stopped worker ${worker}`,
+ [worker],
+ );
+ }
+ const migrated = /^Migrated (\d+) scheduled jobs into session artifacts$/.exec(msg);
+ if (migrated) {
+ return event(
+ entry,
+ "info",
+ "recovery",
+ "recovery-action",
+ daemonSubject(entry),
+ `migrated ${migrated[1]} scheduled jobs into session artifacts`,
+ );
+ }
+ const replacement = /^launched replacement supervisor on \S+$/.exec(msg);
+ if (replacement) {
+ return event(entry, "info", "recovery", "recovery-action", daemonSubject(entry), msg);
+ }
+ const woke = /^Woke session worker for a due scheduled job: \S+$/.exec(msg);
+ if (woke) {
+ // A scheduled wake is a supervisor lifecycle action, not crash recovery.
+ return event(entry, "info", "supervisor", "supervisor-action", daemonSubject(entry), msg);
+ }
+
+ const evictedIdle = /^Evicted idle worker (\S+) root=\S* idleMinutes=(\d+) sessions=(\d+)$/.exec(msg);
+ if (evictedIdle) {
+ const worker = evictedIdle[1]!;
+ return event(
+ entry,
+ "info",
+ "supervisor",
+ "worker-stop",
+ `worker ${worker}`,
+ `evicted idle worker ${worker} (idle ${evictedIdle[2]}m, ${evictedIdle[3]} session(s))`,
+ [worker],
+ );
+ }
+ const evictedEmpty = /^Evicted empty session worker (\S+) root=\S+ on last client detach$/.exec(msg);
+ if (evictedEmpty) {
+ const worker = evictedEmpty[1]!;
+ return event(
+ entry,
+ "info",
+ "supervisor",
+ "worker-stop",
+ `worker ${worker}`,
+ `evicted empty session worker ${worker} on last detach`,
+ [worker],
+ );
+ }
+
+ if (entry.component === "coding-agent.daemon") {
+ // The worker logs its own lifecycle lines; reusing the stderr-body
+ // classifier gives them the same summaries as the supervisor's stderr
+ // forward, so the duplicated log pair collapses in the timeline.
+ if (workerId !== undefined) {
+ const body = classifyWorkerStderrBody(workerId, msg, false);
+ if (body) {
+ return { ...body, timeMs: entry.timeMs, category: "supervisor" };
+ }
+ } else if (/^(?:uncaught exception|unhandled rejection): /.test(msg)) {
+ return event(
+ entry,
+ "critical",
+ "supervisor",
+ "worker-crash",
+ "worker",
+ `worker crashed: ${truncateText(msg, 100)}`,
+ );
+ }
+ }
+
+ if (entry.component === "coding-agent.daemon-supervisor" || entry.component === "coding-agent.daemon") {
+ // Unknown diagnostics still matter during an incident; summarize them.
+ return event(
+ entry,
+ entry.level === "error" ? "error" : "warn",
+ "supervisor",
+ "diagnostic",
+ workerId ? `worker ${workerId}` : daemonSubject(entry),
+ truncateText(msg, SUMMARY_TRUNCATION),
+ );
+ }
+ return undefined;
+}
+
+/**
+ * Classify every entry, dropping the duplicate event the daemon writes both
+ * to the structured log and to the worker stderr forward (same summary, same
+ * second).
+ */
+export function collectIncidentEvents(entries: readonly IncidentLogEntry[], workerPids: WorkerPidMap): IncidentEvent[] {
+ const events: IncidentEvent[] = [];
+ const lastSeen = new Map<string, { timeMs: number; component: string }>();
+ for (const entry of entries) {
+ const incident = classifyIncidentEntry(entry, workerPids);
+ if (!incident) {
+ continue;
+ }
+ // The daemon writes the same lifecycle event both to the structured log
+ // (coding-agent.daemon) and to the worker stderr forward
+ // (coding-agent.daemon-supervisor); drop the second copy when it lands
+ // within 2s. A repeat from the SAME component is a real lifecycle
+ // transition — a worker restarted on its durable id within 2s — and is
+ // never a duplicate.
+ if (LIFECYCLE_CLASSES.has(incident.eventClass)) {
+ const key = `${incident.category}|${incident.summary}`;
+ const previous = lastSeen.get(key);
+ if (
+ previous !== undefined &&
+ previous.component !== entry.component &&
+ Math.abs(incident.timeMs - previous.timeMs) <= 2000
+ ) {
+ continue;
+ }
+ lastSeen.set(key, { timeMs: incident.timeMs, component: entry.component });
+ }
+ events.push(incident);
+ }
+ return events;
+}
+
+// ---------------------------------------------------------------------------
+// Anomaly computation
+// ---------------------------------------------------------------------------
+
+function formatIncidentDuration(ms: number): string {
+ const totalSeconds = Math.max(1, Math.round(ms / 1000));
+ const seconds = totalSeconds % 60;
+ const minutes = Math.floor(totalSeconds / 60) % 60;
+ const hours = Math.floor(totalSeconds / 3600);
+ if (hours > 0) {
+ return `${hours}h${minutes > 0 ? `${minutes}m` : ""}`;
+ }
+ if (minutes > 0) {
+ return `${minutes}m${seconds > 0 ? `${seconds}s` : ""}`;
+ }
+ return `${seconds}s`;
+}
+
+function maxSeverity(events: readonly IncidentEvent[]): IncidentSeverity {
+ let severity: IncidentSeverity = "info";
+ for (const item of events) {
+ if (item.severity === "critical") return "critical";
+ if (item.severity === "error") severity = "error";
+ else if (item.severity === "warn" && severity !== "error") severity = "warn";
+ }
+ return severity;
+}
+
+/**
+ * The densest run of time-sorted events within windowMs of each other,
+ * returned as the run's start index and length. Shared by the stall and burst
+ * scans: only events close together form one incident, so isolated events far
+ * apart never merge, whatever the report window is.
+ */
+function densestWindowRun(items: readonly IncidentEvent[], windowMs: number): { start: number; count: number } {
+ let bestStart = 0;
+ let bestCount = 0;
+ let start = 0;
+ for (let end = 0; end < items.length; end++) {
+ while (items[end]!.timeMs - items[start]!.timeMs > windowMs) {
+ start++;
+ }
+ const count = end - start + 1;
+ if (count > bestCount) {
+ bestCount = count;
+ bestStart = start;
+ }
+ }
+ return { start: bestStart, count: bestCount };
+}
+
+/** Compute stall, error-burst, and event-gap anomaly lines from classified events. */
+export function computeIncidentAnomalies(events: readonly IncidentEvent[]): IncidentEvent[] {
+ const bySubject = new Map<string, IncidentEvent[]>();
+ for (const incident of events) {
+ if (incident.category === "anomaly") {
+ continue;
+ }
+ const group = bySubject.get(incident.subject) ?? [];
+ group.push(incident);
+ bySubject.set(incident.subject, group);
+ }
+
+ const anomalies: IncidentEvent[] = [];
+ const push = (timeMs: number, severity: IncidentSeverity, subject: string, summary: string) => {
+ anomalies.push({ timeMs, severity, category: "anomaly", eventClass: "anomaly", subject, summary, tokens: [] });
+ };
+
+ for (const [subject, group] of bySubject) {
+ group.sort((a, b) => a.timeMs - b.timeMs);
+ const timeouts = group.filter((incident) => incident.eventClass === "timeout");
+ if (timeouts.length >= 2) {
+ // Only timeouts within TIMEOUT_STALL_WINDOW_MS of each other form a
+ // stall; two isolated timeouts hours apart in a long window are not one.
+ const run = densestWindowRun(timeouts, TIMEOUT_STALL_WINDOW_MS);
+ if (run.count >= 2) {
+ const cluster = timeouts.slice(run.start, run.start + run.count);
+ const span = cluster[cluster.length - 1]!.timeMs - cluster[0]!.timeMs;
+ push(
+ cluster[0]!.timeMs,
+ "error",
+ subject,
+ `${subject}: ${cluster.length} command timeouts over ${formatIncidentDuration(span)}`,
+ );
+ }
+ }
+ const burst = group.filter((incident) => BURST_CLASSES.has(incident.eventClass));
+ if (burst.length >= ERROR_BURST_THRESHOLD) {
+ // Only events within ERROR_BURST_WINDOW_MS of each other form a burst;
+ // three isolated warnings days apart in a long window are not one.
+ const run = densestWindowRun(burst, ERROR_BURST_WINDOW_MS);
+ if (run.count >= ERROR_BURST_THRESHOLD) {
+ const cluster = burst.slice(run.start, run.start + run.count);
+ const span = cluster[cluster.length - 1]!.timeMs - cluster[0]!.timeMs;
+ push(
+ cluster[0]!.timeMs,
+ maxSeverity(cluster),
+ subject,
+ `${subject}: ${cluster.length} warnings/errors over ${formatIncidentDuration(span)}`,
+ );
+ }
+ }
+ if (subject.startsWith("session ")) {
+ for (let index = 1; index < group.length; index++) {
+ const gap = group[index]!.timeMs - group[index - 1]!.timeMs;
+ if (gap >= STALL_GAP_MS) {
+ push(
+ group[index - 1]!.timeMs,
+ "warn",
+ subject,
+ `${subject}: ${formatIncidentDuration(gap)} event gap (no logged events)`,
+ );
+ }
+ }
+ }
+ }
+ return anomalies.sort((a, b) => a.timeMs - b.timeMs);
+}
+
+/**
+ * Per subject, the latest timeout of the stall cluster computeIncidentAnomalies
+ * reports — the densest run within TIMEOUT_STALL_WINDOW_MS. The agents-view
+ * notice anchors its dismissal horizon there, so the notice, its horizon, and
+ * the reported cluster always describe the same incident, even when the window
+ * holds several stalls on one subject.
+ */
+export function latestIncidentStallTimeoutBySubject(events: readonly IncidentEvent[]): Map<string, number> {
+ const timeoutsBySubject = new Map<string, IncidentEvent[]>();
+ for (const incident of events) {
+ if (incident.eventClass !== "timeout") {
+ continue;
+ }
+ const group = timeoutsBySubject.get(incident.subject) ?? [];
+ group.push(incident);
+ timeoutsBySubject.set(incident.subject, group);
+ }
+ const latestBySubject = new Map<string, number>();
+ for (const [subject, timeouts] of timeoutsBySubject) {
+ // Callers pass time-sorted events; sort defensively so the run scan
+ // (which assumes ascending times) never sees them out of order.
+ timeouts.sort((a, b) => a.timeMs - b.timeMs);
+ const run = densestWindowRun(timeouts, TIMEOUT_STALL_WINDOW_MS);
+ if (run.count >= 2) {
+ latestBySubject.set(subject, timeouts[run.start + run.count - 1]!.timeMs);
+ }
+ }
+ return latestBySubject;
+}
+
+// ---------------------------------------------------------------------------
+// Timeline rendering
+// ---------------------------------------------------------------------------
+
+function formatIncidentTime(ms: number): string {
+ const date = new Date(ms);
+ const pad = (value: number) => String(value).padStart(2, "0");
+ return `${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())} ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}`;
+}
+
+function colorizeSeverity(severity: IncidentSeverity, label: string): string {
+ switch (severity) {
+ case "critical":
+ case "error":
+ return chalk.red(label);
+ case "warn":
+ return chalk.yellow(label);
+ case "info":
+ return chalk.dim(label);
+ }
+}
+
+interface AggregatedEvent {
+ first: IncidentEvent;
+ last: IncidentEvent;
+ count: number;
+}
+
+function aggregateIncidentEvents(events: readonly IncidentEvent[]): AggregatedEvent[] {
+ const groups = new Map<string, AggregatedEvent>();
+ for (const incident of events) {
+ const key = `${incident.category}|${incident.subject}|${incident.summary}`;
+ const existing = groups.get(key);
+ if (existing) {
+ existing.count += 1;
+ if (incident.timeMs > existing.last.timeMs) {
+ existing.last = incident;
+ }
+ if (incident.timeMs < existing.first.timeMs) {
+ existing.first = incident;
+ }
+ } else {
+ groups.set(key, { first: incident, last: incident, count: 1 });
+ }
+ }
+ return [...groups.values()].sort((a, b) => a.first.timeMs - b.first.timeMs);
+}
+
+function sessionMatches(incident: IncidentEvent, session: string): boolean {
+ // An empty token (e.g. a missing session name) is a prefix of every value
+ // and must not match every session filter.
+ return incident.tokens.some((token) => token !== "" && (token.startsWith(session) || session.startsWith(token)));
+}
+
+/**
+ * Build the full incident timeline text for a window: classified events are
+ * grouped into Supervisor events / Session anomalies / Recovery sections,
+ * repeated identical events are aggregated with counts, and per-subject
+ * stalls, error bursts, and event gaps are surfaced as anomalies.
+ */
+export function buildIncidentReport(
+ entries: readonly IncidentLogEntry[],
+ options: IncidentReportOptions,
+): IncidentReport {
+ const workerPids = collectWorkerPidMap(entries);
+ const allEvents = collectIncidentEvents(entries, workerPids);
+ let events = allEvents.filter(
+ (incident) => incident.timeMs >= options.sinceMs && incident.timeMs <= options.untilMs,
+ );
+ let filteredBySession = false;
+ if (options.session !== undefined) {
+ const session = options.session;
+ const matching = events.filter((incident) => sessionMatches(incident, session));
+ if (matching.length === 0 && events.length > 0) {
+ return {
+ text: `No daemon events between ${formatIncidentTime(options.sinceMs)} and ${formatIncidentTime(options.untilMs)} UTC reference session "${options.session}".`,
+ };
+ }
+ events = matching;
+ filteredBySession = true;
+ }
+ const anomalies = computeIncidentAnomalies(events.filter((incident) => incident.category !== "anomaly"));
+ const timelineEvents = [...events, ...anomalies];
+
+ const sections: Array<{ title: string; lines: string[] }> = [];
+ for (const [title, category] of [
+ ["Supervisor events", "supervisor"],
+ ["Session anomalies", "anomaly"],
+ ["Recovery", "recovery"],
+ ] as const) {
+ const aggregated = aggregateIncidentEvents(timelineEvents.filter((incident) => incident.category === category));
+ const lines = aggregated.map((group) => {
+ const suffix = group.count > 1 ? ` (x${group.count}, until ${formatIncidentTime(group.last.timeMs)})` : "";
+ return ` ${formatIncidentTime(group.first.timeMs)} ${colorizeSeverity(group.first.severity, group.first.severity.padEnd(8))} ${group.first.summary}${suffix}`;
+ });
+ if (lines.length === 0) {
+ lines.push(` ${chalk.dim("(none)")}`);
+ }
+ sections.push({ title, lines });
+ }
+
+ const header = ["Prime Agent incident timeline"];
+ const windowText = `Window: ${formatIncidentTime(options.sinceMs)} → ${formatIncidentTime(options.untilMs)} UTC (${formatIncidentDuration(options.untilMs - options.sinceMs)})`;
+ header.push(windowText);
+ if (options.source !== undefined) {
+ const inWindow = allEvents.filter(
+ (incident) => incident.timeMs >= options.sinceMs && incident.timeMs <= options.untilMs,
+ ).length;
+ const skipped = options.skippedCount ?? 0;
+ const skippedText = skipped > 0 ? `, ${skipped} unreadable skipped` : "";
+ header.push(
+ `Source: ${options.source} (${options.scannedCount ?? entries.length} lines scanned, ${inWindow} events in window${skippedText})`,
+ );
+ }
+ if (filteredBySession) {
+ header.push(`Session filter: ${options.session}`);
+ }
+
+ const text = [...header, "", ...sections.flatMap((section) => [section.title, ...section.lines, ""])]
+ .join("\n")
+ .trimEnd();
+ return { text };
+}
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+const INCIDENT_VALUE_FLAGS = new Set(["--since", "--until", "--session"]);
+
+/** Parse `incident [--since <time>] [--until <time>] [--session <id>]` arguments. */
+export function parseIncidentOptions(args: string[]): IncidentCommandOptions {
+ const options: IncidentCommandOptions = {};
+ for (let index = 0; index < args.length; index++) {
+ const arg = args[index]!;
+ let name = arg;
+ let value: string | undefined;
+ const equalsIndex = arg.indexOf("=");
+ if (arg.startsWith("--") && equalsIndex !== -1) {
+ name = arg.slice(0, equalsIndex);
+ value = arg.slice(equalsIndex + 1);
+ }
+ if (!INCIDENT_VALUE_FLAGS.has(name)) {
+ throw new IncidentUsageError(`Unknown option for incident: ${arg}`);
+ }
+ if (value === undefined) {
+ value = args[++index];
+ if (value === undefined) {
+ throw new IncidentUsageError(`Option ${name} requires a value.`);
+ }
+ }
+ if (!value.trim()) {
+ throw new IncidentUsageError(`Option ${name} requires a value.`);
+ }
+ if (name === "--since") {
+ options.since = value;
+ } else if (name === "--until") {
+ options.until = value;
+ } else {
+ options.session = value;
+ }
+ }
+ return options;
+}
+
+/**
+ * Read the daemon logs: agent.jsonl (plus its .old rotation) when it yields
+ * entries, otherwise the newest per-daemon log file in the logs directory.
+ * The fallback covers a missing, empty, or unreadable agent.jsonl and picks
+ * only the newest per-daemon log; rotated per-daemon .old generations are not
+ * included.
+ */
+export function readIncidentLogEntries(): IncidentLogSource {
+ const logsDir = getLogsDir();
+ const agentLogPath = getAgentLogPath();
+ const structuredFiles = [`${agentLogPath}.old`, agentLogPath]
+ .filter((candidate) => existsSync(candidate))
+ .map((path) => ({ path, kind: "jsonl" as const }));
+ const structured = scanIncidentLogFiles(structuredFiles);
+ if (structured.entries.length > 0) {
+ return structured;
+ }
+ // agent.jsonl is missing, empty, or unreadable: fall back to the newest
+ // per-daemon log. Window filtering happens later; gate on parse yield only.
+ const fallbackPath = newestDaemonLogPath(logsDir);
+ if (fallbackPath === undefined) {
+ return structured;
+ }
+ const fallback = scanIncidentLogFiles([{ path: fallbackPath, kind: "daemon" }]);
+ return fallback.entries.length > 0 ? fallback : structured;
+}
+
+function scanIncidentLogFiles(files: ReadonlyArray<{ path: string; kind: "jsonl" | "daemon" }>): IncidentLogSource {
+ const entries: IncidentLogEntry[] = [];
+ let scannedCount = 0;
+ let skippedCount = 0;
+ for (const file of files) {
+ let contents: string;
+ try {
+ contents = readFileSync(file.path, "utf8");
+ } catch {
+ continue;
+ }
+ for (const line of contents.split("\n")) {
+ if (!line.trim()) {
+ continue;
+ }
+ scannedCount += 1;
+ const entry = file.kind === "jsonl" ? parseIncidentLogLine(line) : parseIncidentDaemonLogLine(line);
+ if (entry) {
+ entries.push(entry);
+ } else {
+ skippedCount += 1;
+ }
+ }
+ }
+ return { entries, scannedCount, skippedCount, source: files.map((file) => file.path).join(", ") };
+}
+
+function newestDaemonLogPath(logsDir: string): string | undefined {
+ let names: string[] = [];
+ try {
+ names = readdirSync(logsDir);
+ } catch {
+ return undefined;
+ }
+ const candidates: Array<{ path: string; mtimeMs: number }> = [];
+ for (const name of names) {
+ if (!DAEMON_LOG_FILE_PATTERN.test(name)) {
+ continue;
+ }
+ const path = join(logsDir, name);
+ try {
+ const stat = statSync(path);
+ // A directory or other non-file matching the log pattern would be
+ // picked as the fallback and then yield no entries (readFileSync
+ // fails on it), hiding older valid daemon logs.
+ if (!stat.isFile()) {
+ continue;
+ }
+ candidates.push({ path, mtimeMs: stat.mtimeMs });
+ } catch {
+ // Lost a race with log rotation or permissions; skip the candidate.
+ }
+ }
+ if (candidates.length === 0) {
+ return undefined;
+ }
+ return candidates.sort((a, b) => b.mtimeMs - a.mtimeMs)[0]!.path;
+}
+
+/** The resolved --since/--until window for an incident report. */
+export interface IncidentWindow {
+ sinceMs: number;
+ untilMs: number;
+}
+
+/** Resolve --since/--until (default: last 24h until now); throws on bad or unordered times. */
+export function resolveIncidentWindow(options: IncidentCommandOptions, now: Date): IncidentWindow {
+ const sinceMs = options.since
+ ? parseIncidentTimeBound(options.since, now, "--since")
+ : now.getTime() - DEFAULT_WINDOW_MS;
+ const untilMs = options.until ? parseIncidentTimeBound(options.until, now, "--until") : now.getTime();
+ if (untilMs <= sinceMs) {
+ throw new IncidentUsageError("--until must be after --since.");
+ }
+ return { sinceMs, untilMs };
+}
+
+/**
+ * Entry point for `prime-agent incident`; prints the timeline to stdout.
+ * Callers that validate the window pass it back so relative `HH:MM` bounds
+ * resolve exactly once instead of again against a later clock reading.
+ */
+export async function runIncident(options: IncidentCommandOptions, window?: IncidentWindow): Promise<void> {
+ const { sinceMs, untilMs } = window ?? resolveIncidentWindow(options, new Date());
+ const logSource = readIncidentLogEntries();
+ if (logSource.entries.length === 0 && logSource.scannedCount === 0) {
+ console.log(`No daemon logs found under ${getLogsDir()}.`);
+ return;
+ }
+ const report = buildIncidentReport(logSource.entries, {
+ sinceMs,
+ untilMs,
+ ...(options.session !== undefined ? { session: options.session } : {}),
+ source: logSource.source,
+ scannedCount: logSource.scannedCount,
+ skippedCount: logSource.skippedCount,
+ });
+ console.log(report.text);
+}
diff --git a/packages/coding-agent/src/cli/public-command.ts b/packages/coding-agent/src/cli/public-command.ts
index 5fa05c40cd..2d946c9223 100644
--- a/packages/coding-agent/src/cli/public-command.ts
+++ b/packages/coding-agent/src/cli/public-command.ts
@@ -19,6 +19,13 @@ import { handleDaemonCommand } from "./daemon-command.js";
import { runPs, runReap, runShutdownAll } from "./daemon-ps.js";
import { DAEMON_UPDATE_RESTART_COORDINATOR_FLAG } from "./daemon-update-restart.js";
import { extractHelpCommandPath, rotateGlobalFlagsBeforeCommand } from "./global-flags.js";
+import {
+ type IncidentCommandOptions,
+ type IncidentWindow,
+ parseIncidentOptions,
+ resolveIncidentWindow,
+ runIncident,
+} from "./incident.js";
export interface PublicCommandResult {
handled: boolean;
@@ -113,6 +120,8 @@ async function runPublicCommand(args: string[]): Promise<PublicCommandResult> {
return runStatus(args.slice(1));
case "doctor":
return runDoctor(args.slice(1));
+ case "incident":
+ return runIncidentCommand(args.slice(1));
case "shutdown":
return runShutdown(args.slice(1));
case "package":
@@ -259,6 +268,21 @@ async function runDoctor(args: string[]): Promise<PublicCommandResult> {
return HANDLED;
}
+async function runIncidentCommand(args: string[]): Promise<PublicCommandResult> {
+ let options: IncidentCommandOptions;
+ let window: IncidentWindow;
+ try {
+ options = parseIncidentOptions(args);
+ // Resolve once: re-resolving later can cross UTC midnight and render a
+ // different window than the one that was validated.
+ window = resolveIncidentWindow(options, new Date());
+ } catch (error) {
+ return fail(error instanceof Error ? error.message : String(error), `Run "${APP_NAME} help incident" for usage.`);
+ }
+ await runIncident(options, window);
+ return HANDLED;
+}
+
async function runShutdown(args: string[]): Promise<PublicCommandResult> {
const options = parseBooleanOptions(args, new Set(["--force", "--json"]), "shutdown");
if (!options) return HANDLED;
diff --git a/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts b/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts
index fb03fc7934..8f036672e1 100644
--- a/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts
+++ b/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts
@@ -13,7 +13,14 @@ import {
visibleWidth,
wrapTextWithAnsi,
} from "@earendil-works/pi-tui";
-import { APP_TITLE, appendRotatingLog, getAgentDir, getClientErrorLogPath, VERSION } from "../../config.js";
+import {
+ APP_TITLE,
+ appendRotatingLog,
+ getAgentDir,
+ getAgentLogPath,
+ getClientErrorLogPath,
+ VERSION,
+} from "../../config.js";
import type { AgentSessionRuntimeConfig } from "../../core/agent-session-config.js";
import { KeybindingsManager } from "../../core/keybindings.js";
import { SessionManager } from "../../core/session-manager.js";
@@ -102,6 +109,14 @@ import {
type UnifiedSessionIndex,
type UnifiedSessionRecord,
} from "./agents-view-state.js";
+import {
+ createIncidentNoticeState,
+ dismissIncidentNoticeState,
+ formatIncidentNoticeLine,
+ INCIDENT_NOTICE_POLL_INTERVAL_MS,
+ type IncidentNoticeState,
+ refreshIncidentNoticeState,
+} from "./incident-notices.js";
import { AgentsViewRosterStore, STALE_ROSTER_DAEMON_MESSAGE } from "./roster-store.js";
import { matchesSearchText } from "./session-view-search.js";
@@ -169,6 +184,10 @@ export type AgentsViewPersistentState = {
// re-entry and render the moment they resolve, even if the first view was left early.
startupNotices?: StartupNotices;
startupNoticesPromise?: Promise<StartupNotices>;
+ // Incident notice state (windowed log entries, log offset, dismissal horizons)
+ // is reused the same way: a dismissed incident must stay dismissed across
+ // re-entry, and the log poll continues from the consumed offset.
+ incidentNoticeState?: IncidentNoticeState;
query?: string;
rosterClient?: DaemonClient;
rosterStore?: AgentsViewRosterStore;
@@ -839,6 +858,7 @@ export class AgentsViewMode implements Component, Focusable {
private resolveRun: ((result: AgentsViewRunResult) => void) | undefined;
private heartbeatPollTimer: NodeJS.Timeout | undefined;
private animationTimer: NodeJS.Timeout | undefined;
+ private incidentNoticeTimer: NodeJS.Timeout | undefined;
private ctrlCExitHintExpiresAt = 0;
private ctrlCExitHintTimer: ReturnType<typeof setTimeout> | undefined;
private deleteConfirmExpiresAt = 0;
@@ -1077,8 +1097,11 @@ export class AgentsViewMode implements Component, Focusable {
this.resolveMissingSelectionAnchor();
void this.refreshHeartbeats();
this.loadStartupNotices();
+ this.refreshIncidentNotices();
this.heartbeatPollTimer = setInterval(() => void this.refreshHeartbeats(), HEARTBEAT_POLL_INTERVAL_MS);
this.heartbeatPollTimer.unref?.();
+ this.incidentNoticeTimer = setInterval(() => this.refreshIncidentNotices(), INCIDENT_NOTICE_POLL_INTERVAL_MS);
+ this.incidentNoticeTimer.unref?.();
this.animationTimer = setInterval(() => {
const hasRunning = this.rows.some((row) => row.section === "running");
const hasStaleAge = this.rows.some((row) => row.summary.lastHeardFromAt !== undefined);
@@ -1110,6 +1133,23 @@ export class AgentsViewMode implements Component, Focusable {
this.handleCtrlC();
return;
}
+ // Esc (tui.select.cancel) dismisses the incident notice while it is the
+ // only thing to cancel: no armed reply, no autocomplete popup, an empty
+ // search prompt. An armed delete confirmation is the more dangerous state:
+ // Esc cancels it (below) and keeps the notice instead of dismissing the
+ // notice and leaving the delete armed to fire on the next press without
+ // a fresh confirmation. Without a visible notice, Esc keeps its back/exit
+ // meaning.
+ if (
+ this.editor.getText().length === 0 &&
+ !this.replyTarget &&
+ !this.editor.isShowingAutocomplete() &&
+ !this.isDeleteConfirmationVisible() &&
+ this.keybindings.matches(data, "tui.select.cancel") &&
+ this.dismissIncidentNotice()
+ ) {
+ return;
+ }
if (this.editor.getText().length === 0 && this.keybindings.matches(data, "app.agents.rename")) {
this.enterRenameMode();
return;
@@ -1186,7 +1226,7 @@ export class AgentsViewMode implements Component, Focusable {
return [];
}
const headerLines = this.splash.render(width);
- const noticeLines = this.renderStartupNotices(width);
+ const noticeLines = [...this.renderIncidentNotice(width), ...this.renderStartupNotices(width)];
if (noticeLines.length > 0) {
headerLines.push("", ...noticeLines);
}
@@ -1258,6 +1298,44 @@ export class AgentsViewMode implements Component, Focusable {
return formatted.flatMap((line) => wrapTextWithAnsi(line, wrapWidth).map((wrapped) => ` ${wrapped}`));
}
+ private incidentNoticeState(): IncidentNoticeState {
+ // Reused across agents-view instances like the startup notices: the
+ // windowed entries, the consumed log offset, and the dismissal horizons
+ // survive leaving and re-entering the view, so a dismissed incident never
+ // comes back and the poll does not re-read consumed bytes.
+ this.persistentState.incidentNoticeState ??= createIncidentNoticeState();
+ return this.persistentState.incidentNoticeState;
+ }
+
+ private refreshIncidentNotices(): void {
+ // Best-effort, like the startup notices: a missing or unreadable log
+ // simply retries a bounded tail on the next poll and never breaks the view.
+ const changed = refreshIncidentNoticeState(this.incidentNoticeState(), getAgentLogPath(), Date.now());
+ if (changed) {
+ this.ui.requestRender();
+ }
+ }
+
+ /** Dismiss the collapsed incident notice; false when none is showing. */
+ private dismissIncidentNotice(): boolean {
+ if (!dismissIncidentNoticeState(this.incidentNoticeState())) {
+ return false;
+ }
+ this.setStatusMessage("Incident notice dismissed");
+ return true;
+ }
+
+ private renderIncidentNotice(width: number): string[] {
+ const notice = this.persistentState.incidentNoticeState?.notice;
+ if (!notice) {
+ return [];
+ }
+ // Same treatment as the startup notices: one-column gutter, wrap instead
+ // of truncating, so the pointer to the incident CLI stays readable.
+ const wrapWidth = Math.max(1, width - 1);
+ return wrapTextWithAnsi(formatIncidentNoticeLine(notice), wrapWidth).map((wrapped) => ` ${wrapped}`);
+ }
+
private handleListNavigation(data: string): boolean {
if (this.keybindings.matches(data, "tui.select.up")) {
this.moveSelection(-1);
@@ -2566,6 +2644,10 @@ export class AgentsViewMode implements Component, Focusable {
clearInterval(this.heartbeatPollTimer);
this.heartbeatPollTimer = undefined;
}
+ if (this.incidentNoticeTimer) {
+ clearInterval(this.incidentNoticeTimer);
+ this.incidentNoticeTimer = undefined;
+ }
if (this.animationTimer) {
clearInterval(this.animationTimer);
this.animationTimer = undefined;
diff --git a/packages/coding-agent/src/modes/agents-view/incident-notices.ts b/packages/coding-agent/src/modes/agents-view/incident-notices.ts
new file mode 100644
index 0000000000..286a831280
--- /dev/null
+++ b/packages/coding-agent/src/modes/agents-view/incident-notices.ts
@@ -0,0 +1,466 @@
+import { Buffer } from "node:buffer";
+import { closeSync, fstatSync, openSync, readSync } from "node:fs";
+import {
+ collectIncidentEvents,
+ collectWorkerPidMap,
+ computeIncidentAnomalies,
+ type IncidentLogEntry,
+ type IncidentSeverity,
+ latestIncidentStallTimeoutBySubject,
+ parseIncidentLogLine,
+} from "../../cli/incident.js";
+import { theme } from "../interactive/theme/theme.js";
+
+/**
+ * Daemon incident notices for the agents view.
+ *
+ * `prime-agent incident` (the classifier in src/cli/incident.ts) already
+ * reconstructs daemon incidents from the shared structured log
+ * (~/.prime/agent/logs/agent.jsonl); this module reuses that classifier —
+ * never re-implementing it — to surface a single collapsed, dismissible
+ * notice line in the agents-view header. A notice appears when the recent
+ * window of the log contains a worker crash, a command-timeout burst, or an
+ * update restart (a supervisor replacement), so the operator sees the
+ * incident without running the CLI by hand. On the initial read the view
+ * matches the CLI's [agent.jsonl.old, agent.jsonl] source with the same
+ * bounded tail, so incidents spanning a log rotation surface too.
+ */
+
+/** Recent-log window, matching the `prime-agent incident` default. */
+export const INCIDENT_NOTICE_WINDOW_MS = 24 * 60 * 60 * 1000;
+/** Initial tail bound: incidents older than the tail bytes are simply not seen. */
+export const INCIDENT_NOTICE_TAIL_BYTES = 512 * 1024;
+/**
+ * Retention cap for windowed entries: every entry keeps the full parsed log
+ * record (fields), and agent.jsonl is the sink for ALL structured logging with
+ * 20MB rotating generations, so a busy day would otherwise retain unbounded
+ * memory and make every poll re-sort and re-classify it all synchronously.
+ * Newest entries win; incidents older than the cap are simply not seen — the
+ * same best-effort spirit as INCIDENT_NOTICE_TAIL_BYTES.
+ */
+export const INCIDENT_NOTICE_MAX_WINDOW_ENTRIES = 20_000;
+/** How often the agents view re-reads appended agent.jsonl bytes. */
+export const INCIDENT_NOTICE_POLL_INTERVAL_MS = 30_000;
+
+const INCIDENT_NOTICE_POINTER = "— run prime-agent incident for the timeline";
+const NEWLINE_BYTE = 0x0a;
+const SEVERITY_RANK: Record<IncidentSeverity, number> = { critical: 3, error: 2, warn: 1, info: 0 };
+
+export type IncidentNoticeKind = "worker-crash" | "timeout-burst" | "update-restart";
+
+export interface IncidentNotice {
+ kind: IncidentNoticeKind;
+ /** Dismissal key (`${kind}|${subject}`): incidents at or before the horizon stay hidden. */
+ key: string;
+ severity: IncidentSeverity;
+ subject: string;
+ timeMs: number;
+ /** Sentence without the pointer suffix; formatIncidentNoticeLine renders the visible line. */
+ text: string;
+}
+
+/** Per-run incident notice state, cached on the agents view's persistentState. */
+export interface IncidentNoticeState {
+ /** Windowed log entries parsed so far, oldest first. */
+ entries: IncidentLogEntry[];
+ /** Byte offset consumed in agent.jsonl; undefined before the first tail read. */
+ logOffset: number | undefined;
+ /** `dev:ino` of agent.jsonl at the last read; a change means rotation or replacement. */
+ logFileId: string | undefined;
+ /** Dismissal horizons by notice key: incidents at or before this timeMs stay hidden. */
+ dismissedHorizons: Record<string, number>;
+ /** The collapsed notice currently worth showing, if any. */
+ notice: IncidentNotice | undefined;
+}
+
+export function createIncidentNoticeState(): IncidentNoticeState {
+ return {
+ entries: [],
+ logOffset: undefined,
+ logFileId: undefined,
+ dismissedHorizons: {},
+ notice: undefined,
+ };
+}
+
+/**
+ * Local time of an incident, for "worker x crashed at 14:32". A bare HH:MM only
+ * reads as today, so an incident from another local calendar day gets the date
+ * prefixed (the tree-selector.ts label-timestamp style): "9/14 23:10", or
+ * "26/9/14 23:10" across a year boundary.
+ */
+export function formatIncidentNoticeTime(timeMs: number, nowMs: number): string {
+ const date = new Date(timeMs);
+ const pad = (value: number) => String(value).padStart(2, "0");
+ const time = `${pad(date.getHours())}:${pad(date.getMinutes())}`;
+ const now = new Date(nowMs);
+ if (
+ date.getFullYear() === now.getFullYear() &&
+ date.getMonth() === now.getMonth() &&
+ date.getDate() === now.getDate()
+ ) {
+ return time;
+ }
+ const month = date.getMonth() + 1;
+ const day = date.getDate();
+ if (date.getFullYear() === now.getFullYear()) {
+ return `${month}/${day} ${time}`;
+ }
+ return `${date.getFullYear().toString().slice(-2)}/${month}/${day} ${time}`;
+}
+
+/** The styled one-line notice rendered in the agents-view header. */
+export function formatIncidentNoticeLine(notice: IncidentNotice): string {
+ return theme.fg("warning", `⚠ ${notice.text} ${INCIDENT_NOTICE_POINTER}`);
+}
+
+function createNotice(
+ kind: IncidentNoticeKind,
+ severity: IncidentSeverity,
+ subject: string,
+ timeMs: number,
+ text: string,
+): IncidentNotice {
+ return { kind, key: `${kind}|${subject}`, severity, subject, timeMs, text };
+}
+
+/**
+ * Derive the notices worth surfacing from windowed agent.jsonl entries, reusing
+ * the incident CLI's classifier. Exactly three incident classes qualify:
+ * worker crashes (any worker-crash event), command-timeout bursts (the
+ * classifier's per-subject "N command timeouts over X" anomaly), and update
+ * restarts (a supervisor-start whose subject already started within the
+ * window, i.e. the supervisor was replaced). A first-ever supervisor start is
+ * routine and never produces a notice. A timeout-burst notice carries the
+ * latest timeout of the stall cluster its anomaly describes — the classifier
+ * anchors the anomaly at the cluster's first timeout — so dismissing it
+ * records a horizon that only covers that burst as dismissed: a later timeout
+ * extending the burst past the horizon re-surfaces the notice, instead of it
+ * staying hidden until the first timeout ages out of the window; an isolated
+ * stray timeout, or a separate later stall, never moves the anchor and never
+ * re-opens it.
+ */
+export function deriveIncidentNotices(entries: readonly IncidentLogEntry[], nowMs: number): IncidentNotice[] {
+ const sinceMs = nowMs - INCIDENT_NOTICE_WINDOW_MS;
+ // CLI window parity: buildIncidentReport windows events by [sinceMs, untilMs],
+ // so a future-dated entry (clock skew, a bogus timestamp) is outside it too.
+ const windowed = entries.filter((entry) => entry.timeMs >= sinceMs && entry.timeMs <= nowMs);
+ const workerPids = collectWorkerPidMap(windowed);
+ const events = collectIncidentEvents(windowed, workerPids);
+ events.sort((a, b) => a.timeMs - b.timeMs);
+ const notices: IncidentNotice[] = [];
+ for (const event of events) {
+ if (event.eventClass === "worker-crash") {
+ // The subject already reads "worker <id>" ("worker" when the id is unknown).
+ notices.push(
+ createNotice(
+ "worker-crash",
+ event.severity,
+ event.subject,
+ event.timeMs,
+ `${event.subject} crashed at ${formatIncidentNoticeTime(event.timeMs, nowMs)}`,
+ ),
+ );
+ }
+ }
+ // Latest timeout of each subject's stall cluster — the same incident the
+ // classifier's per-subject "N command timeouts over X" anomaly describes —
+ // so a notice and its dismissal horizon always refer to one incident:
+ // dismissal advances with a growing burst (a later timeout in the cluster
+ // re-surfaces the notice past the horizon instead of it staying hidden
+ // until the first timeout ages out), while a separate later stall or an
+ // isolated stray timeout never moves the anchor.
+ const latestStallTimeoutBySubject = latestIncidentStallTimeoutBySubject(events);
+ for (const anomaly of computeIncidentAnomalies(events)) {
+ if (anomaly.summary.includes("command timeouts")) {
+ // The anomaly summary already reads "<subject>: N command timeouts over X".
+ const latestTimeoutMs = latestStallTimeoutBySubject.get(anomaly.subject) ?? anomaly.timeMs;
+ notices.push(
+ createNotice("timeout-burst", anomaly.severity, anomaly.subject, latestTimeoutMs, anomaly.summary),
+ );
+ }
+ }
+ // The classifier also emits supervisor-start for failed spawns (lock held,
+ // startup error) at warn/error severity; only a successful start — the
+ // info-severity "listening on" event — counts toward a replacement, or two
+ // failed spawns on one socket would read as a restart.
+ const startedSubjects = new Set<string>();
+ for (const event of events) {
+ if (event.eventClass !== "supervisor-start" || event.severity !== "info") {
+ continue;
+ }
+ if (startedSubjects.has(event.subject)) {
+ notices.push(
+ createNotice(
+ "update-restart",
+ event.severity,
+ event.subject,
+ event.timeMs,
+ `daemon restarted for update at ${formatIncidentNoticeTime(event.timeMs, nowMs)}`,
+ ),
+ );
+ } else {
+ startedSubjects.add(event.subject);
+ }
+ }
+ return notices;
+}
+
+/**
+ * Collapse the derived notices to the single line the header shows: the most
+ * severe wins (critical > error > warn > info), the most recent breaks ties.
+ * Repeated identical events aggregate here — the header never stacks copies.
+ */
+export function selectIncidentNotice(notices: readonly IncidentNotice[]): IncidentNotice | undefined {
+ let best: IncidentNotice | undefined;
+ for (const notice of notices) {
+ if (
+ best === undefined ||
+ SEVERITY_RANK[notice.severity] > SEVERITY_RANK[best.severity] ||
+ (SEVERITY_RANK[notice.severity] === SEVERITY_RANK[best.severity] && notice.timeMs > best.timeMs)
+ ) {
+ best = notice;
+ }
+ }
+ return best;
+}
+
+/** True when the notice sits at or before its key's dismissal horizon. */
+export function isIncidentNoticeDismissed(notice: IncidentNotice, horizons: Record<string, number>): boolean {
+ const horizon = horizons[notice.key];
+ return horizon !== undefined && notice.timeMs <= horizon;
+}
+
+/**
+ * Dismiss the notice currently showing. Records its timeMs as the horizon for
+ * its key, so the same incident (and any older incident on that key) never
+ * re-renders on later polls or view re-entry, while a newer qualifying
+ * incident does. Returns false when no notice is showing.
+ */
+export function dismissIncidentNoticeState(state: IncidentNoticeState): boolean {
+ const notice = state.notice;
+ if (!notice) {
+ return false;
+ }
+ state.dismissedHorizons[notice.key] = Math.max(state.dismissedHorizons[notice.key] ?? 0, notice.timeMs);
+ state.notice = undefined;
+ return true;
+}
+
+function sameIncidentNotice(a: IncidentNotice | undefined, b: IncidentNotice | undefined): boolean {
+ return a?.key === b?.key && a?.timeMs === b?.timeMs && a?.text === b?.text;
+}
+
+interface IncidentLogChunk {
+ lines: string[];
+ nextOffset: number;
+ fileId: string;
+}
+
+/**
+ * Rotation-safe incremental read of agent.jsonl. Without a previous offset — or
+ * after rotation (a changed inode), a shrink (recreation in place), or more
+ * than one tail bound of new bytes — read the bounded tail: the cut may begin
+ * mid-line (drop the torn leading fragment) or exactly at a record boundary
+ * (the byte before the cut is a newline; keep the intact first record, which
+ * dropping would silently lose for the lifetime of the state). Otherwise read
+ * only appended bytes (every read stays bounded). A trailing partial line is
+ * held back (the offset stops at its newline), so a mid-write line parses only
+ * once complete, on a later poll — unless includeFinalPartialLine is set for a
+ * frozen file (the rotated .old), which no later poll can complete; its final
+ * line is returned as-is. A missing or unreadable file returns undefined;
+ * offsets beyond the file size are never re-processed.
+ */
+function readIncidentLogLines(
+ logPath: string,
+ previousOffset: number | undefined,
+ previousFileId: string | undefined,
+ includeFinalPartialLine = false,
+): IncidentLogChunk | undefined {
+ let fd: number;
+ try {
+ fd = openSync(logPath, "r");
+ } catch {
+ return undefined;
+ }
+ try {
+ const stats = fstatSync(fd);
+ const fileId = `${stats.dev}:${stats.ino}`;
+ const rotated = previousFileId !== undefined && fileId !== previousFileId;
+ // Re-tail when nothing was read yet, after a rotation (a new file), when
+ // the file shrank (recreated in place), or when more than one tail
+ // bound appended since the last poll; every read stays bounded and
+ // offsets are never re-processed.
+ const retailed =
+ previousOffset === undefined ||
+ rotated ||
+ previousOffset > stats.size ||
+ stats.size - previousOffset > INCIDENT_NOTICE_TAIL_BYTES;
+ const start = retailed ? Math.max(0, stats.size - INCIDENT_NOTICE_TAIL_BYTES) : previousOffset;
+ if (start >= stats.size) {
+ return { lines: [], nextOffset: stats.size, fileId };
+ }
+ const buffer = Buffer.allocUnsafe(stats.size - start);
+ const bytesRead = readSync(fd, buffer, 0, buffer.length, start);
+ let lineStart = 0;
+ if (retailed && start > 0) {
+ // The bounded tail may begin mid-line (the cut split a record: drop
+ // the torn leading fragment) or exactly at a record boundary (the byte
+ // before the cut is a newline: the first line in the buffer is a
+ // complete record, and dropping it would silently lose a qualifying
+ // incident for the lifetime of the state). Read that one byte to tell
+ // the cases apart; the read happens only on this bounded re-tail path.
+ const preceding = Buffer.alloc(1);
+ const beginsMidLine = readSync(fd, preceding, 0, 1, start - 1) !== 1 || preceding[0] !== NEWLINE_BYTE;
+ if (beginsMidLine) {
+ // A chunk with no newline at all is one mid-write line: hold it
+ // back so the completed line is still parsed by the next poll.
+ const firstNewline = buffer.subarray(0, bytesRead).indexOf(NEWLINE_BYTE);
+ if (firstNewline === -1) {
+ return { lines: [], nextOffset: start, fileId };
+ }
+ lineStart = firstNewline + 1;
+ }
+ }
+ let end = bytesRead;
+ // A frozen file (the rotated .old) never gets the completing write, so
+ // with includeFinalPartialLine its final line is returned as-is: a torn
+ // line parses to undefined and drops harmlessly, while a complete
+ // record missing only its newline must not be lost.
+ if (!includeFinalPartialLine && end > 0 && buffer[end - 1] !== NEWLINE_BYTE) {
+ // Hold back the partially-written final line until it completes.
+ const lastNewline = buffer.subarray(0, bytesRead).lastIndexOf(NEWLINE_BYTE);
+ if (lastNewline < lineStart) {
+ return { lines: [], nextOffset: start, fileId };
+ }
+ end = lastNewline + 1;
+ }
+ const lines = buffer
+ .toString("utf8", lineStart, end)
+ .split("\n")
+ .filter((line) => line.trim().length > 0);
+ return { lines, nextOffset: start + end, fileId };
+ } catch {
+ return undefined;
+ } finally {
+ try {
+ closeSync(fd);
+ } catch {
+ // Best-effort: a failed close must not mask the read result.
+ }
+ }
+}
+
+/**
+ * Keep windowed entries in stable time order across polls: new entries append,
+ * everything older than the window drops, and only the newest
+ * INCIDENT_NOTICE_MAX_WINDOW_ENTRIES survive, so memory and per-poll work stay
+ * bounded. Lines re-read after a rotation or a re-tail collapse harmlessly —
+ * identical lifecycle events dedupe in the classifier, and the collapsed line
+ * never stacks copies.
+ */
+function mergeIncidentWindowedEntries(
+ entries: readonly IncidentLogEntry[],
+ parsed: readonly IncidentLogEntry[],
+ sinceMs: number,
+): IncidentLogEntry[] {
+ if (entries.length === 0 && parsed.length === 0) {
+ return [];
+ }
+ const merged = [...entries, ...parsed];
+ merged.sort((a, b) => a.timeMs - b.timeMs);
+ const windowed = merged.filter((entry) => entry.timeMs >= sinceMs);
+ // Newest entries win: slice the tail of the time-sorted array (entries stay
+ // oldest-first) so retention cannot grow without bound on a busy log day.
+ return windowed.length > INCIDENT_NOTICE_MAX_WINDOW_ENTRIES
+ ? windowed.slice(windowed.length - INCIDENT_NOTICE_MAX_WINDOW_ENTRIES)
+ : windowed;
+}
+
+/**
+ * One best-effort poll: read new agent.jsonl bytes, keep the 24h window,
+ * re-derive the qualifying notices, apply the dismissal horizons, and keep the
+ * single collapsed line worth showing. The first successful read also tails
+ * the rotated agent.jsonl.old — matching the CLI's [agent.jsonl.old,
+ * agent.jsonl] source with the same bounded tail — so incidents spanning a
+ * rotation still surface in a fresh view. Later polls read only appended
+ * agent.jsonl bytes, except when the live read detects a new generation: the
+ * un-consumed tail of the rotated-out one is then completed from .old by
+ * offset continuation (never re-read from its start, which would duplicate
+ * supervisor starts into phantom update restarts). Returns true when that line
+ * changed so the caller can re-render. Never throws for a missing or unreadable log: that poll keeps
+ * the consumed offset (a re-tail would fabricate restarts) but still
+ * re-derives, so the notice expires with its window instead of surviving
+ * forever while the log stays unreadable.
+ */
+export function refreshIncidentNoticeState(state: IncidentNoticeState, logPath: string, nowMs: number): boolean {
+ // The first successful read bridges the rotated generation: the CLI reads
+ // [agent.jsonl.old, agent.jsonl] (readIncidentLogEntries), so a view opened
+ // after a rotation must see pairs that span it — an update restart whose
+ // earlier supervisor start sits in .old, or a burst straddling the files.
+ // A later re-read from .old's start would duplicate supervisor-start lines
+ // into a phantom update restart (the classifier does not dedupe
+ // supervisor-start), which is also why the consumed offset never resets.
+ const firstRead = state.logOffset === undefined && state.logFileId === undefined;
+ const chunk = readIncidentLogLines(logPath, state.logOffset, state.logFileId);
+ const sinceMs = nowMs - INCIDENT_NOTICE_WINDOW_MS;
+ // CLI window parity (buildIncidentReport bounds events by >= since && <=
+ // until): future-dated entries fall outside the window and never surface.
+ const parseWindowedLines = (lines: readonly string[]): IncidentLogEntry[] =>
+ lines
+ .map((line) => parseIncidentLogLine(line))
+ .filter(
+ (entry): entry is IncidentLogEntry =>
+ entry !== undefined && entry.timeMs >= sinceMs && entry.timeMs <= nowMs,
+ );
+ let parsed: IncidentLogEntry[] = [];
+ if (chunk !== undefined) {
+ // Tail the rotated .old with the same bounded tail as the main log: the
+ // CLI reads the same [agent.jsonl.old, agent.jsonl] pair, and the merge
+ // below sorts by time, so the read order here does not matter.
+ if (firstRead) {
+ // The rotated .old is frozen: include its final line even without a
+ // trailing newline — no later poll will ever complete it.
+ const rotated = readIncidentLogLines(`${logPath}.old`, undefined, undefined, true);
+ // A rename rotation can land between the live read above and this one:
+ // the .old path then names the very file the chunk just consumed, and
+ // re-parsing it would double supervisor starts into a phantom update
+ // restart (the classifier does not dedupe supervisor-start) and double
+ // timeout counts. Bridge only a genuinely different generation.
+ if (rotated !== undefined && rotated.fileId !== chunk.fileId) {
+ parsed = parseWindowedLines(rotated.lines);
+ }
+ } else if (state.logFileId !== undefined && chunk.fileId !== state.logFileId) {
+ // A rotation between polls strands the un-consumed tail of the previous
+ // generation at the .old path: no later poll re-reads it, so a crash
+ // logged in that gap would surface in `prime-agent incident` but never
+ // in the notice. The live read just detected the new generation, so
+ // continue the old one from its consumed offset — the file id still
+ // matches, the read is an offset continuation, and no consumed line is
+ // re-parsed (a re-tail on an id mismatch stays bounded).
+ const rotatedTail = readIncidentLogLines(`${logPath}.old`, state.logOffset, state.logFileId, true);
+ if (rotatedTail !== undefined) {
+ parsed = parseWindowedLines(rotatedTail.lines);
+ }
+ }
+ parsed = parsed.concat(parseWindowedLines(chunk.lines));
+ state.logOffset = chunk.nextOffset;
+ state.logFileId = chunk.fileId;
+ }
+ // A missing or unreadable log keeps the consumed offset and file id exactly
+ // as they are: resetting them would make the next poll re-tail and re-parse
+ // consumed lines into a phantom second supervisor start (a false update
+ // restart; the classifier does not dedupe supervisor-start) and doubled
+ // timeout counts. The poll still falls through to the merge/derive path
+ // with no new entries, so the notice ages out of its window while the log
+ // stays unreadable instead of surviving forever; a real rotation is still
+ // caught by the file id changing (or the offset passing the size) on the
+ // next successful read.
+ const previous = state.notice;
+ state.entries = mergeIncidentWindowedEntries(state.entries, parsed, sinceMs);
+ const notices = deriveIncidentNotices(state.entries, nowMs);
+ state.notice = selectIncidentNotice(
+ notices.filter((notice) => !isIncidentNoticeDismissed(notice, state.dismissedHorizons)),
+ );
+ return !sameIncidentNotice(previous, state.notice);
+}
diff --git a/packages/coding-agent/test/agents-view-incident-notice.test.ts b/packages/coding-agent/test/agents-view-incident-notice.test.ts
new file mode 100644
index 0000000000..34c1b9a7a2
--- /dev/null
+++ b/packages/coding-agent/test/agents-view-incident-notice.test.ts
@@ -0,0 +1,297 @@
+import { appendFileSync, linkSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { setKeybindings } from "@earendil-works/pi-tui";
+import stripAnsi from "strip-ansi";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { parseIncidentLogLine } from "../src/cli/incident.js";
+import { ENV_AGENT_DIR, getAgentLogPath } from "../src/config.js";
+import { KeybindingsManager } from "../src/core/keybindings.js";
+import type { ModelRegistry } from "../src/core/model-registry.js";
+import { SettingsManager } from "../src/core/settings-manager.js";
+import { AgentsViewMode, type AgentsViewPersistentState } from "../src/modes/agents-view/agents-view-mode.js";
+import {
+ createIncidentNoticeState,
+ deriveIncidentNotices,
+ dismissIncidentNoticeState,
+ formatIncidentNoticeTime,
+ refreshIncidentNoticeState,
+} from "../src/modes/agents-view/incident-notices.js";
+import type { InteractiveModeUiServices } from "../src/modes/interactive/interactive-mode-services.js";
+import { stopThemeWatcher } from "../src/modes/interactive/theme/theme.js";
+
+vi.mock("../src/config.js", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../src/config.js")>();
+ return { ...actual, appendRotatingLog: vi.fn() };
+});
+
+vi.mock("../src/utils/tools-manager.js", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("../src/utils/tools-manager.js")>();
+ // ensureTool would download fd into the redirected agent dir; tests never need it.
+ return { ...actual, ensureTool: vi.fn(async () => undefined) };
+});
+
+const DAEMON_SOCKET = "/tmp/prime-agent-501/daemon.sock";
+function logLine(fields: Record<string, unknown>): string {
+ return JSON.stringify({ level: "warn", ...fields });
+}
+function tsAgo(base: number, ms: number): string {
+ return new Date(base - ms).toISOString();
+}
+function supervisorStartLine(base: number, minutesAgoValue: number, generation = "e14de15c"): string {
+ return logLine({
+ ts: tsAgo(base, minutesAgoValue * 60_000),
+ component: "coding-agent.daemon-supervisor",
+ msg: `Prime Agent daemon supervisor ${generation} listening on ${DAEMON_SOCKET}`,
+ socketPath: DAEMON_SOCKET,
+ pid: 15026,
+ });
+}
+function workerCrashLine(base: number, workerId: string, secondsAgoValue: number): string {
+ return logLine({
+ ts: tsAgo(base, secondsAgoValue * 1000),
+ component: "coding-agent.daemon-supervisor",
+ msg: `Session worker ${workerId} stderr: uncaught exception: Error: write EPIPE`,
+ });
+}
+function commandTimeoutLine(base: number, minutesAgoValue: number, socketPath: string = DAEMON_SOCKET): string {
+ return logLine({
+ ts: tsAgo(base, minutesAgoValue * 60_000),
+ component: "coding-agent.daemon-supervisor",
+ socketPath,
+ msg: "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach\n at Timeout._onTimeout (node:internal/timers:618:7)",
+ });
+}
+function fixtureEntries(lines: readonly string[]) {
+ return lines.map((line) => parseIncidentLogLine(line)).filter((entry) => entry !== undefined);
+}
+function invoke(method: string, self: object, ...args: unknown[]): unknown {
+ const member = Reflect.get(AgentsViewMode.prototype, method) as ((...a: unknown[]) => unknown) | undefined;
+ if (typeof member !== "function") throw new Error(`AgentsViewMode.${method} no longer exists`);
+ return member.call(self, ...args);
+}
+function createUiServices(): InteractiveModeUiServices {
+ return {
+ settingsManager: SettingsManager.inMemory({ theme: "dark" }),
+ modelRegistry: {} as ModelRegistry,
+ getInitialCwd: () => process.cwd(),
+ getInitialSessionName: () => undefined,
+ getThemes: () => [],
+ };
+}
+const cleanupDirs: string[] = [];
+let previousAgentDir: string | undefined;
+/** Fresh per-test agent dir so getAgentLogPath() points at a fixture log. */
+function useTempAgentDir(): string {
+ const dir = mkdtempSync(join(tmpdir(), "agents-view-incident-"));
+ cleanupDirs.push(dir);
+ process.env[ENV_AGENT_DIR] = dir;
+ mkdirSync(join(dir, "logs"), { recursive: true });
+ return dir;
+}
+function writeAgentLog(lines: readonly string[]): void {
+ writeFileSync(getAgentLogPath(), `${lines.join("\n")}\n`);
+}
+function appendAgentLog(lines: readonly string[]): void {
+ appendFileSync(getAgentLogPath(), `${lines.join("\n")}\n`);
+}
+function newView(persistentState: AgentsViewPersistentState = { savedCatalogLoaded: true }): AgentsViewMode {
+ return new AgentsViewMode({ config: {}, uiServices: createUiServices() }, persistentState);
+}
+/** The incident notice lines renderContent produces (startup notices stay unset). */
+function renderedIncidentLines(view: AgentsViewMode): string[] {
+ const lines = invoke("renderContent", view, 120, 40) as string[];
+ return lines.map(stripAnsi).filter((line) => line.includes("prime-agent incident"));
+}
+beforeAll(() => {
+ setKeybindings(new KeybindingsManager());
+ previousAgentDir = process.env[ENV_AGENT_DIR];
+});
+afterAll(() => {
+ if (previousAgentDir === undefined) {
+ delete process.env[ENV_AGENT_DIR];
+ } else {
+ process.env[ENV_AGENT_DIR] = previousAgentDir;
+ }
+ for (const dir of cleanupDirs) {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("incident notice derivation", () => {
+ it("anchors each subject's timeout burst at its own cluster's latest timeout", () => {
+ const base = Date.now();
+ // Several daemon sockets, each with its own stall: the per-subject
+ // cluster lookup must never leak or mis-anchor across subjects.
+ const otherSocket = "/tmp/prime-agent-501/daemon-other.sock";
+ const spacedSocket = "/tmp/prime-agent-501/daemon-spaced.sock";
+ const spacedTimeouts = [200, 190, 180, 30, 29].map((minutes) => commandTimeoutLine(base, minutes, spacedSocket));
+ const entries = fixtureEntries([
+ commandTimeoutLine(base, 30, DAEMON_SOCKET),
+ commandTimeoutLine(base, 25, otherSocket),
+ commandTimeoutLine(base, 20, DAEMON_SOCKET),
+ commandTimeoutLine(base, 5, otherSocket),
+ ...spacedTimeouts,
+ ]);
+ const bursts = deriveIncidentNotices(entries, base).filter((notice) => notice.kind === "timeout-burst");
+ expect(bursts).toHaveLength(3);
+ const bySubject = new Map(bursts.map((notice) => [notice.subject, notice]));
+ expect(bySubject.get(DAEMON_SOCKET)?.timeMs).toBe(base - 20 * 60_000);
+ expect(bySubject.get(otherSocket)?.timeMs).toBe(base - 5 * 60_000);
+ expect(bySubject.get(spacedSocket)?.text).toContain("3 command timeouts over 20m");
+ expect(bySubject.get(spacedSocket)?.timeMs).toBe(base - 180 * 60_000);
+ });
+
+ it("derives an update-restart only from repeated successful supervisor starts", () => {
+ const base = Date.now();
+ const replaced = fixtureEntries([supervisorStartLine(base, 120), supervisorStartLine(base, 60)]);
+ const restarts = deriveIncidentNotices(replaced, Date.now());
+ expect(restarts).toHaveLength(1);
+ expect(restarts[0]).toMatchObject({
+ kind: "update-restart",
+ severity: "info",
+ subject: DAEMON_SOCKET,
+ timeMs: replaced[1]!.timeMs,
+ });
+ expect(restarts[0]!.text).toBe(
+ `daemon restarted for update at ${formatIncidentNoticeTime(replaced[1]!.timeMs, Date.now())}`,
+ );
+
+ // A first-ever start is routine; a failed startup (lock held) never
+ // counts toward a replacement, or two failed spawns on one socket
+ // would read as an update restart.
+ const failedStartLine = (minutesAgoValue: number) =>
+ logLine({
+ ts: tsAgo(base, minutesAgoValue * 60_000),
+ component: "coding-agent.daemon-supervisor",
+ socketPath: DAEMON_SOCKET,
+ msg: "Daemon supervisor startup failed: lock file is already being held by another process",
+ });
+ expect(deriveIncidentNotices(fixtureEntries([supervisorStartLine(base, 60)]), Date.now())).toEqual([]);
+ expect(
+ deriveIncidentNotices(fixtureEntries([failedStartLine(30), supervisorStartLine(base, 20)]), Date.now()),
+ ).toEqual([]);
+ });
+});
+
+describe("agents view incident notices", () => {
+ it("renders the collapsed worker-crash notice line from the log tail", () => {
+ useTempAgentDir();
+ const base = Date.now();
+ writeAgentLog([supervisorStartLine(base, 600), workerCrashLine(base, "5b1d3aeb91ee", 120)]);
+ const view = newView();
+ try {
+ invoke("refreshIncidentNotices", view);
+ const lines = renderedIncidentLines(view);
+ expect(lines).toHaveLength(1);
+ expect(lines[0]).toContain(
+ `worker 5b1d3aeb91ee crashed at ${formatIncidentNoticeTime(base - 120_000, Date.now())}`,
+ );
+ expect(lines[0]).toContain("prime-agent incident for the timeline");
+ expect(lines[0].startsWith(" \u26a0")).toBe(true);
+ } finally {
+ stopThemeWatcher();
+ }
+ });
+
+ it("cancels an armed delete confirmation with Esc instead of dismissing the notice", () => {
+ useTempAgentDir();
+ writeAgentLog([workerCrashLine(Date.now(), "5b1d3aeb91ee", 120)]);
+ const view = newView();
+ try {
+ invoke("refreshIncidentNotices", view);
+ expect(renderedIncidentLines(view)).toHaveLength(1);
+ // A delete confirmation is armed: Esc must cancel it and keep the
+ // notice, or the next delete press would fire without a fresh
+ // confirmation.
+ invoke("showDeleteConfirmation", view);
+ view.handleInput("\x1b");
+ expect(Reflect.get(view, "deleteConfirmExpiresAt")).toBe(0);
+ expect(renderedIncidentLines(view)).toHaveLength(1);
+ } finally {
+ stopThemeWatcher();
+ }
+ });
+
+ it("dismisses with Esc and never resurrects across later polls", () => {
+ useTempAgentDir();
+ const base = Date.now();
+ writeAgentLog([workerCrashLine(base, "5b1d3aeb91ee", 120)]);
+ const persistentState: AgentsViewPersistentState = { savedCatalogLoaded: true };
+ const view = newView(persistentState);
+ try {
+ invoke("refreshIncidentNotices", view);
+ expect(renderedIncidentLines(view)).toHaveLength(1);
+
+ view.handleInput("\x1b");
+ expect(Reflect.get(view, "statusMessage")).toBe("Incident notice dismissed");
+ // Later polls re-read the same incident from the log; dismissal is sticky.
+ invoke("refreshIncidentNotices", view);
+ expect(renderedIncidentLines(view)).toHaveLength(0);
+ } finally {
+ stopThemeWatcher();
+ }
+ });
+
+ it("re-shows a dismissed timeout-burst when a later timeout extends the burst", () => {
+ useTempAgentDir();
+ const base = Date.now();
+ writeAgentLog([commandTimeoutLine(base, 30), commandTimeoutLine(base, 29)]);
+ const state = createIncidentNoticeState();
+ const logPath = getAgentLogPath();
+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);
+ expect(dismissIncidentNoticeState(state)).toBe(true);
+ // A later timeout extends the burst past the dismissed horizon: the
+ // notice reappears instead of staying hidden until the first timeout
+ // ages out.
+ appendAgentLog([commandTimeoutLine(base, 5)]);
+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);
+ expect(state.notice).toMatchObject({ kind: "timeout-burst", timeMs: base - 5 * 60_000 });
+ });
+
+ it("keeps a dismissed timeout-burst hidden when a later timeout is isolated", () => {
+ useTempAgentDir();
+ const base = Date.now();
+ writeAgentLog([commandTimeoutLine(base, 60), commandTimeoutLine(base, 59)]);
+ const state = createIncidentNoticeState();
+ const logPath = getAgentLogPath();
+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);
+ expect(dismissIncidentNoticeState(state)).toBe(true);
+ appendAgentLog([commandTimeoutLine(base, 5)]);
+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(false);
+ });
+
+ it("surfaces a crash at the end of agent.jsonl.old without a trailing newline", () => {
+ useTempAgentDir();
+ writeFileSync(`${getAgentLogPath()}.old`, workerCrashLine(Date.now(), "5b1d3aeb91ee", 120));
+ writeAgentLog([]);
+ const state = createIncidentNoticeState();
+ refreshIncidentNoticeState(state, getAgentLogPath(), Date.now());
+ expect(state.notice).toMatchObject({ kind: "worker-crash" });
+ });
+
+ it("skips the .old bridge when a rotation makes it the live log's own generation", () => {
+ useTempAgentDir();
+ writeAgentLog([supervisorStartLine(Date.now(), 600)]);
+ linkSync(getAgentLogPath(), `${getAgentLogPath()}.old`);
+ const state = createIncidentNoticeState();
+ refreshIncidentNoticeState(state, getAgentLogPath(), Date.now());
+ expect(state.entries).toHaveLength(1);
+ });
+
+ it("completes the un-consumed tail of a generation that rotates out mid-session", () => {
+ useTempAgentDir();
+ const logPath = getAgentLogPath();
+ writeAgentLog([workerCrashLine(Date.now(), "5b1d3aeb91ee", 60)]);
+ const state = createIncidentNoticeState();
+ refreshIncidentNoticeState(state, logPath, Date.now());
+ appendAgentLog([workerCrashLine(Date.now(), "9f2c7a44b021", 30)]);
+ renameSync(logPath, `${logPath}.old`);
+ writeAgentLog([]);
+ refreshIncidentNoticeState(state, logPath, Date.now());
+ expect(state.notice).toMatchObject({ subject: "worker 9f2c7a44b021" });
+ });
+});
diff --git a/packages/coding-agent/test/agents-view-mode.test.ts b/packages/coding-agent/test/agents-view-mode.test.ts
index 88e4c17e14..79032fd2ee 100644
--- a/packages/coding-agent/test/agents-view-mode.test.ts
+++ b/packages/coding-agent/test/agents-view-mode.test.ts
@@ -1032,6 +1032,7 @@ describe("AgentsViewMode catalog performance", () => {
resolveMissingSelectionAnchor: vi.fn(),
refreshHeartbeats: vi.fn(async () => true),
loadStartupNotices: vi.fn(),
+ refreshIncidentNotices: vi.fn(),
rebuildRows: vi.fn(),
clearCtrlCExitHint: vi.fn(),
clearDeleteConfirmation: vi.fn(),
diff --git a/packages/coding-agent/test/incident.test.ts b/packages/coding-agent/test/incident.test.ts
new file mode 100644
index 0000000000..29f9600dee
--- /dev/null
+++ b/packages/coding-agent/test/incident.test.ts
@@ -0,0 +1,735 @@
+import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import stripAnsi from "strip-ansi";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ buildIncidentReport,
+ collectIncidentEvents,
+ collectWorkerPidMap,
+ computeIncidentAnomalies,
+ type IncidentLogEntry,
+ IncidentUsageError,
+ parseIncidentDaemonLogLine,
+ parseIncidentLogLine,
+ parseIncidentOptions,
+ parseIncidentTimeBound,
+ runIncident,
+} from "../src/cli/incident.js";
+import { ENV_AGENT_DIR } from "../src/config.js";
+
+function agentLogLine(fields: Record<string, unknown>): string {
+ return JSON.stringify({ ts: "2026-09-10T20:00:00.000Z", level: "warn", ...fields });
+}
+
+function entry(line: string): IncidentLogEntry {
+ return parseIncidentLogLine(line)!;
+}
+
+const sinceMs = Date.parse("2026-09-10T20:00:00Z");
+const untilMs = Date.parse("2026-09-10T20:30:00Z");
+
+function reportFor(lines: string[], options: { since?: number; until?: number; session?: string } = {}) {
+ const entries = lines.map((line) => parseIncidentLogLine(line)!).filter((item) => item !== undefined);
+ return stripAnsi(
+ buildIncidentReport(entries, {
+ sinceMs: options.since ?? sinceMs,
+ untilMs: options.until ?? untilMs,
+ ...(options.session !== undefined ? { session: options.session } : {}),
+ }).text,
+ );
+}
+
+/** Supervisor-side log line (daemon-supervisor component). */
+function supervisorLine(ts: string, msg: string, extra: Record<string, unknown> = {}): string {
+ return agentLogLine({ ts, component: "coding-agent.daemon-supervisor", ...extra, msg });
+}
+
+/** Worker daemon startup line, the pid sighting that anchors attribution. */
+function workerStartLine(ts: string, socketPath: string, pid: number): string {
+ return agentLogLine({
+ ts,
+ component: "coding-agent.daemon",
+ socketPath,
+ pid,
+ msg: `Prime Agent daemon listening on ${socketPath}`,
+ });
+}
+
+/** Worker daemon log line for a worker running under the fixture pid. */
+function daemonLine(ts: string, socketPath: string, msg: string): string {
+ return agentLogLine({ ts, component: "coding-agent.daemon", socketPath, pid: 53615, msg });
+}
+
+/** Provider stream failure line (rate_limit 429) attributed by pid. */
+function providerFailureLine(ts: string, pid: number): string {
+ return agentLogLine({
+ ts,
+ level: "error",
+ component: "ai.provider",
+ pid,
+ msg: "provider stream failure",
+ kind: "rate_limit",
+ status: 429,
+ });
+}
+
+// ---------------------------------------------------------------------------
+// Green run: normal startup, worker start/stop, no errors.
+// ---------------------------------------------------------------------------
+
+describe("incident timeline for a clean run", () => {
+ it("shows supervisor and worker events with no anomaly entries", () => {
+ const text = reportFor(
+ [
+ [
+ "2026-09-10T20:00:05.000Z",
+ "Prime Agent daemon supervisor e14de15c listening on /tmp/prime-agent-501/daemon.sock",
+ ],
+ [
+ "2026-09-10T20:01:00.000Z",
+ "Session worker 477fef4e85a8 stderr: Prime Agent daemon listening on /tmp/prime-agent-501/worker-a-477fef4e85a8.sock",
+ ],
+ ["2026-09-10T20:02:00.000Z", "Migrated 2 scheduled jobs into session artifacts"],
+ [
+ "2026-09-10T20:10:00.000Z",
+ "Session worker 477fef4e85a8 stderr: shutdown command received over socket; 1 active session(s) will be closed",
+ ],
+ [
+ "2026-09-10T20:10:01.000Z",
+ "Session worker 477fef4e85a8 stderr: shutting down (exit 0); closing 1 active session(s)",
+ ],
+ [
+ "2026-09-10T20:11:00.000Z",
+ "Evicted empty session worker 477fef4e85a8 root=01a0-abc on last client detach",
+ ],
+ ].map(([ts, msg]) => supervisorLine(ts, msg, { pid: 15026 })),
+ );
+ expect(text).toContain("Supervisor events");
+ expect(text).toContain("worker 477fef4e85a8 started");
+ expect(text).toContain("worker 477fef4e85a8 stop requested (1 active session(s))");
+ expect(text).toContain("worker 477fef4e85a8 stopped (exit 0, 1 session(s) closed)");
+ expect(text).toContain("evicted empty session worker 477fef4e85a8 on last detach");
+ expect(text).toContain("migrated 2 scheduled jobs into session artifacts");
+ const anomalies = text.split("Session anomalies")[1]!.split("Recovery")[0]!;
+ expect(anomalies).toContain("(none)");
+ expect(anomalies).not.toContain("timeout");
+ expect(text).not.toContain("critical");
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Crash + recovery: the incident class from 2026-09-10 (attach timeouts
+// 20:02-20:21, EPIPE worker crash, recovery holding backlogged operations).
+// ---------------------------------------------------------------------------
+
+function incidentFixtureLines(): string[] {
+ const lines: string[] = [];
+ const supervisor = {
+ component: "coding-agent.daemon-supervisor",
+ socketPath: "/tmp/prime-agent-501/daemon.sock",
+ pid: 15026,
+ };
+ const crashedWorker = {
+ component: "coding-agent.daemon",
+ socketPath: "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",
+ pid: 53615,
+ };
+ const supervisorMsg = (ts: string, msg: string) => agentLogLine({ ...supervisor, ts, msg });
+ // Worker starts before the window; provider failures are attributed to it by pid.
+ lines.push(workerStartLine("2026-09-10T19:42:09.064Z", crashedWorker.socketPath, 53615));
+ // Attach timeouts 20:02-20:21.
+ for (const [ts, command] of [
+ ["2026-09-10T20:02:39.764Z", "attach"],
+ ["2026-09-10T20:08:37.410Z", "attach"],
+ ["2026-09-10T20:09:16.853Z", "attach"],
+ ["2026-09-10T20:21:53.062Z", "attach"],
+ ] as const) {
+ lines.push(
+ supervisorMsg(
+ ts,
+ `Supervisor command ${command} failed: Error: Timed out waiting for daemon worker response to ${command}\n at Timeout._onTimeout (node:internal/timers:618:7)`,
+ ),
+ );
+ }
+ // Per-session catch-up timeouts for 2339fb7da605 and one other session.
+ lines.push(
+ supervisorMsg(
+ "2026-09-10T20:02:33.708Z",
+ "Failed to catch up client daemon-client:ac0fbf2a for dcdced964c0d: Error: Timed out waiting for daemon worker response to attach",
+ ),
+ );
+ for (const ts of ["2026-09-10T20:04:50.397Z", "2026-09-10T20:11:29.190Z", "2026-09-10T20:14:57.615Z"]) {
+ lines.push(
+ supervisorMsg(
+ ts,
+ "Failed to catch up client daemon-client:dc5ad892 for 2339fb7da605: Error: Timed out waiting for daemon worker response to attach",
+ ),
+ );
+ }
+ lines.push(
+ supervisorMsg(
+ "2026-09-10T20:21:58.137Z",
+ "Could not list heartbeats from a worker: Timed out waiting for daemon worker response to heartbeats_list",
+ ),
+ supervisorMsg(
+ "2026-09-10T20:02:30.000Z",
+ "Supervisor command list_agent_peers failed: Error: Worker authentication failed\n at handleCommand (chunk.js:1:1)",
+ ),
+ );
+ // Provider stream failures from the overloaded worker (real entries carry
+ // only the worker pid, no socket path).
+ for (let index = 0; index < 30; index++) {
+ lines.push(
+ agentLogLine({
+ ts: `2026-09-10T20:${(2 + Math.floor(index / 2)).toString().padStart(2, "0")}:${(index % 60).toString().padStart(2, "0")}.000Z`,
+ level: "error",
+ component: "ai.provider",
+ pid: 53615,
+ msg: "provider stream failure",
+ kind: index % 5 === 0 ? "server_error" : "rate_limit",
+ status: index % 5 === 0 ? 504 : 429,
+ }),
+ );
+ }
+ // EPIPE crash: logged once by the worker and once via the stderr forward.
+ lines.push(
+ agentLogLine({
+ ...crashedWorker,
+ ts: "2026-09-10T20:23:24.945Z",
+ msg: "uncaught exception: Error: write EPIPE\n at afterWriteDispatched (node:internal/stream_base_commons:159:15)",
+ }),
+ supervisorMsg(
+ "2026-09-10T20:23:24.945Z",
+ "Session worker 5b1d3aeb91ee stderr: uncaught exception: Error: write EPIPE",
+ ),
+ supervisorMsg(
+ "2026-09-10T20:23:24.946Z",
+ "Session worker 5b1d3aeb91ee stderr: at afterWriteDispatched (node:internal/stream_base_commons:159:15)",
+ ),
+ );
+ // Recovery: worker replaced, backlogged uncertain operations held.
+ const operations = [
+ ...Array<string>(408).fill("tool_execution_start"),
+ ...Array<string>(62).fill("auto_retry_end"),
+ ...Array<string>(47).fill("agent_end"),
+ ...Array<string>(16).fill("message_start"),
+ ].join(", ");
+ lines.push(
+ supervisorMsg(
+ "2026-09-10T20:23:29.521Z",
+ `Recovered worker 5b1d3aeb91ee without replaying uncertain operations: ${operations}`,
+ ),
+ supervisorMsg(
+ "2026-09-10T20:23:39.520Z",
+ "Could not adopt worker 5b1d3aeb91ee: Error: Session worker process is no longer running",
+ ),
+ );
+ return lines;
+}
+
+describe("incident timeline for a crash + recovery", () => {
+ it("reconstructs the incident narrative in one report", () => {
+ const text = reportFor(incidentFixtureLines());
+ // Timeouts aggregated with count and span.
+ expect(text).toContain("command attach failed: timed out waiting for worker response (x4, until 09-10 20:21:53)");
+ expect(text).toContain(
+ "client catch-up failed for session 2339fb7da605: Timed out waiting for daemon worker response to attach (x3, until 09-10 20:14:57)",
+ );
+ expect(text).toContain("command list_agent_peers failed: worker authentication failed");
+ // The crash appears once, critical, with the EPIPE cause.
+ expect(text.match(/crashed/g)?.length).toBe(1);
+ expect(text).toContain(
+ "09-10 20:23:24 critical worker 5b1d3aeb91ee crashed: uncaught exception: Error: write EPIPE",
+ );
+ // Recovery with the held backlog and its operation breakdown.
+ expect(text).toContain(
+ "worker 5b1d3aeb91ee recovered; 533 uncertain operations not replayed (tool_execution_start x408, auto_retry_end x62, agent_end x47, message_start x16)",
+ );
+ expect(text).toContain("could not adopt worker 5b1d3aeb91ee: Session worker process is no longer running");
+ // Anomalies: a timeout stall, an auth failure, and provider failures attributed to the crashed worker.
+ expect(text).toContain("/tmp/prime-agent-501/daemon.sock: 5 command timeouts over 19m18s");
+ expect(text).toContain("session 2339fb7da605: 3 command timeouts over 10m7s");
+ expect(text).toContain("provider stream failure (rate_limit 429) for worker 5b1d3aeb91ee (x24");
+ expect(text).toContain("provider stream failure (server_error 504) for worker 5b1d3aeb91ee (x6");
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Window and session filtering.
+// ---------------------------------------------------------------------------
+
+describe("incident window and session filtering", () => {
+ it("excludes events outside the window", () => {
+ const lines = incidentFixtureLines();
+ const text = reportFor(lines, {
+ since: Date.parse("2026-09-10T20:05:00Z"),
+ until: Date.parse("2026-09-10T20:10:00Z"),
+ });
+ expect(text).toContain("command attach failed: timed out waiting for worker response (x2, until 09-10 20:09:16)");
+ expect(text).not.toContain("20:21:53");
+ expect(text).not.toContain("write EPIPE");
+ expect(text).not.toContain("Recovered worker");
+ });
+
+ it("filters to events naming the session, with prefix matching", () => {
+ const lines = incidentFixtureLines();
+ const full = reportFor(lines);
+ const filtered = reportFor(lines, { session: "2339fb7" });
+ expect(filtered).toContain("client catch-up failed for session 2339fb7da605");
+ expect(filtered).toContain("session 2339fb7da605: 3 command timeouts over 10m7s");
+ expect(filtered).toContain("Session filter: 2339fb7");
+ expect(filtered).not.toContain("command attach failed: timed out");
+ expect(filtered).not.toContain("write EPIPE");
+ expect(filtered).not.toContain("dcdced964c0d");
+ expect(full).toContain("dcdced964c0d");
+ });
+
+ it("matches session names quoted in log messages", () => {
+ const text = reportFor(
+ [
+ supervisorLine(
+ "2026-09-10T20:05:00.000Z",
+ 'Supervisor command set_session_name failed: Error: Agent name "Faerie" is unavailable',
+ ),
+ ],
+ { session: "Faerie" },
+ );
+ expect(text).toContain('Agent name "Faerie" is unavailable');
+ });
+
+ it("does not let a passivation event with an empty session name match every filter", () => {
+ // daemon-mode logs `name=""` when the session has no name; that empty
+ // token must not prefix-match every --session value.
+ const text = reportFor(
+ [
+ daemonLine(
+ "2026-09-10T20:05:00.000Z",
+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",
+ 'Passivated idle child sessionId=feedface1234 name="" idleMinutes=5',
+ ),
+ ],
+ { session: "aabbccddeeff" },
+ );
+ expect(text).toContain('reference session "aabbccddeeff"');
+ });
+
+ it("keeps session tokens from worker command failures", () => {
+ const text = reportFor(
+ [
+ daemonLine(
+ "2026-09-10T20:05:00.000Z",
+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",
+ 'daemon command "set_session_name" failed: Error: Agent name "Faerie" is unavailable',
+ ),
+ ],
+ { session: "Faerie" },
+ );
+ expect(text).toContain('Agent name "Faerie" is unavailable');
+ });
+
+ it("says clearly when nothing in the window references the session", () => {
+ const text = reportFor(incidentFixtureLines(), { session: "deadbeef1234" });
+ expect(text).toContain('reference session "deadbeef1234"');
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Time parsing and option parsing.
+// ---------------------------------------------------------------------------
+
+describe("parseIncidentTimeBound", () => {
+ const now = new Date(Date.parse("2026-09-16T22:30:00Z"));
+
+ it("parses ISO date-times, dates, and bare times as UTC", () => {
+ expect(parseIncidentTimeBound("2026-09-16T20:02", now, "--since")).toBe(Date.parse("2026-09-16T20:02:00.000Z"));
+ expect(parseIncidentTimeBound("2026-09-16", now, "--since")).toBe(Date.parse("2026-09-16T00:00:00.000Z"));
+ expect(parseIncidentTimeBound("20:02", now, "--since")).toBe(Date.parse("2026-09-16T20:02:00.000Z"));
+ expect(parseIncidentTimeBound("2026-09-16T20:02:53Z", now, "--since")).toBe(
+ Date.parse("2026-09-16T20:02:53.000Z"),
+ );
+ expect(parseIncidentTimeBound("2026-09-16T22:02+02:00", now, "--since")).toBe(
+ Date.parse("2026-09-16T20:02:00.000Z"),
+ );
+ });
+
+ it("accepts fractional seconds pasted from the log", () => {
+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.764Z", now, "--since")).toBe(
+ Date.parse("2026-09-16T20:02:39.764Z"),
+ );
+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.764", now, "--since")).toBe(
+ Date.parse("2026-09-16T20:02:39.764Z"),
+ );
+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.7", now, "--since")).toBe(
+ Date.parse("2026-09-16T20:02:39.700Z"),
+ );
+ });
+
+ it("rejects timezone offsets with an invalid minute component", () => {
+ // RFC 3339 offsets allow minutes 00-59 only; +00:60 must not shift by an hour.
+ for (const garbage of ["2026-09-16T20:02+00:60", "2026-09-16T20:02-05:90", "2026-09-16T20:02+24:00"]) {
+ expect(() => parseIncidentTimeBound(garbage, now, "--since")).toThrow(IncidentUsageError);
+ }
+ expect(parseIncidentTimeBound("2026-09-16T20:02+05:45", now, "--since")).toBe(
+ Date.parse("2026-09-16T14:17:00.000Z"),
+ );
+ });
+
+ it("rejects garbage and impossible dates with a clear usage error", () => {
+ for (const garbage of ["yesterday", "2026-13-01", "2026-09-32", "25:00", "2026-09-16T"]) {
+ expect(() => parseIncidentTimeBound(garbage, now, "--since")).toThrow(IncidentUsageError);
+ }
+ const message = (() => {
+ try {
+ parseIncidentTimeBound("garbage", now, "--since");
+ } catch (error) {
+ return (error as Error).message;
+ }
+ })();
+ expect(message).toContain("--since");
+ expect(message).toContain("20:02");
+ });
+});
+
+describe("parseIncidentOptions", () => {
+ it("parses separated and =-attached values", () => {
+ expect(parseIncidentOptions(["--since", "20:02", "--until=21:00", "--session", "abc"])).toEqual({
+ since: "20:02",
+ until: "21:00",
+ session: "abc",
+ });
+ expect(parseIncidentOptions([])).toEqual({});
+ });
+
+ it("rejects unknown options, missing values, and empty values", () => {
+ expect(() => parseIncidentOptions(["--json"])).toThrow(IncidentUsageError);
+ expect(() => parseIncidentOptions(["extra"])).toThrow(IncidentUsageError);
+ expect(() => parseIncidentOptions(["--since"])).toThrow(IncidentUsageError);
+ expect(() => parseIncidentOptions(["--since", ""])).toThrow(IncidentUsageError);
+ expect(() => parseIncidentOptions(["--since="])).toThrow(IncidentUsageError);
+ expect(() => parseIncidentOptions(["--session", " "])).toThrow(IncidentUsageError);
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Per-daemon log fallback and malformed-line robustness.
+// ---------------------------------------------------------------------------
+
+describe("parseIncidentDaemonLogLine", () => {
+ it("parses supervisor and worker lines", () => {
+ const supervisor = parseIncidentDaemonLogLine(
+ "[2026-09-10T20:02:39.765Z] supervisor: Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",
+ );
+ expect(supervisor?.component).toBe("coding-agent.daemon-supervisor");
+ expect(supervisor?.timeMs).toBe(Date.parse("2026-09-10T20:02:39.765Z"));
+ const worker = parseIncidentDaemonLogLine("[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE");
+ expect(worker?.component).toBe("coding-agent.daemon");
+ expect(worker?.msg).toBe("uncaught exception: Error: write EPIPE");
+ });
+
+ it("returns undefined for malformed lines", () => {
+ expect(parseIncidentDaemonLogLine("not a log line")).toBeUndefined();
+ expect(parseIncidentDaemonLogLine("[not-a-timestamp] supervisor: msg")).toBeUndefined();
+ expect(parseIncidentDaemonLogLine("[]")).toBeUndefined();
+ });
+});
+
+describe("runIncident over a fixture agent dir", () => {
+ let agentDir: string;
+ let logs: string[];
+
+ beforeEach(() => {
+ agentDir = mkdtempSync(join(tmpdir(), "prime-incident-test-"));
+ logs = [];
+ vi.spyOn(console, "log").mockImplementation((line: string) => {
+ logs.push(line);
+ });
+ process.env[ENV_AGENT_DIR] = agentDir;
+ });
+
+ afterEach(() => {
+ delete process.env[ENV_AGENT_DIR];
+ vi.restoreAllMocks();
+ rmSync(agentDir, { recursive: true, force: true });
+ });
+
+ it("tolerates malformed lines and reports them as skipped", async () => {
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ const contents = [
+ "this is not json",
+ JSON.stringify({ noTs: true, msg: "missing ts" }),
+ supervisorLine(
+ "2026-09-10T20:02:39.764Z",
+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",
+ ),
+ "",
+ ].join("\n");
+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), `${contents}\n`);
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("command attach failed: timed out waiting for worker response");
+ expect(text).toContain("3 lines scanned, 1 events in window, 2 unreadable skipped");
+ });
+
+ it("falls back to the newest per-daemon log when agent.jsonl is absent", async () => {
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ writeFileSync(
+ join(agentDir, "logs", "old-sock.a1b2c3d4.log"),
+ "[2026-09-09T10:00:00.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held\n",
+ );
+ writeFileSync(
+ join(agentDir, "logs", "daemon.sock.98ed5cb2.log"),
+ [
+ "[2026-09-10T20:00:05.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held",
+ "[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE",
+ "[2026-09-10T20:23:29.521Z] supervisor: Recovered worker 5b1d3aeb91ee without replaying uncertain operations: tool_execution_start, agent_end",
+ ].join("\n"),
+ );
+ const decoyDir = join(agentDir, "logs", "daemon.sock.deadbeef.log");
+ mkdirSync(decoyDir);
+ utimesSync(decoyDir, new Date(Date.now() + 60_000), new Date(Date.now() + 60_000));
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("daemon.sock.98ed5cb2.log");
+ expect(text).toContain("supervisor startup blocked: another daemon holds the lock");
+ expect(text).toContain(
+ "worker 5b1d3aeb91ee recovered; 2 uncertain operations not replayed (agent_end x1, tool_execution_start x1)",
+ );
+ });
+
+ it("falls back to daemon logs whose socket basename does not end in .sock", async () => {
+ // A daemon started with `--daemon-socket /tmp/prime-daemon` writes
+ // prime-daemon.<hash>.log, which has no .sock segment to match on.
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ writeFileSync(
+ join(agentDir, "logs", "prime-daemon.a1b2c3d4.log"),
+ "[2026-09-10T20:00:05.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held\n",
+ );
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("prime-daemon.a1b2c3d4.log");
+ expect(text).toContain("supervisor startup blocked: another daemon holds the lock");
+ });
+
+ it("uses the pre-resolved window passed by the caller", async () => {
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ writeFileSync(
+ join(agentDir, "logs", "agent.jsonl"),
+ `${supervisorLine("2026-09-10T20:02:39.764Z", "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach")}\n`,
+ );
+ await runIncident(
+ {},
+ { sinceMs: Date.parse("2026-09-10T20:00:00Z"), untilMs: Date.parse("2026-09-10T20:30:00Z") },
+ );
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("Window: 09-10 20:00:00 → 09-10 20:30:00 UTC (30m)");
+ expect(text).toContain("command attach failed: timed out waiting for worker response");
+ });
+
+ it("falls back to the per-daemon log when agent.jsonl is empty or unreadable", async () => {
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ const daemonLog = [
+ "[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE",
+ "[2026-09-10T20:23:29.521Z] supervisor: Recovered worker 5b1d3aeb91ee without replaying uncertain operations: tool_execution_start, agent_end",
+ ].join("\n");
+ writeFileSync(join(agentDir, "logs", "daemon.sock.98ed5cb2.log"), `${daemonLog}\n`);
+ // Unreadable agent.jsonl: nothing parses.
+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), "this is not json\nalso not json\n");
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ let text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("daemon.sock.98ed5cb2.log");
+ expect(text).toContain("worker crashed: uncaught exception: Error: write EPIPE");
+ expect(text).toContain("worker 5b1d3aeb91ee recovered; 2 uncertain operations not replayed");
+ // Empty agent.jsonl: zero parsed entries also triggers the fallback.
+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), "");
+ logs.length = 0;
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("daemon.sock.98ed5cb2.log");
+ expect(text).toContain("worker crashed: uncaught exception: Error: write EPIPE");
+ });
+
+ it("does not fall back when agent.jsonl parses but has no in-window events", async () => {
+ mkdirSync(join(agentDir, "logs"), { recursive: true });
+ const outOfWindow = supervisorLine(
+ "2026-09-01T10:00:00.000Z",
+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",
+ );
+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), `${outOfWindow}\n`);
+ writeFileSync(
+ join(agentDir, "logs", "daemon.sock.98ed5cb2.log"),
+ "[2026-09-10T20:23:24.945Z] supervisor: Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach\n",
+ );
+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("agent.jsonl");
+ expect(text).not.toContain("daemon.sock.98ed5cb2.log");
+ expect(text).toContain("(none)");
+ });
+
+ it("reports missing logs clearly", async () => {
+ await runIncident({});
+ const text = stripAnsi(logs.join("\n"));
+ expect(text).toContain("No daemon logs found under");
+ });
+});
+
+// ---------------------------------------------------------------------------
+// Unit-level checks for pid attribution and anomaly computation.
+// ---------------------------------------------------------------------------
+
+describe("worker pid attribution and anomalies", () => {
+ it("attributes provider failures to the worker that owns the pid", () => {
+ const lines = [
+ workerStartLine(
+ "2026-09-10T19:42:09.064Z",
+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",
+ 53615,
+ ),
+ providerFailureLine("2026-09-10T20:02:53.374Z", 53615),
+ ];
+ const entries = lines.map((line) => entry(line));
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ expect(events.some((item) => item.summary.includes("for worker 5b1d3aeb91ee"))).toBe(true);
+ });
+
+ it("flags long gaps between events for the same session", () => {
+ const base = {
+ component: "coding-agent.daemon-supervisor",
+ msg: "Failed to catch up client c1 for aabbccddeeff: Error: Daemon worker socket closed",
+ };
+ const entries = [
+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:00:00.000Z" })),
+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:05:00.000Z" })),
+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:30:00.000Z" })),
+ ];
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const anomalies = computeIncidentAnomalies(events);
+ const gap = anomalies.find((item) => item.summary.includes("event gap"));
+ expect(gap?.summary).toBe("session aabbccddeeff: 25m event gap (no logged events)");
+ });
+
+ it("keeps anomalies and aggregation per daemon socket instead of merging daemons", () => {
+ const daemonA = { component: "coding-agent.daemon-supervisor", socketPath: "/tmp/prime-agent-501/daemon.sock" };
+ const daemonB = { component: "coding-agent.daemon-supervisor", socketPath: "/tmp/other/daemon.sock" };
+ const times = [
+ [daemonA, "20:02"],
+ [daemonA, "20:08"],
+ [daemonA, "20:09"],
+ [daemonA, "20:21"],
+ [daemonB, "20:03"],
+ [daemonB, "20:10"],
+ [daemonB, "20:30"],
+ ] as const;
+ const text = reportFor(
+ times.map(([daemon, time]) =>
+ supervisorLine(
+ `2026-09-10T${time}:00.000Z`,
+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",
+ daemon,
+ ),
+ ),
+ );
+ expect(text).toContain("/tmp/prime-agent-501/daemon.sock: 4 command timeouts over 19m");
+ expect(text).toContain("/tmp/other/daemon.sock: 3 command timeouts over 27m");
+ expect(text).toContain("command attach failed: timed out waiting for worker response (x4, until 09-10 20:21:00)");
+ });
+
+ it("does not report isolated failures days apart as one burst or stall", () => {
+ const burstMsg = "Supervisor command send_message failed: Error: Unknown active session: aabbccddeeff";
+ const stallMsg =
+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach";
+ const cases = [
+ ["2026-09-10T20:00:00.000Z", burstMsg],
+ ["2026-09-12T20:00:00.000Z", burstMsg],
+ ["2026-09-14T20:00:00.000Z", burstMsg],
+ ["2026-09-10T20:00:00.000Z", stallMsg],
+ ["2026-09-12T20:00:00.000Z", stallMsg],
+ ];
+ const entries = cases.map(([ts, msg]) => entry(supervisorLine(ts, msg)));
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const anomalies = computeIncidentAnomalies(events);
+ expect(anomalies.some((item) => item.summary.includes("warnings/errors"))).toBe(false);
+ expect(anomalies.some((item) => item.summary.includes("command timeouts"))).toBe(false);
+ });
+
+ it("keeps a same-component worker restart within two seconds", () => {
+ const socketPath = "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock";
+ const text = reportFor([
+ workerStartLine("2026-09-10T20:00:00.000Z", socketPath, 100),
+ workerStartLine("2026-09-10T20:00:01.000Z", socketPath, 200),
+ ]);
+ expect(text).toContain("worker 5b1d3aeb91ee started (x2, until 09-10 20:00:01)");
+ });
+
+ it("attributes provider failures to the worker owning the pid at that time", () => {
+ const workerA = "/tmp/prime-agent-501/worker-98ed5cb228d2-aaaaaaaaaaaa.sock";
+ const workerB = "/tmp/prime-agent-501/worker-98ed5cb228d2-bbbbbbbbbbbb.sock";
+ const lines = [
+ workerStartLine("2026-09-10T20:00:00.000Z", workerA, 53615),
+ providerFailureLine("2026-09-10T20:01:00.000Z", 53615),
+ // The same pid is reused by a replacement worker an hour later.
+ workerStartLine("2026-09-10T21:00:00.000Z", workerB, 53615),
+ providerFailureLine("2026-09-10T21:01:00.000Z", 53615),
+ ];
+ const entries = lines.map((line) => entry(line));
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const summaries = events.filter((item) => item.eventClass === "provider").map((item) => item.summary);
+ expect(summaries).toEqual([
+ "provider stream failure (rate_limit 429) for worker aaaaaaaaaaaa",
+ "provider stream failure (rate_limit 429) for worker bbbbbbbbbbbb",
+ ]);
+ });
+
+ it("classifies worker events for Windows named-pipe socket paths", () => {
+ const socketPath = "\\\\.\\pipe\\prime-agent-worker-98ed5cb228d2-5b1d3aeb91ee";
+ const entries = [
+ entry(workerStartLine("2026-09-10T20:00:00.000Z", socketPath, 53615)),
+ entry(daemonLine("2026-09-10T20:23:24.945Z", socketPath, "uncaught exception: Error: write EPIPE")),
+ ];
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ expect(events.map((item) => item.eventClass)).toEqual(["worker-start", "worker-crash"]);
+ expect(events.every((item) => item.subject === "worker 5b1d3aeb91ee")).toBe(true);
+ });
+
+ it("keys unknown worker diagnostics as worker events", () => {
+ const socketPath = "/tmp/prime-agent-501/worker-98ed5cb228d2-aaaaaaaaaaaa.sock";
+ const entries = [1, 2, 3].map((index) =>
+ entry(daemonLine(`2026-09-10T20:0${index}:00.000Z`, socketPath, `unrecognized worker diagnostic ${index}`)),
+ );
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const anomalies = computeIncidentAnomalies(events);
+ expect(anomalies.some((item) => item.summary.startsWith("worker aaaaaaaaaaaa: 3 warnings/errors over 2m"))).toBe(
+ true,
+ );
+ expect(anomalies.some((item) => item.summary.includes(socketPath))).toBe(false);
+ });
+
+ it("does not attribute a provider failure to a worker sighted later on the pid", () => {
+ const workerB = "/tmp/prime-agent-501/worker-98ed5cb228d2-bbbbbbbbbbbb.sock";
+ const lines = [
+ providerFailureLine("2026-09-10T10:00:00.000Z", 53615),
+ // The pid's only sighting is a worker that starts ten minutes later;
+ // a future sighting is never evidence of ownership at 10:00.
+ workerStartLine("2026-09-10T10:10:00.000Z", workerB, 53615),
+ ];
+ const entries = lines.map((line) => entry(line));
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const summaries = events.filter((item) => item.eventClass === "provider").map((item) => item.summary);
+ expect(summaries).toEqual(["provider stream failure (rate_limit 429) for pid 53615"]);
+ });
+
+ it("flags error bursts that are not timeouts", () => {
+ const entries = [1, 2, 3].map((index) =>
+ entry(
+ supervisorLine(
+ `2026-09-10T20:0${index}:00.000Z`,
+ `Supervisor command send_message failed: Error: Unknown active session: aabbccdd${index}`,
+ ),
+ ),
+ );
+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));
+ const anomalies = computeIncidentAnomalies(events);
+ expect(anomalies.some((item) => item.summary.includes("3 warnings/errors over 2m"))).toBe(true);
+ });
+});
diff --git a/packages/coding-agent/test/public-command.test.ts b/packages/coding-agent/test/public-command.test.ts
index 03449a44c8..e08fe428a4 100644
--- a/packages/coding-agent/test/public-command.test.ts
+++ b/packages/coding-agent/test/public-command.test.ts
@@ -8,6 +8,8 @@ const mocks = vi.hoisted(() => ({
reapCalls: [] as Array<[boolean, boolean]>,
shutdownCalls: [] as Array<[boolean, boolean]>,
mcpCommands: [] as string[][],
+ incidentCalls: [] as Array<Record<string, string | undefined>>,
+ incidentWindows: [] as Array<{ sinceMs: number; untilMs: number } | undefined>,
}));
vi.mock("../src/cli/daemon-command.js", () => ({
@@ -38,6 +40,14 @@ vi.mock("../src/core/settings-manager.js", () => ({
},
}));
+vi.mock("../src/cli/incident.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../src/cli/incident.js")>()),
+ runIncident: async (options: Record<string, string | undefined>, window?: { sinceMs: number; untilMs: number }) => {
+ mocks.incidentCalls.push(options);
+ mocks.incidentWindows.push(window);
+ },
+}));
+
vi.mock("../src/cli/daemon-ps.js", () => ({
runPs: async (json: boolean) => {
mocks.psCalls.push(json);
@@ -51,6 +61,7 @@ vi.mock("../src/cli/daemon-ps.js", () => ({
}));
import { INTERNAL_RUNTIME_COMMAND_MARKER } from "../src/cli/args.js";
+import { formatTopLevelHelp } from "../src/cli/command-registry.js";
import { DAEMON_UPDATE_RESTART_COORDINATOR_FLAG } from "../src/cli/daemon-update-restart.js";
import { handlePublicCommand } from "../src/cli/public-command.js";
@@ -62,6 +73,8 @@ describe("public command routing", () => {
mocks.reapCalls.length = 0;
mocks.shutdownCalls.length = 0;
mocks.mcpCommands.length = 0;
+ mocks.incidentCalls.length = 0;
+ mocks.incidentWindows.length = 0;
process.exitCode = undefined;
vi.spyOn(console, "log").mockImplementation(() => {});
vi.spyOn(console, "error").mockImplementation(() => {});
@@ -484,4 +497,61 @@ describe("public command routing", () => {
expect(process.exitCode).toBe(1);
expect(mocks.daemonCommands).toEqual([]);
});
+
+ it("routes the incident command with parsed window options", async () => {
+ await expect(
+ handlePublicCommand(["incident", "--since", "20:02", "--until=21:00", "--session", "abc"]),
+ ).resolves.toEqual({
+ handled: true,
+ args: [],
+ explicitAgentsView: false,
+ });
+ expect(mocks.incidentCalls).toEqual([{ since: "20:02", until: "21:00", session: "abc" }]);
+ expect(process.exitCode).toBeUndefined();
+ });
+
+ it("resolves the incident window once and passes it to runIncident", async () => {
+ vi.useFakeTimers();
+ // Just before UTC midnight: a second resolution later would land on the
+ // next day and render a different window for relative HH:MM bounds.
+ vi.setSystemTime(new Date("2026-09-16T23:59:59.900Z"));
+ try {
+ await handlePublicCommand(["incident", "--since", "23:00", "--until", "23:30"]);
+ expect(mocks.incidentCalls).toEqual([{ since: "23:00", until: "23:30" }]);
+ expect(mocks.incidentWindows).toEqual([
+ { sinceMs: Date.parse("2026-09-16T23:00:00.000Z"), untilMs: Date.parse("2026-09-16T23:30:00.000Z") },
+ ]);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("rejects unknown incident options with usage guidance", async () => {
+ await expect(handlePublicCommand(["incident", "--json"])).resolves.toMatchObject({ handled: true });
+ expect(process.exitCode).toBe(1);
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining("Unknown option for incident: --json"));
+ expect(mocks.incidentCalls).toEqual([]);
+ });
+
+ it("rejects an incident window where until precedes since with usage guidance", async () => {
+ await expect(
+ handlePublicCommand(["incident", "--since", "2026-09-10T20:30", "--until", "2026-09-10T20:00"]),
+ ).resolves.toMatchObject({ handled: true });
+ expect(process.exitCode).toBe(1);
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining("--until must be after --since."));
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Run "prime-agent help incident" for usage.'));
+ expect(mocks.incidentCalls).toEqual([]);
+ });
+
+ it("rejects a bad incident time with usage guidance", async () => {
+ await expect(handlePublicCommand(["incident", "--since", "yesterday"])).resolves.toMatchObject({ handled: true });
+ expect(process.exitCode).toBe(1);
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Invalid time for --since: "yesterday"'));
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Run "prime-agent help incident" for usage.'));
+ expect(mocks.incidentCalls).toEqual([]);
+ });
+
+ it("shows incident in the top-level command list", () => {
+ expect(formatTopLevelHelp()).toContain("incident");
+ });
});