A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, feature] Daemon incident forensics: prime-agent incident CLI and agents-view notices (#2406)
Seth Kartenf0223369ac85
diff --git a/packages/coding-agent/.changes/agents-view-incident-notice.md b/packages/coding-agent/.changes/agents-view-incident-notice.mdnew file mode 100644index 0000000000..e5dd4a1df0--- /dev/null+++ b/packages/coding-agent/.changes/agents-view-incident-notice.md@@ -0,0 +1 @@+- Added daemon incident notices to the agents view: recent worker crashes, command-timeout bursts, and update restarts surface as a dismissible status line pointing at `prime-agent incident`.diff --git a/packages/coding-agent/.changes/daemon-incident-cli.md b/packages/coding-agent/.changes/daemon-incident-cli.mdnew file mode 100644index 0000000000..9c956c2eeb--- /dev/null+++ b/packages/coding-agent/.changes/daemon-incident-cli.md@@ -0,0 +1 @@+- Added `prime-agent incident` to reconstruct daemon incidents from logs into an operator timeline (supervisor events, session anomalies, recovery actions) with `--since`/`--until` window and `--session` filters.diff --git a/packages/coding-agent/src/cli/command-registry.ts b/packages/coding-agent/src/cli/command-registry.tsindex f160929ae9..b3cdbebcb8 100644--- a/packages/coding-agent/src/cli/command-registry.ts+++ b/packages/coding-agent/src/cli/command-registry.ts@@ -86,6 +86,19 @@ export const COMMAND_SPECS: readonly CommandSpec[] = [summary: "Inspect and safely clean up background services",options: ["--fix Remove stale sockets and stop idle orphaned services", "--json Print JSON"],},+ {+ path: ["incident"],+ usage: "incident [--since <time>] [--until <time>] [--session <id>]",+ summary: "Reconstruct a daemon incident from its logs",+ description:+ "Summarizes the daemon logs for a time window into an operator timeline: supervisor and worker events, session anomalies, and recovery actions. Times without a timezone are read as UTC, matching the log; the default window is the last 24 hours.",+ options: [+ "--since <time> Window start (ISO date/time, date, or HH:MM today; default: 24h ago)",+ "--until <time> Window end (default: now)",+ "--session <id> Only events naming this session id, worker id, or session name (prefix match)",+ ],+ examples: ['incident --since "2026-09-16T20:02" --until "2026-09-16T20:21"', "incident --session 2339fb7da605"],+ },{path: ["shutdown"],usage: "shutdown [--force] [--json]",diff --git a/packages/coding-agent/src/cli/incident.ts b/packages/coding-agent/src/cli/incident.tsnew file mode 100644index 0000000000..dbd9a87c67--- /dev/null+++ b/packages/coding-agent/src/cli/incident.ts@@ -0,0 +1,1279 @@+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";+import { join } from "node:path";+import chalk from "chalk";+import { getAgentLogPath, getLogsDir } from "../config.js";++/**+ * `prime-agent incident` reconstructs what the daemon did during a time window+ * from its diagnostic logs, so an operator does not have to grep hundreds of+ * raw log lines by hand. The primary source is the shared structured log+ * (~/.prime/agent/logs/agent.jsonl, one JSON object per line, written by the+ * coding-agent.daemon-supervisor, coding-agent.daemon, and ai.provider+ * components). When that file is missing or unreadable, the newest per-daemon+ * log file (~/.prime/agent/logs/<socket>.<hash>.log, plain-text lines) is used+ * as a fallback.+ *+ * Message shapes below match the strings the supervisor and session worker+ * actually emit (the DaemonSupervisor.log / DaemonMode.log call sites).+ * Unknown warning lines fall through to a generic per-line summary so new log+ * messages degrade to a readable timeline instead of disappearing.+ */++export type IncidentSeverity = "critical" | "error" | "warn" | "info";++export type IncidentCategory = "supervisor" | "anomaly" | "recovery";++export interface IncidentLogEntry {+ timeMs: number;+ level: string;+ component: string;+ msg: string;+ socketPath?: string;+ pid?: number;+ /** Raw log-record fields, used for provider failure details (kind/status). */+ fields: Record<string, unknown>;+}++export interface IncidentEvent {+ timeMs: number;+ severity: IncidentSeverity;+ category: IncidentCategory;+ eventClass: string;+ subject: string;+ summary: string;+ tokens: string[];+}++export interface IncidentCommandOptions {+ since?: string;+ until?: string;+ session?: string;+}++export interface IncidentReportOptions {+ sinceMs: number;+ untilMs: number;+ session?: string;+ source?: string;+ scannedCount?: number;+ skippedCount?: number;+}++export interface IncidentReport {+ text: string;+}++export interface IncidentLogSource {+ entries: IncidentLogEntry[];+ scannedCount: number;+ skippedCount: number;+ source: string;+}++export class IncidentUsageError extends Error {}++const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000;+const STALL_GAP_MS = 10 * 60 * 1000;+const ERROR_BURST_THRESHOLD = 3;+/** A burst is several warnings/errors close together; isolated failures far apart are not one incident. */+const ERROR_BURST_WINDOW_MS = 10 * 60 * 1000;+/**+ * A stall is repeated command timeouts close together, like the burst window+ * above; timeouts further apart than this never merge into one stall, whatever+ * the report window is. The agents-view dismissal horizon derives from the+ * same bound via latestIncidentStallTimeoutBySubject.+ */+const TIMEOUT_STALL_WINDOW_MS = 30 * 60 * 1000;+const SUMMARY_TRUNCATION = 120;+const RECOVERY_BREAKDOWN_LIMIT = 4;+const WORKER_SOCKET_PATTERN = /^(?:prime-agent-)?worker-[0-9a-f]+-([0-9a-f]{12})(?:\.sock)?$/;+// Per-daemon logs are named `<socket basename>.<hash8>.log` (config.ts+// getDaemonLogPath); the socket basename itself may lack `.sock` for custom+// sockets and Windows named pipes, so key on the hash suffix instead.+const DAEMON_LOG_FILE_PATTERN = /\.[0-9a-f]{8}\.log$/;+const BURST_CLASSES = new Set(["command-failure", "auth", "diagnostic"]);+const LIFECYCLE_CLASSES = new Set(["worker-start", "worker-stop", "worker-crash", "worker-passivation"]);++// ---------------------------------------------------------------------------+// Time parsing+// ---------------------------------------------------------------------------++const FULL_TIME_PATTERN =+ /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.(\d{1,3}))?)?(Z|[+-]\d{2}(?::?\d{2})?)?$/;+const TIME_ONLY_PATTERN = /^(\d{2}):(\d{2})$/;++/**+ * Parse a --since/--until bound. Times without a timezone are read as UTC,+ * matching the timestamps the daemon logs; "20:02" means today at 20:02 UTC.+ */+export function parseIncidentTimeBound(value: string, now: Date, flag: string): number {+ const raw = value.trim();+ if (!raw) {+ throw new IncidentUsageError(`${flag} requires a time.`);+ }+ const invalid = () =>+ new IncidentUsageError(+ `Invalid time for ${flag}: "${value}". Use "2026-09-16T20:02", "2026-09-16", or "20:02" (today, UTC).`,+ );+ const timeOnly = TIME_ONLY_PATTERN.exec(raw);+ if (timeOnly) {+ const hour = Number(timeOnly[1]);+ const minute = Number(timeOnly[2]);+ if (hour > 23 || minute > 59) {+ throw invalid();+ }+ return Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, minute);+ }+ const full = FULL_TIME_PATTERN.exec(raw);+ if (!full) {+ throw invalid();+ }+ const [, year, month, day, hour, minute, second, fraction, zone] = full;+ const baseMs = Date.UTC(+ Number(year),+ Number(month) - 1,+ Number(day),+ hour !== undefined ? Number(hour) : 0,+ minute !== undefined ? Number(minute) : 0,+ second !== undefined ? Number(second) : 0,+ fraction !== undefined ? Number(fraction.padEnd(3, "0")) : 0,+ );+ const base = new Date(baseMs);+ if (+ base.getUTCFullYear() !== Number(year) ||+ base.getUTCMonth() !== Number(month) - 1 ||+ base.getUTCDate() !== Number(day) ||+ (hour !== undefined && base.getUTCHours() !== Number(hour)) ||+ (minute !== undefined && base.getUTCMinutes() !== Number(minute))+ ) {+ throw invalid();+ }+ if (!zone || zone === "Z") {+ return baseMs;+ }+ const sign = zone[0] === "-" ? -1 : 1;+ const digits = zone.slice(1).replace(":", "");+ const offsetMinute = Number(digits.slice(2) || 0);+ const offsetMinutes = Number(digits.slice(0, 2)) * 60 + offsetMinute;+ if (!Number.isFinite(offsetMinutes) || offsetMinutes >= 24 * 60 || offsetMinute >= 60) {+ throw invalid();+ }+ return baseMs - sign * offsetMinutes * 60_000;+}++// ---------------------------------------------------------------------------+// Log line parsing+// ---------------------------------------------------------------------------++function timestampToMs(ts: string): number {+ const ms = Date.parse(ts);+ return Number.isNaN(ms) ? Number.NaN : ms;+}++/** Parse one agent.jsonl line; malformed lines return undefined. */+export function parseIncidentLogLine(line: string): IncidentLogEntry | undefined {+ const trimmed = line.trim();+ if (!trimmed) {+ return undefined;+ }+ let parsed: unknown;+ try {+ parsed = JSON.parse(trimmed);+ } catch {+ return undefined;+ }+ if (typeof parsed !== "object" || parsed === null) {+ return undefined;+ }+ const record = parsed as Record<string, unknown>;+ if (typeof record.ts !== "string" || typeof record.msg !== "string") {+ return undefined;+ }+ const timeMs = timestampToMs(record.ts);+ if (!Number.isFinite(timeMs)) {+ return undefined;+ }+ return {+ timeMs,+ level: typeof record.level === "string" ? record.level : "warn",+ component: typeof record.component === "string" ? record.component : "unknown",+ msg: record.msg,+ ...(typeof record.socketPath === "string" ? { socketPath: record.socketPath } : {}),+ ...(typeof record.pid === "number" ? { pid: record.pid } : {}),+ fields: record,+ };+}++/** Parse one per-daemon log line: `[<ISO>] supervisor: <msg>` or `[<ISO>] <msg>`. */+export function parseIncidentDaemonLogLine(line: string): IncidentLogEntry | undefined {+ const match = /^\[([^\]]+)\]\s*(.*)$/.exec(line.trim());+ if (!match) {+ return undefined;+ }+ const [, ts, rest] = match;+ if (!ts || rest === undefined || !rest.trim()) {+ return undefined;+ }+ const timeMs = timestampToMs(ts);+ if (!Number.isFinite(timeMs)) {+ return undefined;+ }+ const supervisorLine = rest.startsWith("supervisor:");+ const msg = supervisorLine ? rest.slice("supervisor:".length).trim() : rest;+ if (!msg) {+ return undefined;+ }+ return {+ timeMs,+ level: "warn",+ component: supervisorLine ? "coding-agent.daemon-supervisor" : "coding-agent.daemon",+ msg,+ fields: {},+ };+}++// ---------------------------------------------------------------------------+// Classification+// ---------------------------------------------------------------------------++function firstLine(text: string): string {+ return text.split("\n")[0]!.trim();+}++function truncateText(text: string, max: number): string {+ const trimmed = text.trim();+ return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max)}...`;+}++function errorMessage(message: string): string {+ return message.replace(/^Error:\s*/, "");+}++function workerIdFromSocketPath(socketPath: string | undefined): string | undefined {+ if (!socketPath) {+ return undefined;+ }+ // Split on both separators so Windows named-pipe paths (`\\.\pipe\...`)+ // resolve to their last segment on any platform, not only on win32.+ const name = socketPath.split(/[\\/]/).pop()!;+ return WORKER_SOCKET_PATTERN.exec(name)?.[1];+}++/** One log sighting of a worker id owning a pid at a timestamp. */+export interface WorkerPidSighting {+ timeMs: number;+ workerId: string;+}++/**+ * pid -> worker-id sightings, ordered by time. A pid can be reused by a later+ * worker, so attribution picks the owner at the event time, not the union of+ * every id ever seen on the pid.+ */+export type WorkerPidMap = ReadonlyMap<number, ReadonlyArray<WorkerPidSighting>>;++/** Map pid -> worker-id sightings from entries whose socket path names a worker socket. */+export function collectWorkerPidMap(entries: readonly IncidentLogEntry[]): WorkerPidMap {+ const map = new Map<number, WorkerPidSighting[]>();+ for (const entry of entries) {+ const workerId = workerIdFromSocketPath(entry.socketPath);+ if (workerId === undefined || entry.pid === undefined) {+ continue;+ }+ const sightings = map.get(entry.pid) ?? [];+ sightings.push({ timeMs: entry.timeMs, workerId });+ map.set(entry.pid, sightings);+ }+ for (const sightings of map.values()) {+ sightings.sort((a, b) => a.timeMs - b.timeMs);+ }+ return map;+}++/** The worker that owned `pid` at `timeMs`: the latest sighting at or before it. */+function workerIdForPid(workerPids: WorkerPidMap, pid: number | undefined, timeMs: number): string | undefined {+ if (pid === undefined) {+ return undefined;+ }+ const sightings = workerPids.get(pid);+ if (sightings === undefined || sightings.length === 0) {+ return undefined;+ }+ let workerId: string | undefined;+ for (const sighting of sightings) {+ if (sighting.timeMs <= timeMs) {+ workerId = sighting.workerId;+ } else {+ break;+ }+ }+ return workerId;+}++function classifyCommandFailure(+ command: string,+ error: string,+): {+ severity: IncidentSeverity;+ eventClass: string;+ summary: string;+ tokens: string[];+} {+ const err = firstLine(error);+ if (err.includes("Timed out waiting for daemon worker response")) {+ return {+ severity: "error",+ eventClass: "timeout",+ summary: `command ${command} failed: timed out waiting for worker response`,+ tokens: [],+ };+ }+ if (/authentication failed/i.test(err)) {+ return {+ severity: "error",+ eventClass: "auth",+ summary: `command ${command} failed: worker authentication failed`,+ tokens: [],+ };+ }+ if (err.includes("Session worker is starting")) {+ return {+ severity: "warn",+ eventClass: "command-failure",+ summary: `command ${command} failed: session worker is starting`,+ tokens: [],+ };+ }+ if (err.includes("preparing an update")) {+ return {+ severity: "info",+ eventClass: "command-failure",+ summary: `command ${command} failed: update restart in preparation`,+ tokens: [],+ };+ }+ if (err.includes("Session worker is recovering")) {+ return {+ severity: "warn",+ eventClass: "command-failure",+ summary: `command ${command} failed: session worker is recovering`,+ tokens: [],+ };+ }+ const unknownSession = /Unknown active session: (\S+)/.exec(err);+ if (unknownSession) {+ const sessionId = unknownSession[1]!;+ return {+ severity: "warn",+ eventClass: "command-failure",+ summary: `command ${command} failed: unknown active session ${sessionId}`,+ tokens: [sessionId],+ };+ }+ // Quoted strings can carry session names (e.g. Agent name "Faerie").+ const quotedNames = [...err.matchAll(/"([^"\n]+)"/g)].map((match) => match[1]!);+ return {+ severity: "warn",+ eventClass: "command-failure",+ summary: `command ${command} failed: ${truncateText(errorMessage(err), 100)}`,+ tokens: quotedNames,+ };+}++/** Classify the body of a `Session worker <id> stderr: <body>` log line. */+function classifyWorkerStderrBody(+ workerId: string,+ body: string,+ includeGeneric = true,+): Omit<IncidentEvent, "timeMs" | "category"> | undefined {+ const tokens = [workerId];+ if (!body.trim() || /^\s*at\s/.test(body)) {+ // Stack frames and blank lines belong to the previous stderr event.+ return undefined;+ }+ const line = firstLine(body);+ if (/^Prime Agent daemon listening on \S+$/.test(line)) {+ return {+ severity: "info",+ eventClass: "worker-start",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} started`,+ tokens,+ };+ }+ const crash = /^(?:uncaught exception|unhandled rejection): (.+)$/.exec(line);+ if (crash) {+ return {+ severity: "critical",+ eventClass: "worker-crash",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} crashed: ${truncateText(line, 100)}`,+ tokens,+ };+ }+ const shutdown = /^shutting down \(exit (\d+)\); closing (\d+) active session\(s\)$/.exec(line);+ if (shutdown) {+ return {+ severity: "info",+ eventClass: "worker-stop",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} stopped (exit ${shutdown[1]}, ${shutdown[2]} session(s) closed)`,+ tokens,+ };+ }+ const signal = /^received (\S+); shutting down$/.exec(line);+ if (signal) {+ return {+ severity: "info",+ eventClass: "worker-stop",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} received ${signal[1]}; shutting down`,+ tokens,+ };+ }+ const stopRequested = /^shutdown command received over socket; (\d+) active session\(s\) will be closed$/.exec(line);+ if (stopRequested) {+ return {+ severity: "info",+ eventClass: "worker-stop",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} stop requested (${stopRequested[1]} active session(s))`,+ tokens,+ };+ }+ const passivated = /^Passivated idle child sessionId=(\S+) name=("[^"]*"|\S+) idleMinutes=(\d+)$/.exec(line);+ if (passivated) {+ // A missing session name logs as `name=""`; that empty token must not+ // become a filter key.+ const name = passivated[2].replace(/^"|"$/g, "");+ return {+ severity: "info",+ eventClass: "worker-passivation",+ subject: `session ${passivated[1]}`,+ summary: `passivated idle child session ${passivated[1]} (idle ${passivated[3]}m)`,+ tokens: name ? [workerId, passivated[1]!, name] : [workerId, passivated[1]!],+ };+ }+ if (!includeGeneric) {+ return undefined;+ }+ return {+ severity: "warn",+ eventClass: "worker-stderr",+ subject: `worker ${workerId}`,+ summary: `worker ${workerId} stderr: ${truncateText(line, 100)}`,+ tokens,+ };+}++function recoveryBreakdown(operations: string): { count: number; breakdown: string } {+ const list = operations+ .split(",")+ .map((operation) => operation.trim())+ .filter(Boolean);+ if (list.length === 0) {+ return { count: 0, breakdown: "none listed" };+ }+ const counts = new Map<string, number>();+ for (const operation of list) {+ counts.set(operation, (counts.get(operation) ?? 0) + 1);+ }+ const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));+ const top = ranked.slice(0, RECOVERY_BREAKDOWN_LIMIT);+ const shownCount = top.reduce((sum, [, count]) => sum + count, 0);+ const rest = ranked.length > RECOVERY_BREAKDOWN_LIMIT ? `, +${list.length - shownCount} more` : "";+ const breakdown = top.map(([operation, count]) => `${operation} x${count}`).join(", ");+ return { count: list.length, breakdown: `${breakdown}${rest}` };+}++function event(+ entry: IncidentLogEntry,+ severity: IncidentSeverity,+ category: IncidentCategory,+ eventClass: string,+ subject: string,+ summary: string,+ tokens: string[] = [],+): IncidentEvent {+ return { timeMs: entry.timeMs, severity, category, eventClass, subject, summary, tokens };+}++/**+ * Key events on the entry's socket path so anomalies and aggregation never+ * mix entries from different daemons sharing one agent.jsonl; entries without+ * a socket field (the per-daemon fallback log) key on `daemon`.+ */+function daemonSubject(entry: IncidentLogEntry): string {+ return entry.socketPath ?? "daemon";+}++/** Classify a provider-failure entry into an aggregate-friendly anomaly event. */+function providerFailureEvent(entry: IncidentLogEntry, workerPids: WorkerPidMap): IncidentEvent | undefined {+ const workerId = workerIdForPid(workerPids, entry.pid, entry.timeMs);+ const subject = workerId ? `worker ${workerId}` : entry.pid !== undefined ? `pid ${entry.pid}` : "provider";+ if (entry.msg !== "provider stream failure") {+ return event(+ entry,+ "error",+ "anomaly",+ "provider",+ subject,+ truncateText(firstLine(entry.msg), SUMMARY_TRUNCATION),+ [],+ );+ }+ const kind = typeof entry.fields.kind === "string" ? entry.fields.kind : "unknown";+ const statusText = typeof entry.fields.status === "number" ? ` ${entry.fields.status}` : "";+ return {+ timeMs: entry.timeMs,+ severity: "error",+ category: "anomaly",+ eventClass: "provider",+ subject,+ summary: `provider stream failure (${kind}${statusText}) for ${subject}`,+ tokens: workerId ? [workerId] : [],+ };+}++/** Classify one log entry into an incident event; undefined when the entry is noise. */+export function classifyIncidentEntry(entry: IncidentLogEntry, workerPids: WorkerPidMap): IncidentEvent | undefined {+ const msg = firstLine(entry.msg);+ const workerId = workerIdFromSocketPath(entry.socketPath);++ if (entry.component === "ai.provider") {+ return providerFailureEvent(entry, workerPids);+ }++ const stderr = /^Session worker ([0-9a-f]{12}) stderr: ?([\s\S]*)$/.exec(entry.msg);+ if (stderr) {+ const body = classifyWorkerStderrBody(stderr[1]!, stderr[2] ?? "");+ if (!body) {+ return undefined;+ }+ return { ...body, timeMs: entry.timeMs, category: "supervisor" };+ }++ if (/^Prime Agent daemon supervisor \S+ listening on \S+$/.test(msg)) {+ return event(+ entry,+ "info",+ "supervisor",+ "supervisor-start",+ daemonSubject(entry),+ "daemon supervisor listening",+ );+ }+ const startupFailed = /^Daemon supervisor startup failed: (.+)$/.exec(msg);+ if (startupFailed) {+ const err = startupFailed[1]!;+ if (/lock file is already being held/i.test(err)) {+ return event(+ entry,+ "warn",+ "supervisor",+ "supervisor-start",+ daemonSubject(entry),+ "supervisor startup blocked: another daemon holds the lock",+ );+ }+ return event(+ entry,+ "error",+ "supervisor",+ "supervisor-start",+ daemonSubject(entry),+ `supervisor startup failed: ${truncateText(errorMessage(firstLine(err)), 100)}`,+ );+ }+ const supervisorCommand = /^Supervisor command (\S+) failed: (.+)$/.exec(msg);+ if (supervisorCommand) {+ const classified = classifyCommandFailure(supervisorCommand[1]!, supervisorCommand[2]!);+ return event(+ entry,+ classified.severity,+ "supervisor",+ classified.eventClass,+ daemonSubject(entry),+ classified.summary,+ classified.tokens,+ );+ }+ const daemonCommand = /^daemon command "([^"]+)" failed: (.+)$/.exec(msg);+ if (daemonCommand) {+ const classified = classifyCommandFailure(daemonCommand[1]!, daemonCommand[2]!);+ return event(+ entry,+ classified.severity,+ "supervisor",+ classified.eventClass,+ workerId ? `worker ${workerId}` : "worker",+ classified.summary,+ // Keep the session ids/names classified out of the error, not just the worker id.+ workerId ? [workerId, ...classified.tokens] : classified.tokens,+ );+ }+ const catchUp = /^(?:Failed|could not)(?: to)? catch up (?:snapshot )?client \S+(?: for (\S+))?: (.+)$/.exec(msg);+ if (catchUp) {+ const sessionId = catchUp[1];+ const err = catchUp[2]!;+ const timedOut = err.includes("Timed out waiting for daemon worker response");+ return event(+ entry,+ timedOut ? "error" : "warn",+ "supervisor",+ timedOut ? "timeout" : "command-failure",+ sessionId ? `session ${sessionId}` : daemonSubject(entry),+ `client catch-up failed${sessionId ? ` for session ${sessionId}` : ""}: ${truncateText(errorMessage(firstLine(err)), 100)}`,+ sessionId ? [sessionId] : [],+ );+ }+ const heartbeats = /^Could not list heartbeats from a worker: (.+)$/.exec(msg);+ if (heartbeats) {+ const timedOut = heartbeats[1]!.includes("Timed out waiting for daemon worker response");+ return event(+ entry,+ timedOut ? "error" : "warn",+ "supervisor",+ timedOut ? "timeout" : "command-failure",+ daemonSubject(entry),+ `worker heartbeat list failed: ${truncateText(errorMessage(firstLine(heartbeats[1]!)), 100)}`,+ );+ }++ const recovered = /^Recovered worker (\S+) without replaying uncertain operations: (.+)$/.exec(msg);+ if (recovered) {+ const worker = recovered[1]!;+ const { count, breakdown } = recoveryBreakdown(recovered[2]!);+ return event(+ entry,+ "warn",+ "recovery",+ "recovery-replay",+ `worker ${worker}`,+ `worker ${worker} recovered; ${count} uncertain operation${count === 1 ? "" : "s"} not replayed (${breakdown})`,+ [worker],+ );+ }+ const recoveredPlain = /^Recovered worker (\S+)$/.exec(msg);+ if (recoveredPlain) {+ const worker = recoveredPlain[1]!;+ return event(entry, "info", "recovery", "recovery-replay", `worker ${worker}`, `worker ${worker} recovered`, [+ worker,+ ]);+ }+ const adoptFailed = /^Could not adopt worker (\S+): (.+)$/.exec(msg);+ if (adoptFailed) {+ const worker = adoptFailed[1]!;+ return event(+ entry,+ "warn",+ "recovery",+ "recovery-failure",+ `worker ${worker}`,+ `could not adopt worker ${worker}: ${truncateText(errorMessage(firstLine(adoptFailed[2]!)), 100)}`,+ [worker],+ );+ }+ const recoverFailed = /^Could not recover worker (\S+): (.+)$/.exec(msg);+ if (recoverFailed) {+ const worker = recoverFailed[1]!;+ return event(+ entry,+ "error",+ "recovery",+ "recovery-failure",+ `worker ${worker}`,+ `could not recover worker ${worker}: ${truncateText(errorMessage(firstLine(recoverFailed[2]!)), 100)}`,+ [worker],+ );+ }+ const failedAfterRetries = /^Worker (\S+) failed after three recovery attempts$/.exec(msg);+ if (failedAfterRetries) {+ const worker = failedAfterRetries[1]!;+ return event(+ entry,+ "error",+ "recovery",+ "recovery-failure",+ `worker ${worker}`,+ `worker ${worker} failed after three recovery attempts`,+ [worker],+ );+ }+ const unresponsive = /^Worker (\S+) is unresponsive; parked failed after \d+ probe rounds$/.exec(msg);+ if (unresponsive) {+ const worker = unresponsive[1]!;+ return event(entry, "error", "recovery", "recovery-failure", `worker ${worker}`, msg, [worker]);+ }+ const reclaimed = /^Reclaimed stale registration for stopped worker (\S+)$/.exec(msg);+ if (reclaimed) {+ const worker = reclaimed[1]!;+ return event(+ entry,+ "info",+ "recovery",+ "recovery-action",+ `worker ${worker}`,+ `reclaimed stale registration for stopped worker ${worker}`,+ [worker],+ );+ }+ const migrated = /^Migrated (\d+) scheduled jobs into session artifacts$/.exec(msg);+ if (migrated) {+ return event(+ entry,+ "info",+ "recovery",+ "recovery-action",+ daemonSubject(entry),+ `migrated ${migrated[1]} scheduled jobs into session artifacts`,+ );+ }+ const replacement = /^launched replacement supervisor on \S+$/.exec(msg);+ if (replacement) {+ return event(entry, "info", "recovery", "recovery-action", daemonSubject(entry), msg);+ }+ const woke = /^Woke session worker for a due scheduled job: \S+$/.exec(msg);+ if (woke) {+ // A scheduled wake is a supervisor lifecycle action, not crash recovery.+ return event(entry, "info", "supervisor", "supervisor-action", daemonSubject(entry), msg);+ }++ const evictedIdle = /^Evicted idle worker (\S+) root=\S* idleMinutes=(\d+) sessions=(\d+)$/.exec(msg);+ if (evictedIdle) {+ const worker = evictedIdle[1]!;+ return event(+ entry,+ "info",+ "supervisor",+ "worker-stop",+ `worker ${worker}`,+ `evicted idle worker ${worker} (idle ${evictedIdle[2]}m, ${evictedIdle[3]} session(s))`,+ [worker],+ );+ }+ const evictedEmpty = /^Evicted empty session worker (\S+) root=\S+ on last client detach$/.exec(msg);+ if (evictedEmpty) {+ const worker = evictedEmpty[1]!;+ return event(+ entry,+ "info",+ "supervisor",+ "worker-stop",+ `worker ${worker}`,+ `evicted empty session worker ${worker} on last detach`,+ [worker],+ );+ }++ if (entry.component === "coding-agent.daemon") {+ // The worker logs its own lifecycle lines; reusing the stderr-body+ // classifier gives them the same summaries as the supervisor's stderr+ // forward, so the duplicated log pair collapses in the timeline.+ if (workerId !== undefined) {+ const body = classifyWorkerStderrBody(workerId, msg, false);+ if (body) {+ return { ...body, timeMs: entry.timeMs, category: "supervisor" };+ }+ } else if (/^(?:uncaught exception|unhandled rejection): /.test(msg)) {+ return event(+ entry,+ "critical",+ "supervisor",+ "worker-crash",+ "worker",+ `worker crashed: ${truncateText(msg, 100)}`,+ );+ }+ }++ if (entry.component === "coding-agent.daemon-supervisor" || entry.component === "coding-agent.daemon") {+ // Unknown diagnostics still matter during an incident; summarize them.+ return event(+ entry,+ entry.level === "error" ? "error" : "warn",+ "supervisor",+ "diagnostic",+ workerId ? `worker ${workerId}` : daemonSubject(entry),+ truncateText(msg, SUMMARY_TRUNCATION),+ );+ }+ return undefined;+}++/**+ * Classify every entry, dropping the duplicate event the daemon writes both+ * to the structured log and to the worker stderr forward (same summary, same+ * second).+ */+export function collectIncidentEvents(entries: readonly IncidentLogEntry[], workerPids: WorkerPidMap): IncidentEvent[] {+ const events: IncidentEvent[] = [];+ const lastSeen = new Map<string, { timeMs: number; component: string }>();+ for (const entry of entries) {+ const incident = classifyIncidentEntry(entry, workerPids);+ if (!incident) {+ continue;+ }+ // The daemon writes the same lifecycle event both to the structured log+ // (coding-agent.daemon) and to the worker stderr forward+ // (coding-agent.daemon-supervisor); drop the second copy when it lands+ // within 2s. A repeat from the SAME component is a real lifecycle+ // transition — a worker restarted on its durable id within 2s — and is+ // never a duplicate.+ if (LIFECYCLE_CLASSES.has(incident.eventClass)) {+ const key = `${incident.category}|${incident.summary}`;+ const previous = lastSeen.get(key);+ if (+ previous !== undefined &&+ previous.component !== entry.component &&+ Math.abs(incident.timeMs - previous.timeMs) <= 2000+ ) {+ continue;+ }+ lastSeen.set(key, { timeMs: incident.timeMs, component: entry.component });+ }+ events.push(incident);+ }+ return events;+}++// ---------------------------------------------------------------------------+// Anomaly computation+// ---------------------------------------------------------------------------++function formatIncidentDuration(ms: number): string {+ const totalSeconds = Math.max(1, Math.round(ms / 1000));+ const seconds = totalSeconds % 60;+ const minutes = Math.floor(totalSeconds / 60) % 60;+ const hours = Math.floor(totalSeconds / 3600);+ if (hours > 0) {+ return `${hours}h${minutes > 0 ? `${minutes}m` : ""}`;+ }+ if (minutes > 0) {+ return `${minutes}m${seconds > 0 ? `${seconds}s` : ""}`;+ }+ return `${seconds}s`;+}++function maxSeverity(events: readonly IncidentEvent[]): IncidentSeverity {+ let severity: IncidentSeverity = "info";+ for (const item of events) {+ if (item.severity === "critical") return "critical";+ if (item.severity === "error") severity = "error";+ else if (item.severity === "warn" && severity !== "error") severity = "warn";+ }+ return severity;+}++/**+ * The densest run of time-sorted events within windowMs of each other,+ * returned as the run's start index and length. Shared by the stall and burst+ * scans: only events close together form one incident, so isolated events far+ * apart never merge, whatever the report window is.+ */+function densestWindowRun(items: readonly IncidentEvent[], windowMs: number): { start: number; count: number } {+ let bestStart = 0;+ let bestCount = 0;+ let start = 0;+ for (let end = 0; end < items.length; end++) {+ while (items[end]!.timeMs - items[start]!.timeMs > windowMs) {+ start++;+ }+ const count = end - start + 1;+ if (count > bestCount) {+ bestCount = count;+ bestStart = start;+ }+ }+ return { start: bestStart, count: bestCount };+}++/** Compute stall, error-burst, and event-gap anomaly lines from classified events. */+export function computeIncidentAnomalies(events: readonly IncidentEvent[]): IncidentEvent[] {+ const bySubject = new Map<string, IncidentEvent[]>();+ for (const incident of events) {+ if (incident.category === "anomaly") {+ continue;+ }+ const group = bySubject.get(incident.subject) ?? [];+ group.push(incident);+ bySubject.set(incident.subject, group);+ }++ const anomalies: IncidentEvent[] = [];+ const push = (timeMs: number, severity: IncidentSeverity, subject: string, summary: string) => {+ anomalies.push({ timeMs, severity, category: "anomaly", eventClass: "anomaly", subject, summary, tokens: [] });+ };++ for (const [subject, group] of bySubject) {+ group.sort((a, b) => a.timeMs - b.timeMs);+ const timeouts = group.filter((incident) => incident.eventClass === "timeout");+ if (timeouts.length >= 2) {+ // Only timeouts within TIMEOUT_STALL_WINDOW_MS of each other form a+ // stall; two isolated timeouts hours apart in a long window are not one.+ const run = densestWindowRun(timeouts, TIMEOUT_STALL_WINDOW_MS);+ if (run.count >= 2) {+ const cluster = timeouts.slice(run.start, run.start + run.count);+ const span = cluster[cluster.length - 1]!.timeMs - cluster[0]!.timeMs;+ push(+ cluster[0]!.timeMs,+ "error",+ subject,+ `${subject}: ${cluster.length} command timeouts over ${formatIncidentDuration(span)}`,+ );+ }+ }+ const burst = group.filter((incident) => BURST_CLASSES.has(incident.eventClass));+ if (burst.length >= ERROR_BURST_THRESHOLD) {+ // Only events within ERROR_BURST_WINDOW_MS of each other form a burst;+ // three isolated warnings days apart in a long window are not one.+ const run = densestWindowRun(burst, ERROR_BURST_WINDOW_MS);+ if (run.count >= ERROR_BURST_THRESHOLD) {+ const cluster = burst.slice(run.start, run.start + run.count);+ const span = cluster[cluster.length - 1]!.timeMs - cluster[0]!.timeMs;+ push(+ cluster[0]!.timeMs,+ maxSeverity(cluster),+ subject,+ `${subject}: ${cluster.length} warnings/errors over ${formatIncidentDuration(span)}`,+ );+ }+ }+ if (subject.startsWith("session ")) {+ for (let index = 1; index < group.length; index++) {+ const gap = group[index]!.timeMs - group[index - 1]!.timeMs;+ if (gap >= STALL_GAP_MS) {+ push(+ group[index - 1]!.timeMs,+ "warn",+ subject,+ `${subject}: ${formatIncidentDuration(gap)} event gap (no logged events)`,+ );+ }+ }+ }+ }+ return anomalies.sort((a, b) => a.timeMs - b.timeMs);+}++/**+ * Per subject, the latest timeout of the stall cluster computeIncidentAnomalies+ * reports — the densest run within TIMEOUT_STALL_WINDOW_MS. The agents-view+ * notice anchors its dismissal horizon there, so the notice, its horizon, and+ * the reported cluster always describe the same incident, even when the window+ * holds several stalls on one subject.+ */+export function latestIncidentStallTimeoutBySubject(events: readonly IncidentEvent[]): Map<string, number> {+ const timeoutsBySubject = new Map<string, IncidentEvent[]>();+ for (const incident of events) {+ if (incident.eventClass !== "timeout") {+ continue;+ }+ const group = timeoutsBySubject.get(incident.subject) ?? [];+ group.push(incident);+ timeoutsBySubject.set(incident.subject, group);+ }+ const latestBySubject = new Map<string, number>();+ for (const [subject, timeouts] of timeoutsBySubject) {+ // Callers pass time-sorted events; sort defensively so the run scan+ // (which assumes ascending times) never sees them out of order.+ timeouts.sort((a, b) => a.timeMs - b.timeMs);+ const run = densestWindowRun(timeouts, TIMEOUT_STALL_WINDOW_MS);+ if (run.count >= 2) {+ latestBySubject.set(subject, timeouts[run.start + run.count - 1]!.timeMs);+ }+ }+ return latestBySubject;+}++// ---------------------------------------------------------------------------+// Timeline rendering+// ---------------------------------------------------------------------------++function formatIncidentTime(ms: number): string {+ const date = new Date(ms);+ const pad = (value: number) => String(value).padStart(2, "0");+ return `${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())} ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}`;+}++function colorizeSeverity(severity: IncidentSeverity, label: string): string {+ switch (severity) {+ case "critical":+ case "error":+ return chalk.red(label);+ case "warn":+ return chalk.yellow(label);+ case "info":+ return chalk.dim(label);+ }+}++interface AggregatedEvent {+ first: IncidentEvent;+ last: IncidentEvent;+ count: number;+}++function aggregateIncidentEvents(events: readonly IncidentEvent[]): AggregatedEvent[] {+ const groups = new Map<string, AggregatedEvent>();+ for (const incident of events) {+ const key = `${incident.category}|${incident.subject}|${incident.summary}`;+ const existing = groups.get(key);+ if (existing) {+ existing.count += 1;+ if (incident.timeMs > existing.last.timeMs) {+ existing.last = incident;+ }+ if (incident.timeMs < existing.first.timeMs) {+ existing.first = incident;+ }+ } else {+ groups.set(key, { first: incident, last: incident, count: 1 });+ }+ }+ return [...groups.values()].sort((a, b) => a.first.timeMs - b.first.timeMs);+}++function sessionMatches(incident: IncidentEvent, session: string): boolean {+ // An empty token (e.g. a missing session name) is a prefix of every value+ // and must not match every session filter.+ return incident.tokens.some((token) => token !== "" && (token.startsWith(session) || session.startsWith(token)));+}++/**+ * Build the full incident timeline text for a window: classified events are+ * grouped into Supervisor events / Session anomalies / Recovery sections,+ * repeated identical events are aggregated with counts, and per-subject+ * stalls, error bursts, and event gaps are surfaced as anomalies.+ */+export function buildIncidentReport(+ entries: readonly IncidentLogEntry[],+ options: IncidentReportOptions,+): IncidentReport {+ const workerPids = collectWorkerPidMap(entries);+ const allEvents = collectIncidentEvents(entries, workerPids);+ let events = allEvents.filter(+ (incident) => incident.timeMs >= options.sinceMs && incident.timeMs <= options.untilMs,+ );+ let filteredBySession = false;+ if (options.session !== undefined) {+ const session = options.session;+ const matching = events.filter((incident) => sessionMatches(incident, session));+ if (matching.length === 0 && events.length > 0) {+ return {+ text: `No daemon events between ${formatIncidentTime(options.sinceMs)} and ${formatIncidentTime(options.untilMs)} UTC reference session "${options.session}".`,+ };+ }+ events = matching;+ filteredBySession = true;+ }+ const anomalies = computeIncidentAnomalies(events.filter((incident) => incident.category !== "anomaly"));+ const timelineEvents = [...events, ...anomalies];++ const sections: Array<{ title: string; lines: string[] }> = [];+ for (const [title, category] of [+ ["Supervisor events", "supervisor"],+ ["Session anomalies", "anomaly"],+ ["Recovery", "recovery"],+ ] as const) {+ const aggregated = aggregateIncidentEvents(timelineEvents.filter((incident) => incident.category === category));+ const lines = aggregated.map((group) => {+ const suffix = group.count > 1 ? ` (x${group.count}, until ${formatIncidentTime(group.last.timeMs)})` : "";+ return ` ${formatIncidentTime(group.first.timeMs)} ${colorizeSeverity(group.first.severity, group.first.severity.padEnd(8))} ${group.first.summary}${suffix}`;+ });+ if (lines.length === 0) {+ lines.push(` ${chalk.dim("(none)")}`);+ }+ sections.push({ title, lines });+ }++ const header = ["Prime Agent incident timeline"];+ const windowText = `Window: ${formatIncidentTime(options.sinceMs)} → ${formatIncidentTime(options.untilMs)} UTC (${formatIncidentDuration(options.untilMs - options.sinceMs)})`;+ header.push(windowText);+ if (options.source !== undefined) {+ const inWindow = allEvents.filter(+ (incident) => incident.timeMs >= options.sinceMs && incident.timeMs <= options.untilMs,+ ).length;+ const skipped = options.skippedCount ?? 0;+ const skippedText = skipped > 0 ? `, ${skipped} unreadable skipped` : "";+ header.push(+ `Source: ${options.source} (${options.scannedCount ?? entries.length} lines scanned, ${inWindow} events in window${skippedText})`,+ );+ }+ if (filteredBySession) {+ header.push(`Session filter: ${options.session}`);+ }++ const text = [...header, "", ...sections.flatMap((section) => [section.title, ...section.lines, ""])]+ .join("\n")+ .trimEnd();+ return { text };+}++// ---------------------------------------------------------------------------+// CLI+// ---------------------------------------------------------------------------++const INCIDENT_VALUE_FLAGS = new Set(["--since", "--until", "--session"]);++/** Parse `incident [--since <time>] [--until <time>] [--session <id>]` arguments. */+export function parseIncidentOptions(args: string[]): IncidentCommandOptions {+ const options: IncidentCommandOptions = {};+ for (let index = 0; index < args.length; index++) {+ const arg = args[index]!;+ let name = arg;+ let value: string | undefined;+ const equalsIndex = arg.indexOf("=");+ if (arg.startsWith("--") && equalsIndex !== -1) {+ name = arg.slice(0, equalsIndex);+ value = arg.slice(equalsIndex + 1);+ }+ if (!INCIDENT_VALUE_FLAGS.has(name)) {+ throw new IncidentUsageError(`Unknown option for incident: ${arg}`);+ }+ if (value === undefined) {+ value = args[++index];+ if (value === undefined) {+ throw new IncidentUsageError(`Option ${name} requires a value.`);+ }+ }+ if (!value.trim()) {+ throw new IncidentUsageError(`Option ${name} requires a value.`);+ }+ if (name === "--since") {+ options.since = value;+ } else if (name === "--until") {+ options.until = value;+ } else {+ options.session = value;+ }+ }+ return options;+}++/**+ * Read the daemon logs: agent.jsonl (plus its .old rotation) when it yields+ * entries, otherwise the newest per-daemon log file in the logs directory.+ * The fallback covers a missing, empty, or unreadable agent.jsonl and picks+ * only the newest per-daemon log; rotated per-daemon .old generations are not+ * included.+ */+export function readIncidentLogEntries(): IncidentLogSource {+ const logsDir = getLogsDir();+ const agentLogPath = getAgentLogPath();+ const structuredFiles = [`${agentLogPath}.old`, agentLogPath]+ .filter((candidate) => existsSync(candidate))+ .map((path) => ({ path, kind: "jsonl" as const }));+ const structured = scanIncidentLogFiles(structuredFiles);+ if (structured.entries.length > 0) {+ return structured;+ }+ // agent.jsonl is missing, empty, or unreadable: fall back to the newest+ // per-daemon log. Window filtering happens later; gate on parse yield only.+ const fallbackPath = newestDaemonLogPath(logsDir);+ if (fallbackPath === undefined) {+ return structured;+ }+ const fallback = scanIncidentLogFiles([{ path: fallbackPath, kind: "daemon" }]);+ return fallback.entries.length > 0 ? fallback : structured;+}++function scanIncidentLogFiles(files: ReadonlyArray<{ path: string; kind: "jsonl" | "daemon" }>): IncidentLogSource {+ const entries: IncidentLogEntry[] = [];+ let scannedCount = 0;+ let skippedCount = 0;+ for (const file of files) {+ let contents: string;+ try {+ contents = readFileSync(file.path, "utf8");+ } catch {+ continue;+ }+ for (const line of contents.split("\n")) {+ if (!line.trim()) {+ continue;+ }+ scannedCount += 1;+ const entry = file.kind === "jsonl" ? parseIncidentLogLine(line) : parseIncidentDaemonLogLine(line);+ if (entry) {+ entries.push(entry);+ } else {+ skippedCount += 1;+ }+ }+ }+ return { entries, scannedCount, skippedCount, source: files.map((file) => file.path).join(", ") };+}++function newestDaemonLogPath(logsDir: string): string | undefined {+ let names: string[] = [];+ try {+ names = readdirSync(logsDir);+ } catch {+ return undefined;+ }+ const candidates: Array<{ path: string; mtimeMs: number }> = [];+ for (const name of names) {+ if (!DAEMON_LOG_FILE_PATTERN.test(name)) {+ continue;+ }+ const path = join(logsDir, name);+ try {+ const stat = statSync(path);+ // A directory or other non-file matching the log pattern would be+ // picked as the fallback and then yield no entries (readFileSync+ // fails on it), hiding older valid daemon logs.+ if (!stat.isFile()) {+ continue;+ }+ candidates.push({ path, mtimeMs: stat.mtimeMs });+ } catch {+ // Lost a race with log rotation or permissions; skip the candidate.+ }+ }+ if (candidates.length === 0) {+ return undefined;+ }+ return candidates.sort((a, b) => b.mtimeMs - a.mtimeMs)[0]!.path;+}++/** The resolved --since/--until window for an incident report. */+export interface IncidentWindow {+ sinceMs: number;+ untilMs: number;+}++/** Resolve --since/--until (default: last 24h until now); throws on bad or unordered times. */+export function resolveIncidentWindow(options: IncidentCommandOptions, now: Date): IncidentWindow {+ const sinceMs = options.since+ ? parseIncidentTimeBound(options.since, now, "--since")+ : now.getTime() - DEFAULT_WINDOW_MS;+ const untilMs = options.until ? parseIncidentTimeBound(options.until, now, "--until") : now.getTime();+ if (untilMs <= sinceMs) {+ throw new IncidentUsageError("--until must be after --since.");+ }+ return { sinceMs, untilMs };+}++/**+ * Entry point for `prime-agent incident`; prints the timeline to stdout.+ * Callers that validate the window pass it back so relative `HH:MM` bounds+ * resolve exactly once instead of again against a later clock reading.+ */+export async function runIncident(options: IncidentCommandOptions, window?: IncidentWindow): Promise<void> {+ const { sinceMs, untilMs } = window ?? resolveIncidentWindow(options, new Date());+ const logSource = readIncidentLogEntries();+ if (logSource.entries.length === 0 && logSource.scannedCount === 0) {+ console.log(`No daemon logs found under ${getLogsDir()}.`);+ return;+ }+ const report = buildIncidentReport(logSource.entries, {+ sinceMs,+ untilMs,+ ...(options.session !== undefined ? { session: options.session } : {}),+ source: logSource.source,+ scannedCount: logSource.scannedCount,+ skippedCount: logSource.skippedCount,+ });+ console.log(report.text);+}diff --git a/packages/coding-agent/src/cli/public-command.ts b/packages/coding-agent/src/cli/public-command.tsindex 5fa05c40cd..2d946c9223 100644--- a/packages/coding-agent/src/cli/public-command.ts+++ b/packages/coding-agent/src/cli/public-command.ts@@ -19,6 +19,13 @@ import { handleDaemonCommand } from "./daemon-command.js";import { runPs, runReap, runShutdownAll } from "./daemon-ps.js";import { DAEMON_UPDATE_RESTART_COORDINATOR_FLAG } from "./daemon-update-restart.js";import { extractHelpCommandPath, rotateGlobalFlagsBeforeCommand } from "./global-flags.js";+import {+ type IncidentCommandOptions,+ type IncidentWindow,+ parseIncidentOptions,+ resolveIncidentWindow,+ runIncident,+} from "./incident.js";export interface PublicCommandResult {handled: boolean;@@ -113,6 +120,8 @@ async function runPublicCommand(args: string[]): Promise<PublicCommandResult> {return runStatus(args.slice(1));case "doctor":return runDoctor(args.slice(1));+ case "incident":+ return runIncidentCommand(args.slice(1));case "shutdown":return runShutdown(args.slice(1));case "package":@@ -259,6 +268,21 @@ async function runDoctor(args: string[]): Promise<PublicCommandResult> {return HANDLED;}+async function runIncidentCommand(args: string[]): Promise<PublicCommandResult> {+ let options: IncidentCommandOptions;+ let window: IncidentWindow;+ try {+ options = parseIncidentOptions(args);+ // Resolve once: re-resolving later can cross UTC midnight and render a+ // different window than the one that was validated.+ window = resolveIncidentWindow(options, new Date());+ } catch (error) {+ return fail(error instanceof Error ? error.message : String(error), `Run "${APP_NAME} help incident" for usage.`);+ }+ await runIncident(options, window);+ return HANDLED;+}+async function runShutdown(args: string[]): Promise<PublicCommandResult> {const options = parseBooleanOptions(args, new Set(["--force", "--json"]), "shutdown");if (!options) return HANDLED;diff --git a/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts b/packages/coding-agent/src/modes/agents-view/agents-view-mode.tsindex fb03fc7934..8f036672e1 100644--- a/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts+++ b/packages/coding-agent/src/modes/agents-view/agents-view-mode.ts@@ -13,7 +13,14 @@ import {visibleWidth,wrapTextWithAnsi,} from "@earendil-works/pi-tui";-import { APP_TITLE, appendRotatingLog, getAgentDir, getClientErrorLogPath, VERSION } from "../../config.js";+import {+ APP_TITLE,+ appendRotatingLog,+ getAgentDir,+ getAgentLogPath,+ getClientErrorLogPath,+ VERSION,+} from "../../config.js";import type { AgentSessionRuntimeConfig } from "../../core/agent-session-config.js";import { KeybindingsManager } from "../../core/keybindings.js";import { SessionManager } from "../../core/session-manager.js";@@ -102,6 +109,14 @@ import {type UnifiedSessionIndex,type UnifiedSessionRecord,} from "./agents-view-state.js";+import {+ createIncidentNoticeState,+ dismissIncidentNoticeState,+ formatIncidentNoticeLine,+ INCIDENT_NOTICE_POLL_INTERVAL_MS,+ type IncidentNoticeState,+ refreshIncidentNoticeState,+} from "./incident-notices.js";import { AgentsViewRosterStore, STALE_ROSTER_DAEMON_MESSAGE } from "./roster-store.js";import { matchesSearchText } from "./session-view-search.js";@@ -169,6 +184,10 @@ export type AgentsViewPersistentState = {// re-entry and render the moment they resolve, even if the first view was left early.startupNotices?: StartupNotices;startupNoticesPromise?: Promise<StartupNotices>;+ // Incident notice state (windowed log entries, log offset, dismissal horizons)+ // is reused the same way: a dismissed incident must stay dismissed across+ // re-entry, and the log poll continues from the consumed offset.+ incidentNoticeState?: IncidentNoticeState;query?: string;rosterClient?: DaemonClient;rosterStore?: AgentsViewRosterStore;@@ -839,6 +858,7 @@ export class AgentsViewMode implements Component, Focusable {private resolveRun: ((result: AgentsViewRunResult) => void) | undefined;private heartbeatPollTimer: NodeJS.Timeout | undefined;private animationTimer: NodeJS.Timeout | undefined;+ private incidentNoticeTimer: NodeJS.Timeout | undefined;private ctrlCExitHintExpiresAt = 0;private ctrlCExitHintTimer: ReturnType<typeof setTimeout> | undefined;private deleteConfirmExpiresAt = 0;@@ -1077,8 +1097,11 @@ export class AgentsViewMode implements Component, Focusable {this.resolveMissingSelectionAnchor();void this.refreshHeartbeats();this.loadStartupNotices();+ this.refreshIncidentNotices();this.heartbeatPollTimer = setInterval(() => void this.refreshHeartbeats(), HEARTBEAT_POLL_INTERVAL_MS);this.heartbeatPollTimer.unref?.();+ this.incidentNoticeTimer = setInterval(() => this.refreshIncidentNotices(), INCIDENT_NOTICE_POLL_INTERVAL_MS);+ this.incidentNoticeTimer.unref?.();this.animationTimer = setInterval(() => {const hasRunning = this.rows.some((row) => row.section === "running");const hasStaleAge = this.rows.some((row) => row.summary.lastHeardFromAt !== undefined);@@ -1110,6 +1133,23 @@ export class AgentsViewMode implements Component, Focusable {this.handleCtrlC();return;}+ // Esc (tui.select.cancel) dismisses the incident notice while it is the+ // only thing to cancel: no armed reply, no autocomplete popup, an empty+ // search prompt. An armed delete confirmation is the more dangerous state:+ // Esc cancels it (below) and keeps the notice instead of dismissing the+ // notice and leaving the delete armed to fire on the next press without+ // a fresh confirmation. Without a visible notice, Esc keeps its back/exit+ // meaning.+ if (+ this.editor.getText().length === 0 &&+ !this.replyTarget &&+ !this.editor.isShowingAutocomplete() &&+ !this.isDeleteConfirmationVisible() &&+ this.keybindings.matches(data, "tui.select.cancel") &&+ this.dismissIncidentNotice()+ ) {+ return;+ }if (this.editor.getText().length === 0 && this.keybindings.matches(data, "app.agents.rename")) {this.enterRenameMode();return;@@ -1186,7 +1226,7 @@ export class AgentsViewMode implements Component, Focusable {return [];}const headerLines = this.splash.render(width);- const noticeLines = this.renderStartupNotices(width);+ const noticeLines = [...this.renderIncidentNotice(width), ...this.renderStartupNotices(width)];if (noticeLines.length > 0) {headerLines.push("", ...noticeLines);}@@ -1258,6 +1298,44 @@ export class AgentsViewMode implements Component, Focusable {return formatted.flatMap((line) => wrapTextWithAnsi(line, wrapWidth).map((wrapped) => ` ${wrapped}`));}+ private incidentNoticeState(): IncidentNoticeState {+ // Reused across agents-view instances like the startup notices: the+ // windowed entries, the consumed log offset, and the dismissal horizons+ // survive leaving and re-entering the view, so a dismissed incident never+ // comes back and the poll does not re-read consumed bytes.+ this.persistentState.incidentNoticeState ??= createIncidentNoticeState();+ return this.persistentState.incidentNoticeState;+ }++ private refreshIncidentNotices(): void {+ // Best-effort, like the startup notices: a missing or unreadable log+ // simply retries a bounded tail on the next poll and never breaks the view.+ const changed = refreshIncidentNoticeState(this.incidentNoticeState(), getAgentLogPath(), Date.now());+ if (changed) {+ this.ui.requestRender();+ }+ }++ /** Dismiss the collapsed incident notice; false when none is showing. */+ private dismissIncidentNotice(): boolean {+ if (!dismissIncidentNoticeState(this.incidentNoticeState())) {+ return false;+ }+ this.setStatusMessage("Incident notice dismissed");+ return true;+ }++ private renderIncidentNotice(width: number): string[] {+ const notice = this.persistentState.incidentNoticeState?.notice;+ if (!notice) {+ return [];+ }+ // Same treatment as the startup notices: one-column gutter, wrap instead+ // of truncating, so the pointer to the incident CLI stays readable.+ const wrapWidth = Math.max(1, width - 1);+ return wrapTextWithAnsi(formatIncidentNoticeLine(notice), wrapWidth).map((wrapped) => ` ${wrapped}`);+ }+private handleListNavigation(data: string): boolean {if (this.keybindings.matches(data, "tui.select.up")) {this.moveSelection(-1);@@ -2566,6 +2644,10 @@ export class AgentsViewMode implements Component, Focusable {clearInterval(this.heartbeatPollTimer);this.heartbeatPollTimer = undefined;}+ if (this.incidentNoticeTimer) {+ clearInterval(this.incidentNoticeTimer);+ this.incidentNoticeTimer = undefined;+ }if (this.animationTimer) {clearInterval(this.animationTimer);this.animationTimer = undefined;diff --git a/packages/coding-agent/src/modes/agents-view/incident-notices.ts b/packages/coding-agent/src/modes/agents-view/incident-notices.tsnew file mode 100644index 0000000000..286a831280--- /dev/null+++ b/packages/coding-agent/src/modes/agents-view/incident-notices.ts@@ -0,0 +1,466 @@+import { Buffer } from "node:buffer";+import { closeSync, fstatSync, openSync, readSync } from "node:fs";+import {+ collectIncidentEvents,+ collectWorkerPidMap,+ computeIncidentAnomalies,+ type IncidentLogEntry,+ type IncidentSeverity,+ latestIncidentStallTimeoutBySubject,+ parseIncidentLogLine,+} from "../../cli/incident.js";+import { theme } from "../interactive/theme/theme.js";++/**+ * Daemon incident notices for the agents view.+ *+ * `prime-agent incident` (the classifier in src/cli/incident.ts) already+ * reconstructs daemon incidents from the shared structured log+ * (~/.prime/agent/logs/agent.jsonl); this module reuses that classifier —+ * never re-implementing it — to surface a single collapsed, dismissible+ * notice line in the agents-view header. A notice appears when the recent+ * window of the log contains a worker crash, a command-timeout burst, or an+ * update restart (a supervisor replacement), so the operator sees the+ * incident without running the CLI by hand. On the initial read the view+ * matches the CLI's [agent.jsonl.old, agent.jsonl] source with the same+ * bounded tail, so incidents spanning a log rotation surface too.+ */++/** Recent-log window, matching the `prime-agent incident` default. */+export const INCIDENT_NOTICE_WINDOW_MS = 24 * 60 * 60 * 1000;+/** Initial tail bound: incidents older than the tail bytes are simply not seen. */+export const INCIDENT_NOTICE_TAIL_BYTES = 512 * 1024;+/**+ * Retention cap for windowed entries: every entry keeps the full parsed log+ * record (fields), and agent.jsonl is the sink for ALL structured logging with+ * 20MB rotating generations, so a busy day would otherwise retain unbounded+ * memory and make every poll re-sort and re-classify it all synchronously.+ * Newest entries win; incidents older than the cap are simply not seen — the+ * same best-effort spirit as INCIDENT_NOTICE_TAIL_BYTES.+ */+export const INCIDENT_NOTICE_MAX_WINDOW_ENTRIES = 20_000;+/** How often the agents view re-reads appended agent.jsonl bytes. */+export const INCIDENT_NOTICE_POLL_INTERVAL_MS = 30_000;++const INCIDENT_NOTICE_POINTER = "— run prime-agent incident for the timeline";+const NEWLINE_BYTE = 0x0a;+const SEVERITY_RANK: Record<IncidentSeverity, number> = { critical: 3, error: 2, warn: 1, info: 0 };++export type IncidentNoticeKind = "worker-crash" | "timeout-burst" | "update-restart";++export interface IncidentNotice {+ kind: IncidentNoticeKind;+ /** Dismissal key (`${kind}|${subject}`): incidents at or before the horizon stay hidden. */+ key: string;+ severity: IncidentSeverity;+ subject: string;+ timeMs: number;+ /** Sentence without the pointer suffix; formatIncidentNoticeLine renders the visible line. */+ text: string;+}++/** Per-run incident notice state, cached on the agents view's persistentState. */+export interface IncidentNoticeState {+ /** Windowed log entries parsed so far, oldest first. */+ entries: IncidentLogEntry[];+ /** Byte offset consumed in agent.jsonl; undefined before the first tail read. */+ logOffset: number | undefined;+ /** `dev:ino` of agent.jsonl at the last read; a change means rotation or replacement. */+ logFileId: string | undefined;+ /** Dismissal horizons by notice key: incidents at or before this timeMs stay hidden. */+ dismissedHorizons: Record<string, number>;+ /** The collapsed notice currently worth showing, if any. */+ notice: IncidentNotice | undefined;+}++export function createIncidentNoticeState(): IncidentNoticeState {+ return {+ entries: [],+ logOffset: undefined,+ logFileId: undefined,+ dismissedHorizons: {},+ notice: undefined,+ };+}++/**+ * Local time of an incident, for "worker x crashed at 14:32". A bare HH:MM only+ * reads as today, so an incident from another local calendar day gets the date+ * prefixed (the tree-selector.ts label-timestamp style): "9/14 23:10", or+ * "26/9/14 23:10" across a year boundary.+ */+export function formatIncidentNoticeTime(timeMs: number, nowMs: number): string {+ const date = new Date(timeMs);+ const pad = (value: number) => String(value).padStart(2, "0");+ const time = `${pad(date.getHours())}:${pad(date.getMinutes())}`;+ const now = new Date(nowMs);+ if (+ date.getFullYear() === now.getFullYear() &&+ date.getMonth() === now.getMonth() &&+ date.getDate() === now.getDate()+ ) {+ return time;+ }+ const month = date.getMonth() + 1;+ const day = date.getDate();+ if (date.getFullYear() === now.getFullYear()) {+ return `${month}/${day} ${time}`;+ }+ return `${date.getFullYear().toString().slice(-2)}/${month}/${day} ${time}`;+}++/** The styled one-line notice rendered in the agents-view header. */+export function formatIncidentNoticeLine(notice: IncidentNotice): string {+ return theme.fg("warning", `⚠ ${notice.text} ${INCIDENT_NOTICE_POINTER}`);+}++function createNotice(+ kind: IncidentNoticeKind,+ severity: IncidentSeverity,+ subject: string,+ timeMs: number,+ text: string,+): IncidentNotice {+ return { kind, key: `${kind}|${subject}`, severity, subject, timeMs, text };+}++/**+ * Derive the notices worth surfacing from windowed agent.jsonl entries, reusing+ * the incident CLI's classifier. Exactly three incident classes qualify:+ * worker crashes (any worker-crash event), command-timeout bursts (the+ * classifier's per-subject "N command timeouts over X" anomaly), and update+ * restarts (a supervisor-start whose subject already started within the+ * window, i.e. the supervisor was replaced). A first-ever supervisor start is+ * routine and never produces a notice. A timeout-burst notice carries the+ * latest timeout of the stall cluster its anomaly describes — the classifier+ * anchors the anomaly at the cluster's first timeout — so dismissing it+ * records a horizon that only covers that burst as dismissed: a later timeout+ * extending the burst past the horizon re-surfaces the notice, instead of it+ * staying hidden until the first timeout ages out of the window; an isolated+ * stray timeout, or a separate later stall, never moves the anchor and never+ * re-opens it.+ */+export function deriveIncidentNotices(entries: readonly IncidentLogEntry[], nowMs: number): IncidentNotice[] {+ const sinceMs = nowMs - INCIDENT_NOTICE_WINDOW_MS;+ // CLI window parity: buildIncidentReport windows events by [sinceMs, untilMs],+ // so a future-dated entry (clock skew, a bogus timestamp) is outside it too.+ const windowed = entries.filter((entry) => entry.timeMs >= sinceMs && entry.timeMs <= nowMs);+ const workerPids = collectWorkerPidMap(windowed);+ const events = collectIncidentEvents(windowed, workerPids);+ events.sort((a, b) => a.timeMs - b.timeMs);+ const notices: IncidentNotice[] = [];+ for (const event of events) {+ if (event.eventClass === "worker-crash") {+ // The subject already reads "worker <id>" ("worker" when the id is unknown).+ notices.push(+ createNotice(+ "worker-crash",+ event.severity,+ event.subject,+ event.timeMs,+ `${event.subject} crashed at ${formatIncidentNoticeTime(event.timeMs, nowMs)}`,+ ),+ );+ }+ }+ // Latest timeout of each subject's stall cluster — the same incident the+ // classifier's per-subject "N command timeouts over X" anomaly describes —+ // so a notice and its dismissal horizon always refer to one incident:+ // dismissal advances with a growing burst (a later timeout in the cluster+ // re-surfaces the notice past the horizon instead of it staying hidden+ // until the first timeout ages out), while a separate later stall or an+ // isolated stray timeout never moves the anchor.+ const latestStallTimeoutBySubject = latestIncidentStallTimeoutBySubject(events);+ for (const anomaly of computeIncidentAnomalies(events)) {+ if (anomaly.summary.includes("command timeouts")) {+ // The anomaly summary already reads "<subject>: N command timeouts over X".+ const latestTimeoutMs = latestStallTimeoutBySubject.get(anomaly.subject) ?? anomaly.timeMs;+ notices.push(+ createNotice("timeout-burst", anomaly.severity, anomaly.subject, latestTimeoutMs, anomaly.summary),+ );+ }+ }+ // The classifier also emits supervisor-start for failed spawns (lock held,+ // startup error) at warn/error severity; only a successful start — the+ // info-severity "listening on" event — counts toward a replacement, or two+ // failed spawns on one socket would read as a restart.+ const startedSubjects = new Set<string>();+ for (const event of events) {+ if (event.eventClass !== "supervisor-start" || event.severity !== "info") {+ continue;+ }+ if (startedSubjects.has(event.subject)) {+ notices.push(+ createNotice(+ "update-restart",+ event.severity,+ event.subject,+ event.timeMs,+ `daemon restarted for update at ${formatIncidentNoticeTime(event.timeMs, nowMs)}`,+ ),+ );+ } else {+ startedSubjects.add(event.subject);+ }+ }+ return notices;+}++/**+ * Collapse the derived notices to the single line the header shows: the most+ * severe wins (critical > error > warn > info), the most recent breaks ties.+ * Repeated identical events aggregate here — the header never stacks copies.+ */+export function selectIncidentNotice(notices: readonly IncidentNotice[]): IncidentNotice | undefined {+ let best: IncidentNotice | undefined;+ for (const notice of notices) {+ if (+ best === undefined ||+ SEVERITY_RANK[notice.severity] > SEVERITY_RANK[best.severity] ||+ (SEVERITY_RANK[notice.severity] === SEVERITY_RANK[best.severity] && notice.timeMs > best.timeMs)+ ) {+ best = notice;+ }+ }+ return best;+}++/** True when the notice sits at or before its key's dismissal horizon. */+export function isIncidentNoticeDismissed(notice: IncidentNotice, horizons: Record<string, number>): boolean {+ const horizon = horizons[notice.key];+ return horizon !== undefined && notice.timeMs <= horizon;+}++/**+ * Dismiss the notice currently showing. Records its timeMs as the horizon for+ * its key, so the same incident (and any older incident on that key) never+ * re-renders on later polls or view re-entry, while a newer qualifying+ * incident does. Returns false when no notice is showing.+ */+export function dismissIncidentNoticeState(state: IncidentNoticeState): boolean {+ const notice = state.notice;+ if (!notice) {+ return false;+ }+ state.dismissedHorizons[notice.key] = Math.max(state.dismissedHorizons[notice.key] ?? 0, notice.timeMs);+ state.notice = undefined;+ return true;+}++function sameIncidentNotice(a: IncidentNotice | undefined, b: IncidentNotice | undefined): boolean {+ return a?.key === b?.key && a?.timeMs === b?.timeMs && a?.text === b?.text;+}++interface IncidentLogChunk {+ lines: string[];+ nextOffset: number;+ fileId: string;+}++/**+ * Rotation-safe incremental read of agent.jsonl. Without a previous offset — or+ * after rotation (a changed inode), a shrink (recreation in place), or more+ * than one tail bound of new bytes — read the bounded tail: the cut may begin+ * mid-line (drop the torn leading fragment) or exactly at a record boundary+ * (the byte before the cut is a newline; keep the intact first record, which+ * dropping would silently lose for the lifetime of the state). Otherwise read+ * only appended bytes (every read stays bounded). A trailing partial line is+ * held back (the offset stops at its newline), so a mid-write line parses only+ * once complete, on a later poll — unless includeFinalPartialLine is set for a+ * frozen file (the rotated .old), which no later poll can complete; its final+ * line is returned as-is. A missing or unreadable file returns undefined;+ * offsets beyond the file size are never re-processed.+ */+function readIncidentLogLines(+ logPath: string,+ previousOffset: number | undefined,+ previousFileId: string | undefined,+ includeFinalPartialLine = false,+): IncidentLogChunk | undefined {+ let fd: number;+ try {+ fd = openSync(logPath, "r");+ } catch {+ return undefined;+ }+ try {+ const stats = fstatSync(fd);+ const fileId = `${stats.dev}:${stats.ino}`;+ const rotated = previousFileId !== undefined && fileId !== previousFileId;+ // Re-tail when nothing was read yet, after a rotation (a new file), when+ // the file shrank (recreated in place), or when more than one tail+ // bound appended since the last poll; every read stays bounded and+ // offsets are never re-processed.+ const retailed =+ previousOffset === undefined ||+ rotated ||+ previousOffset > stats.size ||+ stats.size - previousOffset > INCIDENT_NOTICE_TAIL_BYTES;+ const start = retailed ? Math.max(0, stats.size - INCIDENT_NOTICE_TAIL_BYTES) : previousOffset;+ if (start >= stats.size) {+ return { lines: [], nextOffset: stats.size, fileId };+ }+ const buffer = Buffer.allocUnsafe(stats.size - start);+ const bytesRead = readSync(fd, buffer, 0, buffer.length, start);+ let lineStart = 0;+ if (retailed && start > 0) {+ // The bounded tail may begin mid-line (the cut split a record: drop+ // the torn leading fragment) or exactly at a record boundary (the byte+ // before the cut is a newline: the first line in the buffer is a+ // complete record, and dropping it would silently lose a qualifying+ // incident for the lifetime of the state). Read that one byte to tell+ // the cases apart; the read happens only on this bounded re-tail path.+ const preceding = Buffer.alloc(1);+ const beginsMidLine = readSync(fd, preceding, 0, 1, start - 1) !== 1 || preceding[0] !== NEWLINE_BYTE;+ if (beginsMidLine) {+ // A chunk with no newline at all is one mid-write line: hold it+ // back so the completed line is still parsed by the next poll.+ const firstNewline = buffer.subarray(0, bytesRead).indexOf(NEWLINE_BYTE);+ if (firstNewline === -1) {+ return { lines: [], nextOffset: start, fileId };+ }+ lineStart = firstNewline + 1;+ }+ }+ let end = bytesRead;+ // A frozen file (the rotated .old) never gets the completing write, so+ // with includeFinalPartialLine its final line is returned as-is: a torn+ // line parses to undefined and drops harmlessly, while a complete+ // record missing only its newline must not be lost.+ if (!includeFinalPartialLine && end > 0 && buffer[end - 1] !== NEWLINE_BYTE) {+ // Hold back the partially-written final line until it completes.+ const lastNewline = buffer.subarray(0, bytesRead).lastIndexOf(NEWLINE_BYTE);+ if (lastNewline < lineStart) {+ return { lines: [], nextOffset: start, fileId };+ }+ end = lastNewline + 1;+ }+ const lines = buffer+ .toString("utf8", lineStart, end)+ .split("\n")+ .filter((line) => line.trim().length > 0);+ return { lines, nextOffset: start + end, fileId };+ } catch {+ return undefined;+ } finally {+ try {+ closeSync(fd);+ } catch {+ // Best-effort: a failed close must not mask the read result.+ }+ }+}++/**+ * Keep windowed entries in stable time order across polls: new entries append,+ * everything older than the window drops, and only the newest+ * INCIDENT_NOTICE_MAX_WINDOW_ENTRIES survive, so memory and per-poll work stay+ * bounded. Lines re-read after a rotation or a re-tail collapse harmlessly —+ * identical lifecycle events dedupe in the classifier, and the collapsed line+ * never stacks copies.+ */+function mergeIncidentWindowedEntries(+ entries: readonly IncidentLogEntry[],+ parsed: readonly IncidentLogEntry[],+ sinceMs: number,+): IncidentLogEntry[] {+ if (entries.length === 0 && parsed.length === 0) {+ return [];+ }+ const merged = [...entries, ...parsed];+ merged.sort((a, b) => a.timeMs - b.timeMs);+ const windowed = merged.filter((entry) => entry.timeMs >= sinceMs);+ // Newest entries win: slice the tail of the time-sorted array (entries stay+ // oldest-first) so retention cannot grow without bound on a busy log day.+ return windowed.length > INCIDENT_NOTICE_MAX_WINDOW_ENTRIES+ ? windowed.slice(windowed.length - INCIDENT_NOTICE_MAX_WINDOW_ENTRIES)+ : windowed;+}++/**+ * One best-effort poll: read new agent.jsonl bytes, keep the 24h window,+ * re-derive the qualifying notices, apply the dismissal horizons, and keep the+ * single collapsed line worth showing. The first successful read also tails+ * the rotated agent.jsonl.old — matching the CLI's [agent.jsonl.old,+ * agent.jsonl] source with the same bounded tail — so incidents spanning a+ * rotation still surface in a fresh view. Later polls read only appended+ * agent.jsonl bytes, except when the live read detects a new generation: the+ * un-consumed tail of the rotated-out one is then completed from .old by+ * offset continuation (never re-read from its start, which would duplicate+ * supervisor starts into phantom update restarts). Returns true when that line+ * changed so the caller can re-render. Never throws for a missing or unreadable log: that poll keeps+ * the consumed offset (a re-tail would fabricate restarts) but still+ * re-derives, so the notice expires with its window instead of surviving+ * forever while the log stays unreadable.+ */+export function refreshIncidentNoticeState(state: IncidentNoticeState, logPath: string, nowMs: number): boolean {+ // The first successful read bridges the rotated generation: the CLI reads+ // [agent.jsonl.old, agent.jsonl] (readIncidentLogEntries), so a view opened+ // after a rotation must see pairs that span it — an update restart whose+ // earlier supervisor start sits in .old, or a burst straddling the files.+ // A later re-read from .old's start would duplicate supervisor-start lines+ // into a phantom update restart (the classifier does not dedupe+ // supervisor-start), which is also why the consumed offset never resets.+ const firstRead = state.logOffset === undefined && state.logFileId === undefined;+ const chunk = readIncidentLogLines(logPath, state.logOffset, state.logFileId);+ const sinceMs = nowMs - INCIDENT_NOTICE_WINDOW_MS;+ // CLI window parity (buildIncidentReport bounds events by >= since && <=+ // until): future-dated entries fall outside the window and never surface.+ const parseWindowedLines = (lines: readonly string[]): IncidentLogEntry[] =>+ lines+ .map((line) => parseIncidentLogLine(line))+ .filter(+ (entry): entry is IncidentLogEntry =>+ entry !== undefined && entry.timeMs >= sinceMs && entry.timeMs <= nowMs,+ );+ let parsed: IncidentLogEntry[] = [];+ if (chunk !== undefined) {+ // Tail the rotated .old with the same bounded tail as the main log: the+ // CLI reads the same [agent.jsonl.old, agent.jsonl] pair, and the merge+ // below sorts by time, so the read order here does not matter.+ if (firstRead) {+ // The rotated .old is frozen: include its final line even without a+ // trailing newline — no later poll will ever complete it.+ const rotated = readIncidentLogLines(`${logPath}.old`, undefined, undefined, true);+ // A rename rotation can land between the live read above and this one:+ // the .old path then names the very file the chunk just consumed, and+ // re-parsing it would double supervisor starts into a phantom update+ // restart (the classifier does not dedupe supervisor-start) and double+ // timeout counts. Bridge only a genuinely different generation.+ if (rotated !== undefined && rotated.fileId !== chunk.fileId) {+ parsed = parseWindowedLines(rotated.lines);+ }+ } else if (state.logFileId !== undefined && chunk.fileId !== state.logFileId) {+ // A rotation between polls strands the un-consumed tail of the previous+ // generation at the .old path: no later poll re-reads it, so a crash+ // logged in that gap would surface in `prime-agent incident` but never+ // in the notice. The live read just detected the new generation, so+ // continue the old one from its consumed offset — the file id still+ // matches, the read is an offset continuation, and no consumed line is+ // re-parsed (a re-tail on an id mismatch stays bounded).+ const rotatedTail = readIncidentLogLines(`${logPath}.old`, state.logOffset, state.logFileId, true);+ if (rotatedTail !== undefined) {+ parsed = parseWindowedLines(rotatedTail.lines);+ }+ }+ parsed = parsed.concat(parseWindowedLines(chunk.lines));+ state.logOffset = chunk.nextOffset;+ state.logFileId = chunk.fileId;+ }+ // A missing or unreadable log keeps the consumed offset and file id exactly+ // as they are: resetting them would make the next poll re-tail and re-parse+ // consumed lines into a phantom second supervisor start (a false update+ // restart; the classifier does not dedupe supervisor-start) and doubled+ // timeout counts. The poll still falls through to the merge/derive path+ // with no new entries, so the notice ages out of its window while the log+ // stays unreadable instead of surviving forever; a real rotation is still+ // caught by the file id changing (or the offset passing the size) on the+ // next successful read.+ const previous = state.notice;+ state.entries = mergeIncidentWindowedEntries(state.entries, parsed, sinceMs);+ const notices = deriveIncidentNotices(state.entries, nowMs);+ state.notice = selectIncidentNotice(+ notices.filter((notice) => !isIncidentNoticeDismissed(notice, state.dismissedHorizons)),+ );+ return !sameIncidentNotice(previous, state.notice);+}diff --git a/packages/coding-agent/test/agents-view-incident-notice.test.ts b/packages/coding-agent/test/agents-view-incident-notice.test.tsnew file mode 100644index 0000000000..34c1b9a7a2--- /dev/null+++ b/packages/coding-agent/test/agents-view-incident-notice.test.ts@@ -0,0 +1,297 @@+import { appendFileSync, linkSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { setKeybindings } from "@earendil-works/pi-tui";+import stripAnsi from "strip-ansi";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { parseIncidentLogLine } from "../src/cli/incident.js";+import { ENV_AGENT_DIR, getAgentLogPath } from "../src/config.js";+import { KeybindingsManager } from "../src/core/keybindings.js";+import type { ModelRegistry } from "../src/core/model-registry.js";+import { SettingsManager } from "../src/core/settings-manager.js";+import { AgentsViewMode, type AgentsViewPersistentState } from "../src/modes/agents-view/agents-view-mode.js";+import {+ createIncidentNoticeState,+ deriveIncidentNotices,+ dismissIncidentNoticeState,+ formatIncidentNoticeTime,+ refreshIncidentNoticeState,+} from "../src/modes/agents-view/incident-notices.js";+import type { InteractiveModeUiServices } from "../src/modes/interactive/interactive-mode-services.js";+import { stopThemeWatcher } from "../src/modes/interactive/theme/theme.js";++vi.mock("../src/config.js", async (importOriginal) => {+ const actual = await importOriginal<typeof import("../src/config.js")>();+ return { ...actual, appendRotatingLog: vi.fn() };+});++vi.mock("../src/utils/tools-manager.js", async (importOriginal) => {+ const actual = await importOriginal<typeof import("../src/utils/tools-manager.js")>();+ // ensureTool would download fd into the redirected agent dir; tests never need it.+ return { ...actual, ensureTool: vi.fn(async () => undefined) };+});++const DAEMON_SOCKET = "/tmp/prime-agent-501/daemon.sock";+function logLine(fields: Record<string, unknown>): string {+ return JSON.stringify({ level: "warn", ...fields });+}+function tsAgo(base: number, ms: number): string {+ return new Date(base - ms).toISOString();+}+function supervisorStartLine(base: number, minutesAgoValue: number, generation = "e14de15c"): string {+ return logLine({+ ts: tsAgo(base, minutesAgoValue * 60_000),+ component: "coding-agent.daemon-supervisor",+ msg: `Prime Agent daemon supervisor ${generation} listening on ${DAEMON_SOCKET}`,+ socketPath: DAEMON_SOCKET,+ pid: 15026,+ });+}+function workerCrashLine(base: number, workerId: string, secondsAgoValue: number): string {+ return logLine({+ ts: tsAgo(base, secondsAgoValue * 1000),+ component: "coding-agent.daemon-supervisor",+ msg: `Session worker ${workerId} stderr: uncaught exception: Error: write EPIPE`,+ });+}+function commandTimeoutLine(base: number, minutesAgoValue: number, socketPath: string = DAEMON_SOCKET): string {+ return logLine({+ ts: tsAgo(base, minutesAgoValue * 60_000),+ component: "coding-agent.daemon-supervisor",+ socketPath,+ msg: "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach\n at Timeout._onTimeout (node:internal/timers:618:7)",+ });+}+function fixtureEntries(lines: readonly string[]) {+ return lines.map((line) => parseIncidentLogLine(line)).filter((entry) => entry !== undefined);+}+function invoke(method: string, self: object, ...args: unknown[]): unknown {+ const member = Reflect.get(AgentsViewMode.prototype, method) as ((...a: unknown[]) => unknown) | undefined;+ if (typeof member !== "function") throw new Error(`AgentsViewMode.${method} no longer exists`);+ return member.call(self, ...args);+}+function createUiServices(): InteractiveModeUiServices {+ return {+ settingsManager: SettingsManager.inMemory({ theme: "dark" }),+ modelRegistry: {} as ModelRegistry,+ getInitialCwd: () => process.cwd(),+ getInitialSessionName: () => undefined,+ getThemes: () => [],+ };+}+const cleanupDirs: string[] = [];+let previousAgentDir: string | undefined;+/** Fresh per-test agent dir so getAgentLogPath() points at a fixture log. */+function useTempAgentDir(): string {+ const dir = mkdtempSync(join(tmpdir(), "agents-view-incident-"));+ cleanupDirs.push(dir);+ process.env[ENV_AGENT_DIR] = dir;+ mkdirSync(join(dir, "logs"), { recursive: true });+ return dir;+}+function writeAgentLog(lines: readonly string[]): void {+ writeFileSync(getAgentLogPath(), `${lines.join("\n")}\n`);+}+function appendAgentLog(lines: readonly string[]): void {+ appendFileSync(getAgentLogPath(), `${lines.join("\n")}\n`);+}+function newView(persistentState: AgentsViewPersistentState = { savedCatalogLoaded: true }): AgentsViewMode {+ return new AgentsViewMode({ config: {}, uiServices: createUiServices() }, persistentState);+}+/** The incident notice lines renderContent produces (startup notices stay unset). */+function renderedIncidentLines(view: AgentsViewMode): string[] {+ const lines = invoke("renderContent", view, 120, 40) as string[];+ return lines.map(stripAnsi).filter((line) => line.includes("prime-agent incident"));+}+beforeAll(() => {+ setKeybindings(new KeybindingsManager());+ previousAgentDir = process.env[ENV_AGENT_DIR];+});+afterAll(() => {+ if (previousAgentDir === undefined) {+ delete process.env[ENV_AGENT_DIR];+ } else {+ process.env[ENV_AGENT_DIR] = previousAgentDir;+ }+ for (const dir of cleanupDirs) {+ rmSync(dir, { recursive: true, force: true });+ }+});+beforeEach(() => {+ vi.clearAllMocks();+});++describe("incident notice derivation", () => {+ it("anchors each subject's timeout burst at its own cluster's latest timeout", () => {+ const base = Date.now();+ // Several daemon sockets, each with its own stall: the per-subject+ // cluster lookup must never leak or mis-anchor across subjects.+ const otherSocket = "/tmp/prime-agent-501/daemon-other.sock";+ const spacedSocket = "/tmp/prime-agent-501/daemon-spaced.sock";+ const spacedTimeouts = [200, 190, 180, 30, 29].map((minutes) => commandTimeoutLine(base, minutes, spacedSocket));+ const entries = fixtureEntries([+ commandTimeoutLine(base, 30, DAEMON_SOCKET),+ commandTimeoutLine(base, 25, otherSocket),+ commandTimeoutLine(base, 20, DAEMON_SOCKET),+ commandTimeoutLine(base, 5, otherSocket),+ ...spacedTimeouts,+ ]);+ const bursts = deriveIncidentNotices(entries, base).filter((notice) => notice.kind === "timeout-burst");+ expect(bursts).toHaveLength(3);+ const bySubject = new Map(bursts.map((notice) => [notice.subject, notice]));+ expect(bySubject.get(DAEMON_SOCKET)?.timeMs).toBe(base - 20 * 60_000);+ expect(bySubject.get(otherSocket)?.timeMs).toBe(base - 5 * 60_000);+ expect(bySubject.get(spacedSocket)?.text).toContain("3 command timeouts over 20m");+ expect(bySubject.get(spacedSocket)?.timeMs).toBe(base - 180 * 60_000);+ });++ it("derives an update-restart only from repeated successful supervisor starts", () => {+ const base = Date.now();+ const replaced = fixtureEntries([supervisorStartLine(base, 120), supervisorStartLine(base, 60)]);+ const restarts = deriveIncidentNotices(replaced, Date.now());+ expect(restarts).toHaveLength(1);+ expect(restarts[0]).toMatchObject({+ kind: "update-restart",+ severity: "info",+ subject: DAEMON_SOCKET,+ timeMs: replaced[1]!.timeMs,+ });+ expect(restarts[0]!.text).toBe(+ `daemon restarted for update at ${formatIncidentNoticeTime(replaced[1]!.timeMs, Date.now())}`,+ );++ // A first-ever start is routine; a failed startup (lock held) never+ // counts toward a replacement, or two failed spawns on one socket+ // would read as an update restart.+ const failedStartLine = (minutesAgoValue: number) =>+ logLine({+ ts: tsAgo(base, minutesAgoValue * 60_000),+ component: "coding-agent.daemon-supervisor",+ socketPath: DAEMON_SOCKET,+ msg: "Daemon supervisor startup failed: lock file is already being held by another process",+ });+ expect(deriveIncidentNotices(fixtureEntries([supervisorStartLine(base, 60)]), Date.now())).toEqual([]);+ expect(+ deriveIncidentNotices(fixtureEntries([failedStartLine(30), supervisorStartLine(base, 20)]), Date.now()),+ ).toEqual([]);+ });+});++describe("agents view incident notices", () => {+ it("renders the collapsed worker-crash notice line from the log tail", () => {+ useTempAgentDir();+ const base = Date.now();+ writeAgentLog([supervisorStartLine(base, 600), workerCrashLine(base, "5b1d3aeb91ee", 120)]);+ const view = newView();+ try {+ invoke("refreshIncidentNotices", view);+ const lines = renderedIncidentLines(view);+ expect(lines).toHaveLength(1);+ expect(lines[0]).toContain(+ `worker 5b1d3aeb91ee crashed at ${formatIncidentNoticeTime(base - 120_000, Date.now())}`,+ );+ expect(lines[0]).toContain("prime-agent incident for the timeline");+ expect(lines[0].startsWith(" \u26a0")).toBe(true);+ } finally {+ stopThemeWatcher();+ }+ });++ it("cancels an armed delete confirmation with Esc instead of dismissing the notice", () => {+ useTempAgentDir();+ writeAgentLog([workerCrashLine(Date.now(), "5b1d3aeb91ee", 120)]);+ const view = newView();+ try {+ invoke("refreshIncidentNotices", view);+ expect(renderedIncidentLines(view)).toHaveLength(1);+ // A delete confirmation is armed: Esc must cancel it and keep the+ // notice, or the next delete press would fire without a fresh+ // confirmation.+ invoke("showDeleteConfirmation", view);+ view.handleInput("\x1b");+ expect(Reflect.get(view, "deleteConfirmExpiresAt")).toBe(0);+ expect(renderedIncidentLines(view)).toHaveLength(1);+ } finally {+ stopThemeWatcher();+ }+ });++ it("dismisses with Esc and never resurrects across later polls", () => {+ useTempAgentDir();+ const base = Date.now();+ writeAgentLog([workerCrashLine(base, "5b1d3aeb91ee", 120)]);+ const persistentState: AgentsViewPersistentState = { savedCatalogLoaded: true };+ const view = newView(persistentState);+ try {+ invoke("refreshIncidentNotices", view);+ expect(renderedIncidentLines(view)).toHaveLength(1);++ view.handleInput("\x1b");+ expect(Reflect.get(view, "statusMessage")).toBe("Incident notice dismissed");+ // Later polls re-read the same incident from the log; dismissal is sticky.+ invoke("refreshIncidentNotices", view);+ expect(renderedIncidentLines(view)).toHaveLength(0);+ } finally {+ stopThemeWatcher();+ }+ });++ it("re-shows a dismissed timeout-burst when a later timeout extends the burst", () => {+ useTempAgentDir();+ const base = Date.now();+ writeAgentLog([commandTimeoutLine(base, 30), commandTimeoutLine(base, 29)]);+ const state = createIncidentNoticeState();+ const logPath = getAgentLogPath();+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);+ expect(dismissIncidentNoticeState(state)).toBe(true);+ // A later timeout extends the burst past the dismissed horizon: the+ // notice reappears instead of staying hidden until the first timeout+ // ages out.+ appendAgentLog([commandTimeoutLine(base, 5)]);+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);+ expect(state.notice).toMatchObject({ kind: "timeout-burst", timeMs: base - 5 * 60_000 });+ });++ it("keeps a dismissed timeout-burst hidden when a later timeout is isolated", () => {+ useTempAgentDir();+ const base = Date.now();+ writeAgentLog([commandTimeoutLine(base, 60), commandTimeoutLine(base, 59)]);+ const state = createIncidentNoticeState();+ const logPath = getAgentLogPath();+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(true);+ expect(dismissIncidentNoticeState(state)).toBe(true);+ appendAgentLog([commandTimeoutLine(base, 5)]);+ expect(refreshIncidentNoticeState(state, logPath, base)).toBe(false);+ });++ it("surfaces a crash at the end of agent.jsonl.old without a trailing newline", () => {+ useTempAgentDir();+ writeFileSync(`${getAgentLogPath()}.old`, workerCrashLine(Date.now(), "5b1d3aeb91ee", 120));+ writeAgentLog([]);+ const state = createIncidentNoticeState();+ refreshIncidentNoticeState(state, getAgentLogPath(), Date.now());+ expect(state.notice).toMatchObject({ kind: "worker-crash" });+ });++ it("skips the .old bridge when a rotation makes it the live log's own generation", () => {+ useTempAgentDir();+ writeAgentLog([supervisorStartLine(Date.now(), 600)]);+ linkSync(getAgentLogPath(), `${getAgentLogPath()}.old`);+ const state = createIncidentNoticeState();+ refreshIncidentNoticeState(state, getAgentLogPath(), Date.now());+ expect(state.entries).toHaveLength(1);+ });++ it("completes the un-consumed tail of a generation that rotates out mid-session", () => {+ useTempAgentDir();+ const logPath = getAgentLogPath();+ writeAgentLog([workerCrashLine(Date.now(), "5b1d3aeb91ee", 60)]);+ const state = createIncidentNoticeState();+ refreshIncidentNoticeState(state, logPath, Date.now());+ appendAgentLog([workerCrashLine(Date.now(), "9f2c7a44b021", 30)]);+ renameSync(logPath, `${logPath}.old`);+ writeAgentLog([]);+ refreshIncidentNoticeState(state, logPath, Date.now());+ expect(state.notice).toMatchObject({ subject: "worker 9f2c7a44b021" });+ });+});diff --git a/packages/coding-agent/test/agents-view-mode.test.ts b/packages/coding-agent/test/agents-view-mode.test.tsindex 88e4c17e14..79032fd2ee 100644--- a/packages/coding-agent/test/agents-view-mode.test.ts+++ b/packages/coding-agent/test/agents-view-mode.test.ts@@ -1032,6 +1032,7 @@ describe("AgentsViewMode catalog performance", () => {resolveMissingSelectionAnchor: vi.fn(),refreshHeartbeats: vi.fn(async () => true),loadStartupNotices: vi.fn(),+ refreshIncidentNotices: vi.fn(),rebuildRows: vi.fn(),clearCtrlCExitHint: vi.fn(),clearDeleteConfirmation: vi.fn(),diff --git a/packages/coding-agent/test/incident.test.ts b/packages/coding-agent/test/incident.test.tsnew file mode 100644index 0000000000..29f9600dee--- /dev/null+++ b/packages/coding-agent/test/incident.test.ts@@ -0,0 +1,735 @@+import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import stripAnsi from "strip-ansi";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ buildIncidentReport,+ collectIncidentEvents,+ collectWorkerPidMap,+ computeIncidentAnomalies,+ type IncidentLogEntry,+ IncidentUsageError,+ parseIncidentDaemonLogLine,+ parseIncidentLogLine,+ parseIncidentOptions,+ parseIncidentTimeBound,+ runIncident,+} from "../src/cli/incident.js";+import { ENV_AGENT_DIR } from "../src/config.js";++function agentLogLine(fields: Record<string, unknown>): string {+ return JSON.stringify({ ts: "2026-09-10T20:00:00.000Z", level: "warn", ...fields });+}++function entry(line: string): IncidentLogEntry {+ return parseIncidentLogLine(line)!;+}++const sinceMs = Date.parse("2026-09-10T20:00:00Z");+const untilMs = Date.parse("2026-09-10T20:30:00Z");++function reportFor(lines: string[], options: { since?: number; until?: number; session?: string } = {}) {+ const entries = lines.map((line) => parseIncidentLogLine(line)!).filter((item) => item !== undefined);+ return stripAnsi(+ buildIncidentReport(entries, {+ sinceMs: options.since ?? sinceMs,+ untilMs: options.until ?? untilMs,+ ...(options.session !== undefined ? { session: options.session } : {}),+ }).text,+ );+}++/** Supervisor-side log line (daemon-supervisor component). */+function supervisorLine(ts: string, msg: string, extra: Record<string, unknown> = {}): string {+ return agentLogLine({ ts, component: "coding-agent.daemon-supervisor", ...extra, msg });+}++/** Worker daemon startup line, the pid sighting that anchors attribution. */+function workerStartLine(ts: string, socketPath: string, pid: number): string {+ return agentLogLine({+ ts,+ component: "coding-agent.daemon",+ socketPath,+ pid,+ msg: `Prime Agent daemon listening on ${socketPath}`,+ });+}++/** Worker daemon log line for a worker running under the fixture pid. */+function daemonLine(ts: string, socketPath: string, msg: string): string {+ return agentLogLine({ ts, component: "coding-agent.daemon", socketPath, pid: 53615, msg });+}++/** Provider stream failure line (rate_limit 429) attributed by pid. */+function providerFailureLine(ts: string, pid: number): string {+ return agentLogLine({+ ts,+ level: "error",+ component: "ai.provider",+ pid,+ msg: "provider stream failure",+ kind: "rate_limit",+ status: 429,+ });+}++// ---------------------------------------------------------------------------+// Green run: normal startup, worker start/stop, no errors.+// ---------------------------------------------------------------------------++describe("incident timeline for a clean run", () => {+ it("shows supervisor and worker events with no anomaly entries", () => {+ const text = reportFor(+ [+ [+ "2026-09-10T20:00:05.000Z",+ "Prime Agent daemon supervisor e14de15c listening on /tmp/prime-agent-501/daemon.sock",+ ],+ [+ "2026-09-10T20:01:00.000Z",+ "Session worker 477fef4e85a8 stderr: Prime Agent daemon listening on /tmp/prime-agent-501/worker-a-477fef4e85a8.sock",+ ],+ ["2026-09-10T20:02:00.000Z", "Migrated 2 scheduled jobs into session artifacts"],+ [+ "2026-09-10T20:10:00.000Z",+ "Session worker 477fef4e85a8 stderr: shutdown command received over socket; 1 active session(s) will be closed",+ ],+ [+ "2026-09-10T20:10:01.000Z",+ "Session worker 477fef4e85a8 stderr: shutting down (exit 0); closing 1 active session(s)",+ ],+ [+ "2026-09-10T20:11:00.000Z",+ "Evicted empty session worker 477fef4e85a8 root=01a0-abc on last client detach",+ ],+ ].map(([ts, msg]) => supervisorLine(ts, msg, { pid: 15026 })),+ );+ expect(text).toContain("Supervisor events");+ expect(text).toContain("worker 477fef4e85a8 started");+ expect(text).toContain("worker 477fef4e85a8 stop requested (1 active session(s))");+ expect(text).toContain("worker 477fef4e85a8 stopped (exit 0, 1 session(s) closed)");+ expect(text).toContain("evicted empty session worker 477fef4e85a8 on last detach");+ expect(text).toContain("migrated 2 scheduled jobs into session artifacts");+ const anomalies = text.split("Session anomalies")[1]!.split("Recovery")[0]!;+ expect(anomalies).toContain("(none)");+ expect(anomalies).not.toContain("timeout");+ expect(text).not.toContain("critical");+ });+});++// ---------------------------------------------------------------------------+// Crash + recovery: the incident class from 2026-09-10 (attach timeouts+// 20:02-20:21, EPIPE worker crash, recovery holding backlogged operations).+// ---------------------------------------------------------------------------++function incidentFixtureLines(): string[] {+ const lines: string[] = [];+ const supervisor = {+ component: "coding-agent.daemon-supervisor",+ socketPath: "/tmp/prime-agent-501/daemon.sock",+ pid: 15026,+ };+ const crashedWorker = {+ component: "coding-agent.daemon",+ socketPath: "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",+ pid: 53615,+ };+ const supervisorMsg = (ts: string, msg: string) => agentLogLine({ ...supervisor, ts, msg });+ // Worker starts before the window; provider failures are attributed to it by pid.+ lines.push(workerStartLine("2026-09-10T19:42:09.064Z", crashedWorker.socketPath, 53615));+ // Attach timeouts 20:02-20:21.+ for (const [ts, command] of [+ ["2026-09-10T20:02:39.764Z", "attach"],+ ["2026-09-10T20:08:37.410Z", "attach"],+ ["2026-09-10T20:09:16.853Z", "attach"],+ ["2026-09-10T20:21:53.062Z", "attach"],+ ] as const) {+ lines.push(+ supervisorMsg(+ ts,+ `Supervisor command ${command} failed: Error: Timed out waiting for daemon worker response to ${command}\n at Timeout._onTimeout (node:internal/timers:618:7)`,+ ),+ );+ }+ // Per-session catch-up timeouts for 2339fb7da605 and one other session.+ lines.push(+ supervisorMsg(+ "2026-09-10T20:02:33.708Z",+ "Failed to catch up client daemon-client:ac0fbf2a for dcdced964c0d: Error: Timed out waiting for daemon worker response to attach",+ ),+ );+ for (const ts of ["2026-09-10T20:04:50.397Z", "2026-09-10T20:11:29.190Z", "2026-09-10T20:14:57.615Z"]) {+ lines.push(+ supervisorMsg(+ ts,+ "Failed to catch up client daemon-client:dc5ad892 for 2339fb7da605: Error: Timed out waiting for daemon worker response to attach",+ ),+ );+ }+ lines.push(+ supervisorMsg(+ "2026-09-10T20:21:58.137Z",+ "Could not list heartbeats from a worker: Timed out waiting for daemon worker response to heartbeats_list",+ ),+ supervisorMsg(+ "2026-09-10T20:02:30.000Z",+ "Supervisor command list_agent_peers failed: Error: Worker authentication failed\n at handleCommand (chunk.js:1:1)",+ ),+ );+ // Provider stream failures from the overloaded worker (real entries carry+ // only the worker pid, no socket path).+ for (let index = 0; index < 30; index++) {+ lines.push(+ agentLogLine({+ ts: `2026-09-10T20:${(2 + Math.floor(index / 2)).toString().padStart(2, "0")}:${(index % 60).toString().padStart(2, "0")}.000Z`,+ level: "error",+ component: "ai.provider",+ pid: 53615,+ msg: "provider stream failure",+ kind: index % 5 === 0 ? "server_error" : "rate_limit",+ status: index % 5 === 0 ? 504 : 429,+ }),+ );+ }+ // EPIPE crash: logged once by the worker and once via the stderr forward.+ lines.push(+ agentLogLine({+ ...crashedWorker,+ ts: "2026-09-10T20:23:24.945Z",+ msg: "uncaught exception: Error: write EPIPE\n at afterWriteDispatched (node:internal/stream_base_commons:159:15)",+ }),+ supervisorMsg(+ "2026-09-10T20:23:24.945Z",+ "Session worker 5b1d3aeb91ee stderr: uncaught exception: Error: write EPIPE",+ ),+ supervisorMsg(+ "2026-09-10T20:23:24.946Z",+ "Session worker 5b1d3aeb91ee stderr: at afterWriteDispatched (node:internal/stream_base_commons:159:15)",+ ),+ );+ // Recovery: worker replaced, backlogged uncertain operations held.+ const operations = [+ ...Array<string>(408).fill("tool_execution_start"),+ ...Array<string>(62).fill("auto_retry_end"),+ ...Array<string>(47).fill("agent_end"),+ ...Array<string>(16).fill("message_start"),+ ].join(", ");+ lines.push(+ supervisorMsg(+ "2026-09-10T20:23:29.521Z",+ `Recovered worker 5b1d3aeb91ee without replaying uncertain operations: ${operations}`,+ ),+ supervisorMsg(+ "2026-09-10T20:23:39.520Z",+ "Could not adopt worker 5b1d3aeb91ee: Error: Session worker process is no longer running",+ ),+ );+ return lines;+}++describe("incident timeline for a crash + recovery", () => {+ it("reconstructs the incident narrative in one report", () => {+ const text = reportFor(incidentFixtureLines());+ // Timeouts aggregated with count and span.+ expect(text).toContain("command attach failed: timed out waiting for worker response (x4, until 09-10 20:21:53)");+ expect(text).toContain(+ "client catch-up failed for session 2339fb7da605: Timed out waiting for daemon worker response to attach (x3, until 09-10 20:14:57)",+ );+ expect(text).toContain("command list_agent_peers failed: worker authentication failed");+ // The crash appears once, critical, with the EPIPE cause.+ expect(text.match(/crashed/g)?.length).toBe(1);+ expect(text).toContain(+ "09-10 20:23:24 critical worker 5b1d3aeb91ee crashed: uncaught exception: Error: write EPIPE",+ );+ // Recovery with the held backlog and its operation breakdown.+ expect(text).toContain(+ "worker 5b1d3aeb91ee recovered; 533 uncertain operations not replayed (tool_execution_start x408, auto_retry_end x62, agent_end x47, message_start x16)",+ );+ expect(text).toContain("could not adopt worker 5b1d3aeb91ee: Session worker process is no longer running");+ // Anomalies: a timeout stall, an auth failure, and provider failures attributed to the crashed worker.+ expect(text).toContain("/tmp/prime-agent-501/daemon.sock: 5 command timeouts over 19m18s");+ expect(text).toContain("session 2339fb7da605: 3 command timeouts over 10m7s");+ expect(text).toContain("provider stream failure (rate_limit 429) for worker 5b1d3aeb91ee (x24");+ expect(text).toContain("provider stream failure (server_error 504) for worker 5b1d3aeb91ee (x6");+ });+});++// ---------------------------------------------------------------------------+// Window and session filtering.+// ---------------------------------------------------------------------------++describe("incident window and session filtering", () => {+ it("excludes events outside the window", () => {+ const lines = incidentFixtureLines();+ const text = reportFor(lines, {+ since: Date.parse("2026-09-10T20:05:00Z"),+ until: Date.parse("2026-09-10T20:10:00Z"),+ });+ expect(text).toContain("command attach failed: timed out waiting for worker response (x2, until 09-10 20:09:16)");+ expect(text).not.toContain("20:21:53");+ expect(text).not.toContain("write EPIPE");+ expect(text).not.toContain("Recovered worker");+ });++ it("filters to events naming the session, with prefix matching", () => {+ const lines = incidentFixtureLines();+ const full = reportFor(lines);+ const filtered = reportFor(lines, { session: "2339fb7" });+ expect(filtered).toContain("client catch-up failed for session 2339fb7da605");+ expect(filtered).toContain("session 2339fb7da605: 3 command timeouts over 10m7s");+ expect(filtered).toContain("Session filter: 2339fb7");+ expect(filtered).not.toContain("command attach failed: timed out");+ expect(filtered).not.toContain("write EPIPE");+ expect(filtered).not.toContain("dcdced964c0d");+ expect(full).toContain("dcdced964c0d");+ });++ it("matches session names quoted in log messages", () => {+ const text = reportFor(+ [+ supervisorLine(+ "2026-09-10T20:05:00.000Z",+ 'Supervisor command set_session_name failed: Error: Agent name "Faerie" is unavailable',+ ),+ ],+ { session: "Faerie" },+ );+ expect(text).toContain('Agent name "Faerie" is unavailable');+ });++ it("does not let a passivation event with an empty session name match every filter", () => {+ // daemon-mode logs `name=""` when the session has no name; that empty+ // token must not prefix-match every --session value.+ const text = reportFor(+ [+ daemonLine(+ "2026-09-10T20:05:00.000Z",+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",+ 'Passivated idle child sessionId=feedface1234 name="" idleMinutes=5',+ ),+ ],+ { session: "aabbccddeeff" },+ );+ expect(text).toContain('reference session "aabbccddeeff"');+ });++ it("keeps session tokens from worker command failures", () => {+ const text = reportFor(+ [+ daemonLine(+ "2026-09-10T20:05:00.000Z",+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",+ 'daemon command "set_session_name" failed: Error: Agent name "Faerie" is unavailable',+ ),+ ],+ { session: "Faerie" },+ );+ expect(text).toContain('Agent name "Faerie" is unavailable');+ });++ it("says clearly when nothing in the window references the session", () => {+ const text = reportFor(incidentFixtureLines(), { session: "deadbeef1234" });+ expect(text).toContain('reference session "deadbeef1234"');+ });+});++// ---------------------------------------------------------------------------+// Time parsing and option parsing.+// ---------------------------------------------------------------------------++describe("parseIncidentTimeBound", () => {+ const now = new Date(Date.parse("2026-09-16T22:30:00Z"));++ it("parses ISO date-times, dates, and bare times as UTC", () => {+ expect(parseIncidentTimeBound("2026-09-16T20:02", now, "--since")).toBe(Date.parse("2026-09-16T20:02:00.000Z"));+ expect(parseIncidentTimeBound("2026-09-16", now, "--since")).toBe(Date.parse("2026-09-16T00:00:00.000Z"));+ expect(parseIncidentTimeBound("20:02", now, "--since")).toBe(Date.parse("2026-09-16T20:02:00.000Z"));+ expect(parseIncidentTimeBound("2026-09-16T20:02:53Z", now, "--since")).toBe(+ Date.parse("2026-09-16T20:02:53.000Z"),+ );+ expect(parseIncidentTimeBound("2026-09-16T22:02+02:00", now, "--since")).toBe(+ Date.parse("2026-09-16T20:02:00.000Z"),+ );+ });++ it("accepts fractional seconds pasted from the log", () => {+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.764Z", now, "--since")).toBe(+ Date.parse("2026-09-16T20:02:39.764Z"),+ );+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.764", now, "--since")).toBe(+ Date.parse("2026-09-16T20:02:39.764Z"),+ );+ expect(parseIncidentTimeBound("2026-09-16T20:02:39.7", now, "--since")).toBe(+ Date.parse("2026-09-16T20:02:39.700Z"),+ );+ });++ it("rejects timezone offsets with an invalid minute component", () => {+ // RFC 3339 offsets allow minutes 00-59 only; +00:60 must not shift by an hour.+ for (const garbage of ["2026-09-16T20:02+00:60", "2026-09-16T20:02-05:90", "2026-09-16T20:02+24:00"]) {+ expect(() => parseIncidentTimeBound(garbage, now, "--since")).toThrow(IncidentUsageError);+ }+ expect(parseIncidentTimeBound("2026-09-16T20:02+05:45", now, "--since")).toBe(+ Date.parse("2026-09-16T14:17:00.000Z"),+ );+ });++ it("rejects garbage and impossible dates with a clear usage error", () => {+ for (const garbage of ["yesterday", "2026-13-01", "2026-09-32", "25:00", "2026-09-16T"]) {+ expect(() => parseIncidentTimeBound(garbage, now, "--since")).toThrow(IncidentUsageError);+ }+ const message = (() => {+ try {+ parseIncidentTimeBound("garbage", now, "--since");+ } catch (error) {+ return (error as Error).message;+ }+ })();+ expect(message).toContain("--since");+ expect(message).toContain("20:02");+ });+});++describe("parseIncidentOptions", () => {+ it("parses separated and =-attached values", () => {+ expect(parseIncidentOptions(["--since", "20:02", "--until=21:00", "--session", "abc"])).toEqual({+ since: "20:02",+ until: "21:00",+ session: "abc",+ });+ expect(parseIncidentOptions([])).toEqual({});+ });++ it("rejects unknown options, missing values, and empty values", () => {+ expect(() => parseIncidentOptions(["--json"])).toThrow(IncidentUsageError);+ expect(() => parseIncidentOptions(["extra"])).toThrow(IncidentUsageError);+ expect(() => parseIncidentOptions(["--since"])).toThrow(IncidentUsageError);+ expect(() => parseIncidentOptions(["--since", ""])).toThrow(IncidentUsageError);+ expect(() => parseIncidentOptions(["--since="])).toThrow(IncidentUsageError);+ expect(() => parseIncidentOptions(["--session", " "])).toThrow(IncidentUsageError);+ });+});++// ---------------------------------------------------------------------------+// Per-daemon log fallback and malformed-line robustness.+// ---------------------------------------------------------------------------++describe("parseIncidentDaemonLogLine", () => {+ it("parses supervisor and worker lines", () => {+ const supervisor = parseIncidentDaemonLogLine(+ "[2026-09-10T20:02:39.765Z] supervisor: Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",+ );+ expect(supervisor?.component).toBe("coding-agent.daemon-supervisor");+ expect(supervisor?.timeMs).toBe(Date.parse("2026-09-10T20:02:39.765Z"));+ const worker = parseIncidentDaemonLogLine("[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE");+ expect(worker?.component).toBe("coding-agent.daemon");+ expect(worker?.msg).toBe("uncaught exception: Error: write EPIPE");+ });++ it("returns undefined for malformed lines", () => {+ expect(parseIncidentDaemonLogLine("not a log line")).toBeUndefined();+ expect(parseIncidentDaemonLogLine("[not-a-timestamp] supervisor: msg")).toBeUndefined();+ expect(parseIncidentDaemonLogLine("[]")).toBeUndefined();+ });+});++describe("runIncident over a fixture agent dir", () => {+ let agentDir: string;+ let logs: string[];++ beforeEach(() => {+ agentDir = mkdtempSync(join(tmpdir(), "prime-incident-test-"));+ logs = [];+ vi.spyOn(console, "log").mockImplementation((line: string) => {+ logs.push(line);+ });+ process.env[ENV_AGENT_DIR] = agentDir;+ });++ afterEach(() => {+ delete process.env[ENV_AGENT_DIR];+ vi.restoreAllMocks();+ rmSync(agentDir, { recursive: true, force: true });+ });++ it("tolerates malformed lines and reports them as skipped", async () => {+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ const contents = [+ "this is not json",+ JSON.stringify({ noTs: true, msg: "missing ts" }),+ supervisorLine(+ "2026-09-10T20:02:39.764Z",+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",+ ),+ "",+ ].join("\n");+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), `${contents}\n`);+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("command attach failed: timed out waiting for worker response");+ expect(text).toContain("3 lines scanned, 1 events in window, 2 unreadable skipped");+ });++ it("falls back to the newest per-daemon log when agent.jsonl is absent", async () => {+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ writeFileSync(+ join(agentDir, "logs", "old-sock.a1b2c3d4.log"),+ "[2026-09-09T10:00:00.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held\n",+ );+ writeFileSync(+ join(agentDir, "logs", "daemon.sock.98ed5cb2.log"),+ [+ "[2026-09-10T20:00:05.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held",+ "[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE",+ "[2026-09-10T20:23:29.521Z] supervisor: Recovered worker 5b1d3aeb91ee without replaying uncertain operations: tool_execution_start, agent_end",+ ].join("\n"),+ );+ const decoyDir = join(agentDir, "logs", "daemon.sock.deadbeef.log");+ mkdirSync(decoyDir);+ utimesSync(decoyDir, new Date(Date.now() + 60_000), new Date(Date.now() + 60_000));+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("daemon.sock.98ed5cb2.log");+ expect(text).toContain("supervisor startup blocked: another daemon holds the lock");+ expect(text).toContain(+ "worker 5b1d3aeb91ee recovered; 2 uncertain operations not replayed (agent_end x1, tool_execution_start x1)",+ );+ });++ it("falls back to daemon logs whose socket basename does not end in .sock", async () => {+ // A daemon started with `--daemon-socket /tmp/prime-daemon` writes+ // prime-daemon.<hash>.log, which has no .sock segment to match on.+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ writeFileSync(+ join(agentDir, "logs", "prime-daemon.a1b2c3d4.log"),+ "[2026-09-10T20:00:05.000Z] supervisor: Daemon supervisor startup failed: Error: Lock file is already being held\n",+ );+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("prime-daemon.a1b2c3d4.log");+ expect(text).toContain("supervisor startup blocked: another daemon holds the lock");+ });++ it("uses the pre-resolved window passed by the caller", async () => {+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ writeFileSync(+ join(agentDir, "logs", "agent.jsonl"),+ `${supervisorLine("2026-09-10T20:02:39.764Z", "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach")}\n`,+ );+ await runIncident(+ {},+ { sinceMs: Date.parse("2026-09-10T20:00:00Z"), untilMs: Date.parse("2026-09-10T20:30:00Z") },+ );+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("Window: 09-10 20:00:00 → 09-10 20:30:00 UTC (30m)");+ expect(text).toContain("command attach failed: timed out waiting for worker response");+ });++ it("falls back to the per-daemon log when agent.jsonl is empty or unreadable", async () => {+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ const daemonLog = [+ "[2026-09-10T20:23:24.945Z] uncaught exception: Error: write EPIPE",+ "[2026-09-10T20:23:29.521Z] supervisor: Recovered worker 5b1d3aeb91ee without replaying uncertain operations: tool_execution_start, agent_end",+ ].join("\n");+ writeFileSync(join(agentDir, "logs", "daemon.sock.98ed5cb2.log"), `${daemonLog}\n`);+ // Unreadable agent.jsonl: nothing parses.+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), "this is not json\nalso not json\n");+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ let text = stripAnsi(logs.join("\n"));+ expect(text).toContain("daemon.sock.98ed5cb2.log");+ expect(text).toContain("worker crashed: uncaught exception: Error: write EPIPE");+ expect(text).toContain("worker 5b1d3aeb91ee recovered; 2 uncertain operations not replayed");+ // Empty agent.jsonl: zero parsed entries also triggers the fallback.+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), "");+ logs.length = 0;+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ text = stripAnsi(logs.join("\n"));+ expect(text).toContain("daemon.sock.98ed5cb2.log");+ expect(text).toContain("worker crashed: uncaught exception: Error: write EPIPE");+ });++ it("does not fall back when agent.jsonl parses but has no in-window events", async () => {+ mkdirSync(join(agentDir, "logs"), { recursive: true });+ const outOfWindow = supervisorLine(+ "2026-09-01T10:00:00.000Z",+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",+ );+ writeFileSync(join(agentDir, "logs", "agent.jsonl"), `${outOfWindow}\n`);+ writeFileSync(+ join(agentDir, "logs", "daemon.sock.98ed5cb2.log"),+ "[2026-09-10T20:23:24.945Z] supervisor: Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach\n",+ );+ await runIncident({ since: "2026-09-10T20:00", until: "2026-09-10T20:30" });+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("agent.jsonl");+ expect(text).not.toContain("daemon.sock.98ed5cb2.log");+ expect(text).toContain("(none)");+ });++ it("reports missing logs clearly", async () => {+ await runIncident({});+ const text = stripAnsi(logs.join("\n"));+ expect(text).toContain("No daemon logs found under");+ });+});++// ---------------------------------------------------------------------------+// Unit-level checks for pid attribution and anomaly computation.+// ---------------------------------------------------------------------------++describe("worker pid attribution and anomalies", () => {+ it("attributes provider failures to the worker that owns the pid", () => {+ const lines = [+ workerStartLine(+ "2026-09-10T19:42:09.064Z",+ "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock",+ 53615,+ ),+ providerFailureLine("2026-09-10T20:02:53.374Z", 53615),+ ];+ const entries = lines.map((line) => entry(line));+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ expect(events.some((item) => item.summary.includes("for worker 5b1d3aeb91ee"))).toBe(true);+ });++ it("flags long gaps between events for the same session", () => {+ const base = {+ component: "coding-agent.daemon-supervisor",+ msg: "Failed to catch up client c1 for aabbccddeeff: Error: Daemon worker socket closed",+ };+ const entries = [+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:00:00.000Z" })),+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:05:00.000Z" })),+ entry(agentLogLine({ ...base, ts: "2026-09-10T20:30:00.000Z" })),+ ];+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const anomalies = computeIncidentAnomalies(events);+ const gap = anomalies.find((item) => item.summary.includes("event gap"));+ expect(gap?.summary).toBe("session aabbccddeeff: 25m event gap (no logged events)");+ });++ it("keeps anomalies and aggregation per daemon socket instead of merging daemons", () => {+ const daemonA = { component: "coding-agent.daemon-supervisor", socketPath: "/tmp/prime-agent-501/daemon.sock" };+ const daemonB = { component: "coding-agent.daemon-supervisor", socketPath: "/tmp/other/daemon.sock" };+ const times = [+ [daemonA, "20:02"],+ [daemonA, "20:08"],+ [daemonA, "20:09"],+ [daemonA, "20:21"],+ [daemonB, "20:03"],+ [daemonB, "20:10"],+ [daemonB, "20:30"],+ ] as const;+ const text = reportFor(+ times.map(([daemon, time]) =>+ supervisorLine(+ `2026-09-10T${time}:00.000Z`,+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach",+ daemon,+ ),+ ),+ );+ expect(text).toContain("/tmp/prime-agent-501/daemon.sock: 4 command timeouts over 19m");+ expect(text).toContain("/tmp/other/daemon.sock: 3 command timeouts over 27m");+ expect(text).toContain("command attach failed: timed out waiting for worker response (x4, until 09-10 20:21:00)");+ });++ it("does not report isolated failures days apart as one burst or stall", () => {+ const burstMsg = "Supervisor command send_message failed: Error: Unknown active session: aabbccddeeff";+ const stallMsg =+ "Supervisor command attach failed: Error: Timed out waiting for daemon worker response to attach";+ const cases = [+ ["2026-09-10T20:00:00.000Z", burstMsg],+ ["2026-09-12T20:00:00.000Z", burstMsg],+ ["2026-09-14T20:00:00.000Z", burstMsg],+ ["2026-09-10T20:00:00.000Z", stallMsg],+ ["2026-09-12T20:00:00.000Z", stallMsg],+ ];+ const entries = cases.map(([ts, msg]) => entry(supervisorLine(ts, msg)));+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const anomalies = computeIncidentAnomalies(events);+ expect(anomalies.some((item) => item.summary.includes("warnings/errors"))).toBe(false);+ expect(anomalies.some((item) => item.summary.includes("command timeouts"))).toBe(false);+ });++ it("keeps a same-component worker restart within two seconds", () => {+ const socketPath = "/tmp/prime-agent-501/worker-98ed5cb228d2-5b1d3aeb91ee.sock";+ const text = reportFor([+ workerStartLine("2026-09-10T20:00:00.000Z", socketPath, 100),+ workerStartLine("2026-09-10T20:00:01.000Z", socketPath, 200),+ ]);+ expect(text).toContain("worker 5b1d3aeb91ee started (x2, until 09-10 20:00:01)");+ });++ it("attributes provider failures to the worker owning the pid at that time", () => {+ const workerA = "/tmp/prime-agent-501/worker-98ed5cb228d2-aaaaaaaaaaaa.sock";+ const workerB = "/tmp/prime-agent-501/worker-98ed5cb228d2-bbbbbbbbbbbb.sock";+ const lines = [+ workerStartLine("2026-09-10T20:00:00.000Z", workerA, 53615),+ providerFailureLine("2026-09-10T20:01:00.000Z", 53615),+ // The same pid is reused by a replacement worker an hour later.+ workerStartLine("2026-09-10T21:00:00.000Z", workerB, 53615),+ providerFailureLine("2026-09-10T21:01:00.000Z", 53615),+ ];+ const entries = lines.map((line) => entry(line));+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const summaries = events.filter((item) => item.eventClass === "provider").map((item) => item.summary);+ expect(summaries).toEqual([+ "provider stream failure (rate_limit 429) for worker aaaaaaaaaaaa",+ "provider stream failure (rate_limit 429) for worker bbbbbbbbbbbb",+ ]);+ });++ it("classifies worker events for Windows named-pipe socket paths", () => {+ const socketPath = "\\\\.\\pipe\\prime-agent-worker-98ed5cb228d2-5b1d3aeb91ee";+ const entries = [+ entry(workerStartLine("2026-09-10T20:00:00.000Z", socketPath, 53615)),+ entry(daemonLine("2026-09-10T20:23:24.945Z", socketPath, "uncaught exception: Error: write EPIPE")),+ ];+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ expect(events.map((item) => item.eventClass)).toEqual(["worker-start", "worker-crash"]);+ expect(events.every((item) => item.subject === "worker 5b1d3aeb91ee")).toBe(true);+ });++ it("keys unknown worker diagnostics as worker events", () => {+ const socketPath = "/tmp/prime-agent-501/worker-98ed5cb228d2-aaaaaaaaaaaa.sock";+ const entries = [1, 2, 3].map((index) =>+ entry(daemonLine(`2026-09-10T20:0${index}:00.000Z`, socketPath, `unrecognized worker diagnostic ${index}`)),+ );+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const anomalies = computeIncidentAnomalies(events);+ expect(anomalies.some((item) => item.summary.startsWith("worker aaaaaaaaaaaa: 3 warnings/errors over 2m"))).toBe(+ true,+ );+ expect(anomalies.some((item) => item.summary.includes(socketPath))).toBe(false);+ });++ it("does not attribute a provider failure to a worker sighted later on the pid", () => {+ const workerB = "/tmp/prime-agent-501/worker-98ed5cb228d2-bbbbbbbbbbbb.sock";+ const lines = [+ providerFailureLine("2026-09-10T10:00:00.000Z", 53615),+ // The pid's only sighting is a worker that starts ten minutes later;+ // a future sighting is never evidence of ownership at 10:00.+ workerStartLine("2026-09-10T10:10:00.000Z", workerB, 53615),+ ];+ const entries = lines.map((line) => entry(line));+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const summaries = events.filter((item) => item.eventClass === "provider").map((item) => item.summary);+ expect(summaries).toEqual(["provider stream failure (rate_limit 429) for pid 53615"]);+ });++ it("flags error bursts that are not timeouts", () => {+ const entries = [1, 2, 3].map((index) =>+ entry(+ supervisorLine(+ `2026-09-10T20:0${index}:00.000Z`,+ `Supervisor command send_message failed: Error: Unknown active session: aabbccdd${index}`,+ ),+ ),+ );+ const events = collectIncidentEvents(entries, collectWorkerPidMap(entries));+ const anomalies = computeIncidentAnomalies(events);+ expect(anomalies.some((item) => item.summary.includes("3 warnings/errors over 2m"))).toBe(true);+ });+});diff --git a/packages/coding-agent/test/public-command.test.ts b/packages/coding-agent/test/public-command.test.tsindex 03449a44c8..e08fe428a4 100644--- a/packages/coding-agent/test/public-command.test.ts+++ b/packages/coding-agent/test/public-command.test.ts@@ -8,6 +8,8 @@ const mocks = vi.hoisted(() => ({reapCalls: [] as Array<[boolean, boolean]>,shutdownCalls: [] as Array<[boolean, boolean]>,mcpCommands: [] as string[][],+ incidentCalls: [] as Array<Record<string, string | undefined>>,+ incidentWindows: [] as Array<{ sinceMs: number; untilMs: number } | undefined>,}));vi.mock("../src/cli/daemon-command.js", () => ({@@ -38,6 +40,14 @@ vi.mock("../src/core/settings-manager.js", () => ({},}));+vi.mock("../src/cli/incident.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../src/cli/incident.js")>()),+ runIncident: async (options: Record<string, string | undefined>, window?: { sinceMs: number; untilMs: number }) => {+ mocks.incidentCalls.push(options);+ mocks.incidentWindows.push(window);+ },+}));+vi.mock("../src/cli/daemon-ps.js", () => ({runPs: async (json: boolean) => {mocks.psCalls.push(json);@@ -51,6 +61,7 @@ vi.mock("../src/cli/daemon-ps.js", () => ({}));import { INTERNAL_RUNTIME_COMMAND_MARKER } from "../src/cli/args.js";+import { formatTopLevelHelp } from "../src/cli/command-registry.js";import { DAEMON_UPDATE_RESTART_COORDINATOR_FLAG } from "../src/cli/daemon-update-restart.js";import { handlePublicCommand } from "../src/cli/public-command.js";@@ -62,6 +73,8 @@ describe("public command routing", () => {mocks.reapCalls.length = 0;mocks.shutdownCalls.length = 0;mocks.mcpCommands.length = 0;+ mocks.incidentCalls.length = 0;+ mocks.incidentWindows.length = 0;process.exitCode = undefined;vi.spyOn(console, "log").mockImplementation(() => {});vi.spyOn(console, "error").mockImplementation(() => {});@@ -484,4 +497,61 @@ describe("public command routing", () => {expect(process.exitCode).toBe(1);expect(mocks.daemonCommands).toEqual([]);});++ it("routes the incident command with parsed window options", async () => {+ await expect(+ handlePublicCommand(["incident", "--since", "20:02", "--until=21:00", "--session", "abc"]),+ ).resolves.toEqual({+ handled: true,+ args: [],+ explicitAgentsView: false,+ });+ expect(mocks.incidentCalls).toEqual([{ since: "20:02", until: "21:00", session: "abc" }]);+ expect(process.exitCode).toBeUndefined();+ });++ it("resolves the incident window once and passes it to runIncident", async () => {+ vi.useFakeTimers();+ // Just before UTC midnight: a second resolution later would land on the+ // next day and render a different window for relative HH:MM bounds.+ vi.setSystemTime(new Date("2026-09-16T23:59:59.900Z"));+ try {+ await handlePublicCommand(["incident", "--since", "23:00", "--until", "23:30"]);+ expect(mocks.incidentCalls).toEqual([{ since: "23:00", until: "23:30" }]);+ expect(mocks.incidentWindows).toEqual([+ { sinceMs: Date.parse("2026-09-16T23:00:00.000Z"), untilMs: Date.parse("2026-09-16T23:30:00.000Z") },+ ]);+ } finally {+ vi.useRealTimers();+ }+ });++ it("rejects unknown incident options with usage guidance", async () => {+ await expect(handlePublicCommand(["incident", "--json"])).resolves.toMatchObject({ handled: true });+ expect(process.exitCode).toBe(1);+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining("Unknown option for incident: --json"));+ expect(mocks.incidentCalls).toEqual([]);+ });++ it("rejects an incident window where until precedes since with usage guidance", async () => {+ await expect(+ handlePublicCommand(["incident", "--since", "2026-09-10T20:30", "--until", "2026-09-10T20:00"]),+ ).resolves.toMatchObject({ handled: true });+ expect(process.exitCode).toBe(1);+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining("--until must be after --since."));+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Run "prime-agent help incident" for usage.'));+ expect(mocks.incidentCalls).toEqual([]);+ });++ it("rejects a bad incident time with usage guidance", async () => {+ await expect(handlePublicCommand(["incident", "--since", "yesterday"])).resolves.toMatchObject({ handled: true });+ expect(process.exitCode).toBe(1);+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Invalid time for --since: "yesterday"'));+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Run "prime-agent help incident" for usage.'));+ expect(mocks.incidentCalls).toEqual([]);+ });++ it("shows incident in the top-level command list", () => {+ expect(formatTopLevelHelp()).toContain("incident");+ });});