Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): make stall recovery backend-aware (#543)
Gustavo Ye648c9fd5f09c
diff --git a/src/daemon/agent-driver/src/adapters/codex/index.ts b/src/daemon/agent-driver/src/adapters/codex/index.tsindex e54d85871..d48cdfa44 100644--- a/src/daemon/agent-driver/src/adapters/codex/index.ts+++ b/src/daemon/agent-driver/src/adapters/codex/index.ts@@ -44,6 +44,12 @@ export class CodexDriver implements BackendAdapter {transport: { kind: "stdio_rpc", protocol: "codex.app-server.v1" },wakeStart: "immediate",terminalOwnership: "transport_request",+ turnSilence: {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ },} as const;private readonly eventNormalizer = new CodexEventNormalizer();diff --git a/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts b/src/daemon/agent-driver/src/adapters/codex/normalizer.test.tsindex 053c55dce..768d35bad 100644--- a/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts+++ b/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts@@ -201,13 +201,36 @@ describe("CodexEventNormalizer — turn id tracking (for turn/steer expectedTurnthreadId: "root-thread",turn: { id: "next-turn", status: "inProgress" },}))).toEqual([- { kind: "turn_owner", receipt: "codex:root-thread:next-turn" },+ {+ kind: "turn_owner",+ receipt: "codex:root-thread:next-turn",+ nativeTurnId: "next-turn",+ },{ kind: "thinking", text: "" },]);expect(n.normalizeLine(completed)).toEqual([]);expect(n.currentTurnId).toBe("next-turn");});+ it("classifies retrying stream errors as recovery and preserves nested fatal messages", () => {+ const n = new CodexEventNormalizer();+ adoptRootTurn(n, "root-thread", "root-turn");++ expect(n.normalizeLine(notify("error", {+ threadId: "root-thread",+ turnId: "root-turn",+ willRetry: true,+ error: { message: "Reconnecting... 1/5", additionalDetails: "sensitive transport detail" },+ }))).toEqual([{ kind: "runtime_recovery", stage: "retrying", source: "codex_stream" }]);++ expect(n.normalizeLine(notify("error", {+ threadId: "root-thread",+ turnId: "root-turn",+ willRetry: false,+ error: { message: "request failed" },+ }))).toEqual([{ kind: "error", message: "request failed" }]);+ });+it("clears terminal ownership when a new thread is explicitly adopted and rejects an unowned terminal", () => {const n = new CodexEventNormalizer();adoptRootTurn(n, "old-thread", "old-turn");diff --git a/src/daemon/agent-driver/src/adapters/codex/normalizer.ts b/src/daemon/agent-driver/src/adapters/codex/normalizer.tsindex ef96b1907..0803a82d6 100644--- a/src/daemon/agent-driver/src/adapters/codex/normalizer.ts+++ b/src/daemon/agent-driver/src/adapters/codex/normalizer.ts@@ -103,7 +103,11 @@ export class CodexEventNormalizer {this.turnId = params.turn.id;this.terminalTurn = null;return [- { kind: "turn_owner", receipt: this.turnReceipt(params.threadId, params.turn.id) },+ {+ kind: "turn_owner",+ receipt: this.turnReceipt(params.threadId, params.turn.id),+ nativeTurnId: params.turn.id,+ },{ kind: "thinking", text: "" },];@@ -145,7 +149,10 @@ export class CodexEventNormalizer {return [{ kind: "turn_end", sessionId: this.threadId ?? undefined, turnOwner: this.turnReceipt(params.threadId, params.turn.id) }];case "error":- return [{ kind: "error", message: params?.message ?? "Codex error" }];+ if (params?.willRetry === true) {+ return [{ kind: "runtime_recovery", stage: "retrying", source: "codex_stream" }];+ }+ return [{ kind: "error", message: params?.error?.message ?? params?.message ?? "Codex error" }];case "thread/tokenUsage/updated":case "account/rateLimits/updated":diff --git a/src/daemon/agent-driver/src/adapters/pi/index.test.ts b/src/daemon/agent-driver/src/adapters/pi/index.test.tsindex bf93e4707..768dafbc8 100644--- a/src/daemon/agent-driver/src/adapters/pi/index.test.ts+++ b/src/daemon/agent-driver/src/adapters/pi/index.test.ts@@ -50,6 +50,15 @@ function fakeDeps() {}describe("PiDriver.openLane — AGENTS.md packing", () => {+ it("maps SDK retry lifecycle into bounded recovery activity", () => {+ expect(mapPiSdkEvent({ type: "auto_retry_start" }, "sess_1", { sawTextDelta: false }))+ .toEqual([{ kind: "runtime_recovery", stage: "retrying", source: "pi_auto_retry" }]);+ expect(mapPiSdkEvent({ type: "auto_retry_end" }, "sess_1", { sawTextDelta: false }))+ .toEqual([{ kind: "runtime_recovery", stage: "recovered", source: "pi_auto_retry" }]);+ expect(PI_IGNORED_EVENT_TYPES).not.toContain("auto_retry_start");+ expect(PI_IGNORED_EVENT_TYPES).not.toContain("auto_retry_end");+ });+it("exposes SDK-only parser and encoder no-ops", () => {const driver = new PiDriver(() => fakeDeps());expect(driver.normalizeLine()).toEqual([]);diff --git a/src/daemon/agent-driver/src/adapters/pi/index.ts b/src/daemon/agent-driver/src/adapters/pi/index.tsindex ba2f44ec5..85071be52 100644--- a/src/daemon/agent-driver/src/adapters/pi/index.ts+++ b/src/daemon/agent-driver/src/adapters/pi/index.ts@@ -152,8 +152,6 @@ export const PI_IGNORED_EVENT_TYPES = ["queue_update","session_info_changed","thinking_level_changed",- "auto_retry_start",- "auto_retry_end","agent_end",] as const;@@ -176,6 +174,10 @@ export function mapPiSdkEvent(event: any, sessionId: string, state: { sawTextDel}}switch (event?.type) {+ case "auto_retry_start":+ return [{ kind: "runtime_recovery", stage: "retrying", source: "pi_auto_retry" }];+ case "auto_retry_end":+ return [{ kind: "runtime_recovery", stage: "recovered", source: "pi_auto_retry" }];case "tool_execution_start":return [{ kind: "tool_call", name: event.toolName ?? "unknown_tool", input: event.args ?? {} }];case "tool_execution_end":diff --git a/src/daemon/agent-driver/src/contract.ts b/src/daemon/agent-driver/src/contract.tsindex ad074dfd5..5c395757c 100644--- a/src/daemon/agent-driver/src/contract.ts+++ b/src/daemon/agent-driver/src/contract.ts@@ -291,6 +291,7 @@ export type CoreAgentEventPayload =readonly error: AgentDriverError;}| { readonly type: "turn_started"; readonly turnId: string; readonly commandIds: readonly string[] }+ | { readonly type: "backend_turn_started"; readonly turnId: string; readonly backendTurnId: string }| { readonly type: "thinking_delta"; readonly turnId: string; readonly text: string }| { readonly type: "text_delta"; readonly turnId: string; readonly text: string }| {@@ -325,6 +326,12 @@ export type CoreAgentEventPayload =readonly source?: string;readonly message: string;}+ | {+ readonly type: "recovery";+ readonly turnId?: string;+ readonly stage: "retrying" | "recovered";+ readonly source?: string;+ }| {readonly type: "token_usage";readonly turnId?: string;@@ -383,6 +390,13 @@ export interface AgentSessionSnapshot {| "reviewing"| "tool_wait"| "working";+ readonly turnSilence: {+ readonly nativeIdleTimeoutMs: number;+ readonly daemonGraceMs: number;+ readonly recoveryGraceMs: number;+ readonly maxRecoveryExtensions: number;+ readonly normalBudgetMs: number;+ };readonly metrics: {readonly physicalOpenCount: number;readonly turnCount: number;diff --git a/src/daemon/agent-driver/src/controller/logical-session.test.ts b/src/daemon/agent-driver/src/controller/logical-session.test.tsindex 0dc9d06ed..6fbeaa385 100644--- a/src/daemon/agent-driver/src/controller/logical-session.test.ts+++ b/src/daemon/agent-driver/src/controller/logical-session.test.ts@@ -268,6 +268,41 @@ async function take(}describe.each(["claude", "codex", "cursor", "opencode", "pi"] as const)("%s logical-session conformance", (backend) => {+ it("projects the public bounded turn-silence defaults", async () => {+ const { session } = makeSession(backend);+ expect(session.snapshot().diagnostics.turnSilence).toEqual({+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 360_000,+ });+ await session.stop({ reason: "shutdown", forceAfterMs: 10 });+ });++ it("projects adapter recovery as exact-turn public recovery events", async () => {+ const { session, driver } = makeSession(backend);+ const iterator = session.events[Symbol.asyncIterator]();+ const started = await session.start({ id: "recovery-root", kind: "user", text: "start" });+ expect(started.status).toBe("accepted");+ const turnId = started.status === "accepted" ? started.turnId : "unreachable";++ await emit(driver, { kind: "runtime_recovery", stage: "retrying", source: `${backend}.retry` });+ await emit(driver, { kind: "runtime_recovery", stage: "recovered", source: `${backend}.retry` });++ const recoveryEvents: Array<AgentEvent<BuiltinBackendSpecs, BuiltinBackendId>> = [];+ while (recoveryEvents.length < 2) {+ const next = await iterator.next();+ if (next.done) break;+ if (next.value.type === "recovery") recoveryEvents.push(next.value);+ }+ expect(recoveryEvents).toEqual([+ expect.objectContaining({ type: "recovery", turnId, stage: "retrying", source: `${backend}.retry` }),+ expect.objectContaining({ type: "recovery", turnId, stage: "recovered", source: `${backend}.retry` }),+ ]);+ await session.stop({ reason: "shutdown", forceAfterMs: 10 });+ });+it("passes the exported black-box conformance suite", async () => {const { session, driver } = makeSession(backend);await runAgentDriverConformance(async () => ({@@ -430,6 +465,35 @@ describe("runtime-lane admission state machine", () => {await session.stop({ reason: "shutdown", forceAfterMs: 10 });});+ it("publishes only bounded opaque native turn ids", async () => {+ const validLane = new ControlledRuntimeLane();+ const validSession = makeSession("claude", { lane: validLane }).session;+ const validIterator = validSession.events[Symbol.asyncIterator]();+ await validSession.start({ id: "one", kind: "user", text: "start" });+ validLane.emit({ kind: "turn_owner", receipt: "claude:test:1", nativeTurnId: "native-turn_7" });+ expect((await take(validIterator as never, 3)).map((event) => event.type)).toEqual([+ "command_accepted",+ "turn_started",+ "backend_turn_started",+ ]);+ await validSession.stop({ reason: "shutdown", forceAfterMs: 10 });++ const hostileLane = new ControlledRuntimeLane();+ const hostileSession = makeSession("claude", { lane: hostileLane }).session;+ const hostileIterator = hostileSession.events[Symbol.asyncIterator]();+ await hostileSession.start({ id: "one", kind: "user", text: "start" });+ hostileLane.emit({+ kind: "turn_owner",+ receipt: "claude:test:1",+ nativeTurnId: "OPENAI_API_KEY=native-secret /Users/private",+ });+ hostileLane.emit({ kind: "text", text: "next safe event" });+ const hostileEvents = await take(hostileIterator as never, 3);+ expect(hostileEvents.map((event) => event.type)).toEqual(["command_accepted", "turn_started", "text_delta"]);+ expect(JSON.stringify(hostileEvents)).not.toMatch(/native-secret|\/Users\/private/);+ await hostileSession.stop({ reason: "shutdown", forceAfterMs: 10 });+ });+it("ignores a forged terminal owner and completes only for the admission receipt", async () => {const lane = new ControlledRuntimeLane();const { session } = makeSession("claude", { lane });@@ -590,6 +654,13 @@ describe("logical delivery diagnostics", () => {expect(session.snapshot().diagnostics).toEqual({deliveryPhase: "idle",+ turnSilence: {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 360_000,+ },metrics: {physicalOpenCount: 0,turnCount: 0,diff --git a/src/daemon/agent-driver/src/controller/logical-session.ts b/src/daemon/agent-driver/src/controller/logical-session.tsindex a190f9a1e..61987822f 100644--- a/src/daemon/agent-driver/src/controller/logical-session.ts+++ b/src/daemon/agent-driver/src/controller/logical-session.ts@@ -27,6 +27,12 @@ import type {LaneAdmission,RuntimeLane,} from "../internal/adapter.js";+import {+ DEFAULT_DAEMON_GRACE_MS,+ DEFAULT_MAX_RECOVERY_EXTENSIONS,+ DEFAULT_NATIVE_IDLE_TIMEOUT_MS,+ DEFAULT_RECOVERY_GRACE_MS,+} from "../internal/adapter.js";import { BufferedEventQueue } from "./event-queue.js";import { writeAgentFile } from "../internal/agentFile.js";import { scrubDriverErrorMessage } from "../internal/errors.js";@@ -162,6 +168,7 @@ implements AgentSession<Specs, Id> {private readonly eventQueue: BufferedEventQueue<AgentEvent<Specs, Id>>;private readonly behavior;+ private readonly turnSilence: AgentSessionSnapshot["diagnostics"]["turnSilence"];constructor(readonly backend: Id,@@ -180,6 +187,18 @@ implements AgentSession<Specs, Id> {) {this.capabilities = capabilities;this.behavior = this.capabilities as import("../contract.js").BackendCapabilities;+ const declaredSilence = adapter.execution.turnSilence;+ const nativeIdleTimeoutMs = declaredSilence?.nativeIdleTimeoutMs ?? DEFAULT_NATIVE_IDLE_TIMEOUT_MS;+ const daemonGraceMs = declaredSilence?.daemonGraceMs ?? DEFAULT_DAEMON_GRACE_MS;+ const recoveryGraceMs = declaredSilence?.recoveryGraceMs ?? DEFAULT_RECOVERY_GRACE_MS;+ const maxRecoveryExtensions = declaredSilence?.maxRecoveryExtensions ?? DEFAULT_MAX_RECOVERY_EXTENSIONS;+ this.turnSilence = {+ nativeIdleTimeoutMs,+ daemonGraceMs,+ recoveryGraceMs,+ maxRecoveryExtensions,+ normalBudgetMs: nativeIdleTimeoutMs + daemonGraceMs,+ };this.resumeOutcome = launch.resumeSessionId ? "pending" : "not_requested";this.sessionInstanceId = host.createId();this.closed = new Promise((resolve) => { this.resolveClosed = resolve; });@@ -296,6 +315,7 @@ implements AgentSession<Specs, Id> {lastEventSequence: this.eventSequence,diagnostics: {deliveryPhase: this.deliveryPhase(),+ turnSilence: this.turnSilence,metrics: {physicalOpenCount: this.metricValue(this.physicalOpenCount),turnCount: this.metricValue(this.turnCount),@@ -617,6 +637,14 @@ implements AgentSession<Specs, Id> {return;}this.activeTurn.terminalOwner = event.receipt;+ const nativeTurnId = event.nativeTurnId?.trim();+ if (nativeTurnId && nativeTurnId.length <= 512 && /^[A-Za-z0-9._:-]+$/.test(nativeTurnId)) {+ this.emit({+ type: "backend_turn_started",+ turnId: this.activeTurn.turnId,+ backendTurnId: nativeTurnId,+ });+ }return;case "thinking":if (turnId) this.emit({ type: "thinking_delta", turnId, text: event.text });@@ -658,8 +686,19 @@ implements AgentSession<Specs, Id> {message: scrubDriverErrorMessage(event.message, "Runtime diagnostic"),});return;+ case "runtime_recovery":+ this.emit({+ type: "recovery",+ turnId,+ stage: event.stage,+ source: event.source,+ });+ return;case "runtime_metric":- if (event.name === "sse_reconnect" && event.increment === 1) this.sseReconnectCount += 1;+ if (event.name === "sse_reconnect" && event.increment === 1) {+ this.sseReconnectCount += 1;+ this.emit({ type: "recovery", turnId, stage: "retrying", source: "transport_reconnect" });+ }return;case "telemetry": {const details = jsonValue(event.attrs) as JsonObject;diff --git a/src/daemon/agent-driver/src/internal/adapter.ts b/src/daemon/agent-driver/src/internal/adapter.tsindex a9040caa3..de4593520 100644--- a/src/daemon/agent-driver/src/internal/adapter.ts+++ b/src/daemon/agent-driver/src/internal/adapter.ts@@ -35,8 +35,19 @@ export interface BackendExecution {readonly transport: AdapterTransport;readonly wakeStart: "immediate" | "deferred";readonly terminalOwnership: TerminalOwnership;+ readonly turnSilence?: {+ readonly nativeIdleTimeoutMs: number;+ readonly daemonGraceMs: number;+ readonly recoveryGraceMs: number;+ readonly maxRecoveryExtensions: number;+ };}+export const DEFAULT_NATIVE_IDLE_TIMEOUT_MS = 300_000;+export const DEFAULT_DAEMON_GRACE_MS = 60_000;+export const DEFAULT_RECOVERY_GRACE_MS = 60_000;+export const DEFAULT_MAX_RECOVERY_EXTENSIONS = 1;+export type InputMode = "busy" | "idle";export interface EncodeMessageOptions { mode?: InputMode }@@ -52,8 +63,9 @@ export type AdapterEvent =| { kind: "review_finished" }| { kind: "internal_progress"; source?: string; itemType?: string; payloadBytes?: number }| { kind: "runtime_diagnostic"; severity?: string; source?: string; message: string }+ | { kind: "runtime_recovery"; stage: "retrying" | "recovered"; source?: string }| { kind: "runtime_metric"; name: "sse_reconnect"; increment: 1 }- | { kind: "turn_owner"; receipt: string }+ | { kind: "turn_owner"; receipt: string; nativeTurnId?: string }| { kind: "turn_end"; sessionId?: string; turnOwner?: string }| { kind: "error"; message: string }| { kind: "telemetry"; name: "token_usage" | "rate_limits"; source: string; usageKind?: string; attrs: Record<string, unknown> };diff --git a/src/daemon/agent-driver/src/registry.test.ts b/src/daemon/agent-driver/src/registry.test.tsindex 876fa114b..472220852 100644--- a/src/daemon/agent-driver/src/registry.test.ts+++ b/src/daemon/agent-driver/src/registry.test.ts@@ -40,6 +40,12 @@ describe("driver.capabilities", () => {transport: { kind: "stdio_rpc", protocol: "codex.app-server.v1" },wakeStart: "immediate",terminalOwnership: "transport_request",+ turnSilence: {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ },},cursor: {lifetime: "session",@@ -155,4 +161,40 @@ describe("adapter registration runtime boundary", () => {adapter,)).toThrow("delivery conflicts with its execution lifetime");});++ it("rejects malformed turn-silence policies instead of disabling stall detection", () => {+ const adapter = {+ id: "sixth",+ instructionDelivery: { kind: "native" },+ execution: {+ lifetime: "session",+ transport: { kind: "stdio_stream", protocol: "sixth.test.v1" },+ wakeStart: "immediate",+ terminalOwnership: "vendor_message",+ turnSilence: {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ },+ },+ probe() {},+ openLane() {},+ };+ const invalidPolicies = [+ null,+ { ...adapter.execution.turnSilence, nativeIdleTimeoutMs: 0 },+ { ...adapter.execution.turnSilence, daemonGraceMs: -1 },+ { ...adapter.execution.turnSilence, recoveryGraceMs: Number.POSITIVE_INFINITY },+ { ...adapter.execution.turnSilence, maxRecoveryExtensions: 0.5 },+ { ...adapter.execution.turnSilence, nativeIdleTimeoutMs: Number.MAX_SAFE_INTEGER, daemonGraceMs: 1 },+ ];+ for (const turnSilence of invalidPolicies) {+ expect(() => assertAdapterCompatibility(+ "sixth",+ EXPECTED.claude,+ { ...adapter, execution: { ...adapter.execution, turnSilence } },+ )).toThrow("invalid turnSilence declaration");+ }+ });});diff --git a/src/daemon/agent-driver/src/registry.ts b/src/daemon/agent-driver/src/registry.tsindex fb10dc8fe..6219e94cb 100644--- a/src/daemon/agent-driver/src/registry.ts+++ b/src/daemon/agent-driver/src/registry.ts@@ -146,6 +146,23 @@ export function assertAdapterCompatibility() {throw new Error(`Adapter ${registrationId} has an invalid transport declaration`);}+ if (execution.turnSilence !== undefined) {+ const silence = execution.turnSilence as Record<string, unknown> | undefined;+ const safeInteger = (value: unknown, minimum: number): value is number =>+ typeof value === "number" && Number.isSafeInteger(value) && value >= minimum;+ if (+ !silence+ || typeof silence !== "object"+ || Array.isArray(silence)+ || !safeInteger(silence.nativeIdleTimeoutMs, 1)+ || !safeInteger(silence.daemonGraceMs, 0)+ || !safeInteger(silence.recoveryGraceMs, 0)+ || !safeInteger(silence.maxRecoveryExtensions, 0)+ || !Number.isSafeInteger(silence.nativeIdleTimeoutMs + silence.daemonGraceMs)+ ) {+ throw new Error(`Adapter ${registrationId} has an invalid turnSilence declaration`);+ }+ }const capabilities = registeredCapabilities as Record<string, unknown> | undefined;const declaredLifetime = capabilities?.sessionLifetime === "persistent" ? "session" : "turn";if (execution.lifetime !== declaredLifetime) {diff --git a/src/daemon/src/cli/daemonRunner.test.ts b/src/daemon/src/cli/daemonRunner.test.tsindex d55b5eb0b..c836a355f 100644--- a/src/daemon/src/cli/daemonRunner.test.ts+++ b/src/daemon/src/cli/daemonRunner.test.ts@@ -289,7 +289,7 @@ describe("daemon runner logger", () => {}});- it("preserves projected FSM timestamps through the runner bundle seam", async () => {+ it("exports projected native-liveness diagnostics and FSM timestamps through the runner bundle seam", async () => {resetDiagnosticHarness();vi.useFakeTimers();const listenersBefore = processListenerSnapshot();@@ -316,12 +316,19 @@ describe("daemon runner logger", () => {traceSink.write(JSON.stringify({recordKind: "fsm",agentId: "bot_1",- event: "register",+ event: "runtime_signal",status: "idle",turnActive: false,inbox: 0,lastDeliverAt: null,lastProgressAt: eventTime,+ lastNativeActivityAt: eventTime - 10,+ lastNativeActivityKind: "backend_turn_started",+ runtimePhase: "inference",+ backendTurnId: "native-turn-7",+ turnSilenceBudgetMs: 360_000,+ nativeDeadlineAt: eventTime + 360_000,+ recoveryExtensionsUsed: 1,idleSince: eventTime,resetting: false,resettingSince: null,@@ -331,6 +338,7 @@ describe("daemon runner logger", () => {nowMs: eventTime,timeIso: new Date(eventTime).toISOString(),sinceProgressMs: 0,+ sinceNativeActivityMs: 10,sinceDeliverMs: null,sinceStoppingMs: null,}));@@ -388,9 +396,17 @@ describe("daemon runner logger", () => {expect.objectContaining({recordKind: "fsm",agentId: "bot_1",- event: "register",+ event: "runtime_signal",nowMs: eventTime,timeMs: eventTime,+ lastNativeActivityAt: eventTime - 10,+ lastNativeActivityKind: "backend_turn_started",+ runtimePhase: "inference",+ backendTurnId: "native-turn-7",+ turnSilenceBudgetMs: 360_000,+ nativeDeadlineAt: eventTime + 360_000,+ recoveryExtensionsUsed: 1,+ sinceNativeActivityMs: 10,}),expect.objectContaining({recordKind: "turn_span",diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.tsindex b17ced323..afb25d074 100644--- a/src/daemon/src/daemon/createDaemon.test.ts+++ b/src/daemon/src/daemon/createDaemon.test.ts@@ -1,6 +1,6 @@import { describe, it, expect, vi, afterEach } from "vitest";import { EventEmitter } from "events";-import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import {@@ -453,9 +453,9 @@ for await (const line of createInterface({ input: process.stdin })) {messageReminderClock: {now: () => now,setTimer: (callback) => {- const timer = { callback, cancelled: false };+ const timer = { callback, cancelled: false, unref() {} };timers.push(timer);- return { unref() {} } as ReturnType<typeof setTimeout>;+ return timer as unknown as ReturnType<typeof setTimeout>;},clearTimer: (handle) => {const index = timers.findIndex((timer) => timer === (handle as unknown));@@ -474,6 +474,13 @@ for await (const line of createInterface({ input: process.stdin })) {headers: { authorization: "Bearer vch_test", "content-type": "application/json" },body: JSON.stringify({ channel: "/demo#1234/general", sentSeq, remindAfterMs }),});+ const sentFrames = () => sockets[0]!.sent.map((frame) => JSON.parse(frame) as Record<string, unknown>);+ const wakeAcked = (launchId: string) => sentFrames().some((frame) =>+ frame.type === "agent_wake_ack" && frame.launchId === launchId && frame.status === "ok");+ const stopAcked = () => sentFrames().some((frame) =>+ frame.type === "agent_stopped_ack" && frame.agentId === "bot_1" && frame.status === "ok");+ const reminderDeliveries = () => deliver.mock.calls.filter(([, message]) =>+ typeof message.id === "string" && message.id.includes(":reminder:"));try {sockets[0]!.emit("open");@@ -484,20 +491,20 @@ for await (const line of createInterface({ input: process.stdin })) {launchId: "launch_1",unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 1 },}));- await vi.waitFor(() => expect(deliver).toHaveBeenCalledTimes(1));+ await vi.waitFor(() => expect(wakeAcked("launch_1")).toBe(true));now = 2_000;expect(await (await arm(7)).json()).toEqual({ armed: true, dueAt: 62_000 });runTimer(0);- expect(deliver).toHaveBeenCalledTimes(2);- expect(deliver).toHaveBeenLastCalledWith("bot_1", expect.objectContaining({+ expect(reminderDeliveries()).toHaveLength(1);+ expect(reminderDeliveries().at(-1)).toEqual(["bot_1", expect.objectContaining({text: expect.stringContaining("/demo#1234/general#7"),- }));+ })]);await arm(8);expect(await (await arm(9, 0)).json()).toEqual({ armed: false, reason: "disabled" });runTimer(1);- expect(deliver).toHaveBeenCalledTimes(2);+ expect(reminderDeliveries()).toHaveLength(1);await arm(10);sockets[0]!.emit("message", JSON.stringify({@@ -507,9 +514,10 @@ for await (const line of createInterface({ input: process.stdin })) {launchId: "launch_2",unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 11 },}));- await new Promise((resolve) => setTimeout(resolve, 0));+ await vi.waitFor(() => expect(wakeAcked("launch_2")).toBe(true));+ await vi.waitFor(() => expect(timers[2]?.cancelled).toBe(true));runTimer(2);- expect(deliver).toHaveBeenCalledTimes(2);+ expect(reminderDeliveries()).toHaveLength(1);await arm(12);sockets[0]!.emit("message", JSON.stringify({@@ -519,27 +527,32 @@ for await (const line of createInterface({ input: process.stdin })) {launchId: "launch_duplicate",unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 11 },}));- await new Promise((resolve) => setTimeout(resolve, 0));+ await vi.waitFor(() => expect(wakeAcked("launch_duplicate")).toBe(true));+ expect(timers[3]?.cancelled).toBe(false);runTimer(3);- expect(deliver).toHaveBeenCalledTimes(3);- expect(deliver).toHaveBeenLastCalledWith("bot_1", expect.objectContaining({+ expect(reminderDeliveries()).toHaveLength(2);+ expect(reminderDeliveries().at(-1)).toEqual(["bot_1", expect.objectContaining({text: expect.stringContaining("/demo#1234/general#12"),- }));+ })]);await arm(13);sockets[0]!.emit("message", JSON.stringify({ type: "agent:stop", agentId: "bot_1" }));+ await vi.waitFor(() => expect(stopAcked()).toBe(true));+ await vi.waitFor(() => expect(timers[4]?.cancelled).toBe(true));runTimer(4);- expect(deliver).toHaveBeenCalledTimes(3);+ expect(reminderDeliveries()).toHaveLength(2);await arm(14);sockets[0]!.emit("message", JSON.stringify({ type: "bot:removed", botId: "bot_1" }));+ await vi.waitFor(() => expect(timers[5]?.cancelled).toBe(true));runTimer(5);- expect(deliver).toHaveBeenCalledTimes(3);+ expect(reminderDeliveries()).toHaveLength(2);await arm(15);await daemon.stop();+ expect(timers[6]?.cancelled).toBe(true);runTimer(6);- expect(deliver).toHaveBeenCalledTimes(3);+ expect(reminderDeliveries()).toHaveLength(2);} finally {// stop is idempotent enough for the failure path and keeps the loopback// server from leaking if an assertion above throws early.@@ -550,6 +563,11 @@ for await (const line of createInterface({ input: process.stdin })) {it("resets self-sleep only for newer wakes and suspends it while an agent works", async () => {const sockets: FakeSocket[] = [];const sessions: DaemonFakeSession[] = [];+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "daemon-self-sleep-"));+ startupSweepDirs.push(workingDirectoryBase);+ // The real driver SDK prepares the agent directory before session_started.+ // This test uses a fake session, so reproduce that precondition explicitly.+ mkdirSync(join(workingDirectoryBase, "bot_1"));const timers: Array<{callback: () => void;delayMs: number;@@ -590,6 +608,7 @@ for await (const line of createInterface({ input: process.stdin })) {return session;},capabilities: [],+ workingDirectoryBase,onSelfSleep,selfSleepClock: clock,});@@ -600,6 +619,13 @@ for await (const line of createInterface({ input: process.stdin })) {launchId: `launch_${seq}`,unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: seq },}));+ const sentFrames = () => sockets[0]!.sent.map((frame) => JSON.parse(frame) as Record<string, unknown>);+ const wakeAckCount = (seq: number) => sentFrames()+ .filter((frame) => frame.type === "agent_wake_ack" && frame.launchId === `launch_${seq}` && frame.status === "ok")+ .length;+ const activityCount = (state: "running" | "idle") => sentFrames()+ .filter((frame) => frame.type === "agent_activity" && frame.agentId === "bot_1" && frame.state === state)+ .length;try {expect(timers).toHaveLength(1);@@ -607,20 +633,25 @@ for await (const line of createInterface({ input: process.stdin })) {sockets[0]!.emit("open");wake(1);await vi.waitFor(() => expect(sessions).toHaveLength(1));+ await vi.waitFor(() => expect(wakeAckCount(1)).toBe(1));+ await vi.waitFor(() => expect(activityCount("running")).toBe(1), { timeout: 5_000 });await vi.waitFor(() => expect(timers).toHaveLength(2));- expect(timers.every((timer) => timer.cancelled)).toBe(true);+ expect(timers.slice(0, 2).every((timer) => timer.cancelled)).toBe(true);await sessions[0]!.fire("runtime_event", { kind: "turn_end", sessionId: "test-session" });+ await vi.waitFor(() => expect(activityCount("idle")).toBe(1), { timeout: 5_000 });await vi.waitFor(() => expect(timers).toHaveLength(3));expect(timers[2]?.cancelled).toBe(false);wake(1);- await new Promise<void>((resolve) => setImmediate(resolve));+ await vi.waitFor(() => expect(wakeAckCount(1)).toBe(2));expect(timers).toHaveLength(3);wake(2);+ await vi.waitFor(() => expect(wakeAckCount(2)).toBe(1));await vi.waitFor(() => expect(timers.length).toBeGreaterThanOrEqual(4));expect(timers[2]?.cancelled).toBe(true);+ expect(timers[3]?.cancelled).toBe(false);timers[2]?.callback();expect(onSelfSleep).not.toHaveBeenCalled();} finally {@@ -1151,6 +1182,11 @@ describe("createDaemon — logging", () => {}) as unknown as typeof fetch;const channel = "/demo#1234/general";+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "working-unread-coverage-"));+ startupSweepDirs.push(workingDirectoryBase);+ // The real driver SDK prepares this directory before session_started. This+ // test uses a fake session, so reproduce that precondition explicitly.+ mkdirSync(join(workingDirectoryBase, "bot_working"));const starts: Array<{ id: string; text: string }> = [];const sends: Array<{ id: string; text: string; sequence?: number }> = [];const sockets: FakeSocket[] = [];@@ -1160,6 +1196,7 @@ describe("createDaemon — logging", () => {serverWsUrl: "ws://x",webSocketFactory: factory(sockets) as any,runtimeReport: [{ id: "codex" }],+ workingDirectoryBase,driverFor: () => fullFakeDriver("codex"),sessionFactory: daemonSessionFactory({onStart: (input) => starts.push(input),diff --git a/src/daemon/src/diagnostics/privacyPolicy.test.ts b/src/daemon/src/diagnostics/privacyPolicy.test.tsindex 734c874a5..92acc6f8a 100644--- a/src/daemon/src/diagnostics/privacyPolicy.test.ts+++ b/src/daemon/src/diagnostics/privacyPolicy.test.ts@@ -49,11 +49,12 @@ const EXPECTED_MANAGER_EVENTS = ["begin_reset","rewake_after_reset","runtime_signal",+ "stall_control_failed","admission_started","admission_settled",] as const;-const EXPECTED_MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "force_exit", "gated_hold"] as const;+const EXPECTED_MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "clear_stall_recovery", "force_exit", "gated_hold"] as const;const EXPECTED_AGENT_STATUSES = ["idle", "starting", "running", "stopping"] as const;const EXPECTED_DERIVED_ACTIVITIES = ["idle", "starting", "running", "stopping"] as const;const EXPECTED_TURN_OUTCOMES = ["clean", "errored"] as const;@@ -77,6 +78,18 @@ const EXPECTED_ABORT_CAUSES = ["force_exit","other",] as const;+const EXPECTED_NATIVE_ACTIVITY_KINDS = [+ "turn_started",+ "backend_turn_started",+ "thinking",+ "text",+ "tool_call",+ "tool_output",+ "internal_progress",+ "recovery",+ "turn_end",+] as const;+const EXPECTED_RUNTIME_PHASES = ["idle", "admission", "inference", "tool", "recovery", "terminal"] as const;const EXPECTED_LOG_LEVELS = ["debug", "info", "warn", "error"] as const;const EXPECTED_EXIT_SIGNALS = ["SIGABRT",@@ -137,6 +150,13 @@ const FSM_FIELD_RULES: Readonly<Record<string, ProjectionFieldRule>> = {inbox: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },lastDeliverAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },lastProgressAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },+ lastNativeActivityAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },+ lastNativeActivityKind: { type: "enum", values: EXPECTED_NATIVE_ACTIVITY_KINDS, nullable: true, optional: true },+ runtimePhase: { type: "enum", values: EXPECTED_RUNTIME_PHASES, optional: true },+ backendTurnId: { type: "string", maxChars: 512, nullable: true, optional: true },+ turnSilenceBudgetMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },+ nativeDeadlineAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true, optional: true },+ recoveryExtensionsUsed: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },idleSince: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },resetting: { type: "boolean" },resettingSince: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },@@ -155,6 +175,7 @@ const FSM_FIELD_RULES: Readonly<Record<string, ProjectionFieldRule>> = {nowMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },timeIso: { type: "timestamp" },sinceProgressMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },+ sinceNativeActivityMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },sinceDeliverMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },sinceStoppingMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },traceTurnId: { type: "string", maxChars: 1_024, optional: true },@@ -375,6 +396,13 @@ function baseFsm(overrides: Record<string, unknown> = {}): Record<string, unknowinbox: 2,lastDeliverAt: 100,lastProgressAt: 110,+ lastNativeActivityAt: 115,+ lastNativeActivityKind: "thinking",+ runtimePhase: "inference",+ backendTurnId: "native-turn-7",+ turnSilenceBudgetMs: 360_000,+ nativeDeadlineAt: 475_000,+ recoveryExtensionsUsed: 0,idleSince: null,resetting: false,resettingSince: null,@@ -384,6 +412,7 @@ function baseFsm(overrides: Record<string, unknown> = {}): Record<string, unknownowMs: 120,timeIso: "1970-01-01T00:00:00.120Z",sinceProgressMs: 10,+ sinceNativeActivityMs: 5,sinceDeliverMs: 20,sinceStoppingMs: null,traceTurnId: "launch_1:1",@@ -518,6 +547,13 @@ describe("B2c privacy policy", () => {inbox: 2,lastDeliverAt: 100,lastProgressAt: 110,+ lastNativeActivityAt: 115,+ lastNativeActivityKind: "thinking",+ runtimePhase: "inference",+ backendTurnId: "native-turn-7",+ turnSilenceBudgetMs: 360_000,+ nativeDeadlineAt: 475_000,+ recoveryExtensionsUsed: 0,idleSince: null,resetting: false,resettingSince: null,@@ -527,6 +563,7 @@ describe("B2c privacy policy", () => {nowMs: 120,timeIso: "1970-01-01T00:00:00.120Z",sinceProgressMs: 10,+ sinceNativeActivityMs: 5,sinceDeliverMs: 20,sinceStoppingMs: null,traceTurnId: "launch_1:1",@@ -537,6 +574,12 @@ describe("B2c privacy policy", () => {});expect(JSON.stringify(projected)).not.toMatch(/PROMPT_LEAK|SEND_LEAK|RESPONSE_LEAK|THINKING_LEAK|TOOL_LEAK|ERROR_LEAK|RECENT_LEAK|UNKNOWN_LEAK/);+ const credentialBearingNativeId = api.projectFsmTraceRow(baseFsm({+ backendTurnId: "OPENAI_API_KEY=native-turn-secret /Users/private/native-turn",+ }), "target-agent");+ expect(credentialBearingNativeId).not.toBeNull();+ expect(JSON.stringify(credentialBearingNativeId)).not.toMatch(/native-turn-secret|\/Users\/private/);+for (const deliveryPhase of EXPECTED_DELIVERY_PHASES) {expect(api.projectFsmTraceRow(baseFsm({ deliveryPhase }), "target-agent"), deliveryPhase).toMatchObject({ deliveryPhase });}diff --git a/src/daemon/src/diagnostics/privacyPolicy.ts b/src/daemon/src/diagnostics/privacyPolicy.tsindex 454d8cbc2..4ba27e1d1 100644--- a/src/daemon/src/diagnostics/privacyPolicy.ts+++ b/src/daemon/src/diagnostics/privacyPolicy.ts@@ -164,8 +164,8 @@ export function projectDaemonLogRow(value: unknown, targetAgentId: string): Reco};}-const MANAGER_EVENTS = ["register", "wake", "spawned", "session", "root_work", "turn_end", "exit", "tick", "reset_session", "begin_reset", "rewake_after_reset", "runtime_signal", "admission_started", "admission_settled"] as const;-const MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "force_exit", "gated_hold"] as const;+const MANAGER_EVENTS = ["register", "wake", "spawned", "session", "root_work", "turn_end", "exit", "tick", "reset_session", "begin_reset", "rewake_after_reset", "runtime_signal", "stall_control_failed", "admission_started", "admission_settled"] as const;+const MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "clear_stall_recovery", "force_exit", "gated_hold"] as const;const AGENT_STATUSES = ["idle", "starting", "running", "stopping"] as const;const DELIVERY_PHASES = ["idle", "admission_wait", "steering", "next_turn_queued", "compacting", "reviewing", "tool_wait", "working"] as const;const RESUME_OUTCOMES = ["not_requested", "pending", "resumed", "reset_required", "failed"] as const;@@ -174,6 +174,8 @@ const TERMINATION_CAUSES = ["runtime_error", "killed_stalled", "other"] as constconst SPAWN_FAILURE_REASONS = ["ENOENT", "handshake_timeout", "pre_handshake_exit", "spawn_threw", "other"] as const;const TERMINATION_SEMANTICS = ["killed_stalled", "idle_stop", "force_exit", "other"] as const;const ABORT_CAUSES = ["start_threw", "start_rejected", "send_threw", "spawn_failure", "handshake_timeout", "reset", "nap", "model_switch", "requested_stop", "shutdown", "physical_exit", "terminate_stalled", "force_exit", "other"] as const;+const NATIVE_ACTIVITY_KINDS = ["turn_started", "backend_turn_started", "thinking", "text", "tool_call", "tool_output", "internal_progress", "recovery", "turn_end"] as const;+const RUNTIME_PHASES = ["idle", "admission", "inference", "tool", "recovery", "terminal"] as const;const EXIT_SIGNALS = new Set(["SIGABRT", "SIGALRM", "SIGBREAK", "SIGBUS", "SIGCHLD", "SIGCONT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINFO", "SIGINT", "SIGIO", "SIGIOT", "SIGKILL", "SIGLOST", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", "SIGQUIT", "SIGSEGV", "SIGSTKFLT", "SIGSTOP", "SIGSYS", "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ", "other"]);function enumValue(value: unknown, values: readonly string[], bucket = false, maxChars = 64): string | null {@@ -204,6 +206,12 @@ function copyString(row: Record<string, unknown>, output: Record<string, unknownreturn true;}+function copyScrubbedString(row: Record<string, unknown>, output: Record<string, unknown>, key: string, maxChars: number, optional = false, nullable = false): boolean {+ if (!copyString(row, output, key, maxChars, optional, nullable)) return false;+ if (typeof output[key] === "string") output[key] = scrubDiagnosticText(output[key] as string);+ return true;+}+function projectTraceBase(row: Record<string, unknown>, targetAgentId: string): Record<string, unknown> | null {if (row.agentId !== targetAgentId || !boundedString(row.agentId, 128) || !canonicalTime(row.timeIso)) return null;const output: Record<string, unknown> = { recordType: "fsm", recordKind: row.recordKind, agentId: row.agentId };@@ -233,11 +241,30 @@ function projectFsm(row: Record<string, unknown>, targetAgentId: string): Record|| !copyInteger(row, output, "lastDeliverAt", { nullable: true })|| !copyInteger(row, output, "lastProgressAt")|| !copyInteger(row, output, "idleSince", { nullable: true })) return null;+ if (!copyInteger(row, output, "lastNativeActivityAt", { optional: true })+ || !copyScrubbedString(row, output, "backendTurnId", 512, true, true)+ || !copyInteger(row, output, "turnSilenceBudgetMs", { optional: true })+ || !copyInteger(row, output, "nativeDeadlineAt", { optional: true, nullable: true })+ || !copyInteger(row, output, "recoveryExtensionsUsed", { optional: true })) return null;+ if ("lastNativeActivityKind" in row && row.lastNativeActivityKind !== undefined) {+ if (row.lastNativeActivityKind === null) output.lastNativeActivityKind = null;+ else {+ const kind = enumValue(row.lastNativeActivityKind, NATIVE_ACTIVITY_KINDS);+ if (!kind) return null;+ output.lastNativeActivityKind = kind;+ }+ }+ if ("runtimePhase" in row && row.runtimePhase !== undefined) {+ const phase = enumValue(row.runtimePhase, RUNTIME_PHASES);+ if (!phase) return null;+ output.runtimePhase = phase;+ }output.resetting = row.resetting;if (!copyInteger(row, output, "resettingSince", { nullable: true })|| !copyInteger(row, output, "stoppingSince", { nullable: true })) return null;output.deliveryPhase = deliveryPhase;if (!copyInteger(row, output, "sinceProgressMs")+ || !copyInteger(row, output, "sinceNativeActivityMs", { optional: true })|| !copyInteger(row, output, "sinceDeliverMs", { nullable: true })|| !copyInteger(row, output, "sinceStoppingMs", { nullable: true })|| !projectOptionalSpanMetadata(row, output)) return null;diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.tsindex c2d47a2aa..2842809b5 100644--- a/src/daemon/src/manager/managerPolicy.test.ts+++ b/src/daemon/src/manager/managerPolicy.test.ts@@ -6,6 +6,7 @@ import {type ManagerState,type AgentState,type AgentStatus,+ type TurnSilencePolicy,} from "./managerPolicy.js";interface LegacyCaps {@@ -54,6 +55,30 @@ function spawnRoot(state: ManagerState, nowMs: number, turnId = "turn-a"): Managreturn reduceManager(next, { type: "spawned", agentId: "a", nowMs }).state;}+function spawnRootWithSilence(+ state: ManagerState,+ nowMs: number,+ turnSilence: TurnSilencePolicy,+ turnId = "turn-a",+): ManagerState {+ let next = reduceManager(state, {+ type: "attach_session",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ nowMs,+ turnSilence,+ }).state;+ next = reduceManager(next, {+ type: "turn_started",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ turnId,+ commandIds: [],+ nowMs,+ }).state;+ return reduceManager(next, { type: "spawned", agentId: "a", nowMs }).state;+}+function completeRoot(state: ManagerState, turnId: string, nowMs: number) {return reduceManager(state, {type: "turn_completed",@@ -232,6 +257,161 @@ describe("reduceManager — turn_end behavior", () => {});describe("reduceManager — tick: stall + idle hibernation", () => {+ it("allows one same-session stall recovery, then forgets that backend session on a repeat stall", () => {+ let s = createInitialManagerState(100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { id: "m1", text: "m1" }, nowMs: 0 }).state;+ s = spawnRoot(s, 0, "turn-1");+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;++ const firstStall = reduceManager(s, { type: "tick", nowMs: 100 });+ expect(firstStall.effects).toEqual([{+ type: "terminate_stalled",+ agentId: "a",+ recordSessionId: "sess-poison",+ }]);+ expect(firstStall.state.agents.a).toMatchObject({+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ });++ s = reduceManager(firstStall.state, {+ type: "turn_completed",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ turnId: "turn-1",+ nowMs: 101,+ endReason: "errored",+ terminationCause: "killed_stalled",+ }).state;+ expect(s.agents.a.stalledSessionId).toBe("sess-poison");+ s = reduceManager(s, { type: "wake", agentId: "a", message: { id: "m2", text: "m2" }, nowMs: 102 }).state;+ const respawn = reduceManager(s, { type: "exit", agentId: "a" });+ expect(respawn.effects).toContainEqual({+ type: "spawn",+ agentId: "a",+ messages: [{ id: "m2", text: "m2" }],+ resumeSessionId: "sess-poison",+ });++ s = reduceManager(respawn.state, {+ type: "attach_session",+ agentId: "a",+ sessionInstanceId: "session-instance-2",+ nowMs: 103,+ }).state;+ s = reduceManager(s, {+ type: "turn_started",+ agentId: "a",+ sessionInstanceId: "session-instance-2",+ turnId: "turn-2",+ commandIds: [],+ nowMs: 103,+ }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 103 }).state;+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;++ const secondStall = reduceManager(s, { type: "tick", nowMs: 203 });+ expect(secondStall.effects).toEqual([{+ type: "terminate_stalled",+ agentId: "a",+ forgetSessionId: "sess-poison",+ }]);+ expect(secondStall.state.agents.a.sessionId).toBeNull();+ expect(secondStall.state.agents.a.stalledSessionId).toBeNull();++ s = reduceManager(secondStall.state, {+ type: "wake",+ agentId: "a",+ message: { id: "m3", text: "m3" },+ nowMs: 204,+ }).state;+ expect(reduceManager(s, { type: "exit", agentId: "a" }).effects).toContainEqual({+ type: "spawn",+ agentId: "a",+ messages: [{ id: "m3", text: "m3" }],+ resumeSessionId: null,+ });+ });++ it("resets the stall breaker after a clean completion or a different backend session", () => {+ let s = createInitialManagerState(100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = spawnRoot(s, 0, "turn-1");+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-1" }).state;+ s = reduceManager(s, { type: "tick", nowMs: 100 }).state;+ expect(s.agents.a.stalledSessionId).toBe("sess-1");++ const clean = reduceManager(s, {+ type: "turn_completed",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ turnId: "turn-1",+ nowMs: 101,+ });+ expect(clean.effects).toEqual([{+ type: "clear_stall_recovery",+ agentId: "a",+ sessionId: "sess-1",+ }]);+ s = clean.state;+ expect(s.agents.a.stalledSessionId).toBeNull();++ s = { ...s, agents: { ...s.agents, a: { ...s.agents.a, stalledSessionId: "sess-1" } } };+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-2" }).state;+ expect(s.agents.a.stalledSessionId).toBeNull();+ expect(s.agents.a.sessionId).toBe("sess-2");+ });++ it("rolls back attempt/fence transitions and keeps a failed clear fail-closed", () => {+ let s = createInitialManagerState(100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = spawnRoot(s, 0, "turn-1");+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;++ const first = reduceManager(s, { type: "tick", nowMs: 100 }).state;+ s = reduceManager(first, {+ type: "stall_control_failed",+ agentId: "a",+ sessionId: "sess-poison",+ transition: "attempt",+ }).state;+ expect(s.agents.a).toMatchObject({+ status: "running",+ sessionId: "sess-poison",+ stalledSessionId: null,+ stoppingSince: null,+ });++ s = { ...s, agents: { ...s.agents, a: { ...s.agents.a, stalledSessionId: "sess-poison" } } };+ const repeated = reduceManager(s, { type: "tick", nowMs: 101 }).state;+ expect(repeated.agents.a).toMatchObject({ status: "stopping", sessionId: null, stalledSessionId: null });+ s = reduceManager(repeated, {+ type: "stall_control_failed",+ agentId: "a",+ sessionId: "sess-poison",+ transition: "fence",+ }).state;+ expect(s.agents.a).toMatchObject({+ status: "running",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ stoppingSince: null,+ });++ const completed = completeRoot(s, "turn-1", 102).state;+ expect(completed.agents.a.stalledSessionId).toBeNull();+ s = reduceManager(completed, {+ type: "stall_control_failed",+ agentId: "a",+ sessionId: "sess-poison",+ transition: "clear",+ }).state;+ expect(s.agents.a.stalledSessionId).toBe("sess-poison");+ });+it("terminates a stalled per-turn agent past the stale threshold", () => {let s = createInitialManagerState(100); // staleThresholdMs = 100s = register(s, "a", PER_TURN);@@ -251,6 +431,91 @@ describe("reduceManager — tick: stall + idle hibernation", () => {expect(reduceManager(s, { type: "tick", nowMs: 50 }).effects).toEqual([]);});+ it("keeps a high-reasoning turn alive through 112s of semantic silence without consuming the breaker", () => {+ const silence: TurnSilencePolicy = {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 360_000,+ };+ let s = register(createInitialManagerState(), "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "reason" }, nowMs: 0 }).state;+ s = spawnRootWithSilence(s, 0, silence, "reasoning-turn");+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-reasoning" }).state;++ expect(reduceManager(s, { type: "tick", nowMs: 112_000 }).effects).toEqual([]);+ s = reduceManager(s, {+ type: "runtime_signal",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ turnId: "reasoning-turn",+ kind: "thinking",+ phase: "inference",+ nowMs: 112_000,+ }).state;++ expect(s.agents.a).toMatchObject({+ status: "running",+ stalledSessionId: null,+ lastProgressAt: 0,+ lastNativeActivityAt: 112_000,+ lastNativeActivityKind: "thinking",+ runtimePhase: "inference",+ });+ expect(s.agents.a.execution.lease).toMatchObject({+ nativeDeadlineAt: 472_000,+ recoveryExtensionsUsed: 0,+ });+ expect(reduceManager(s, { type: "tick", nowMs: 471_999 }).effects).toEqual([]);+ expect(reduceManager(s, { type: "tick", nowMs: 472_000 }).effects).toEqual([{+ type: "terminate_stalled",+ agentId: "a",+ recordSessionId: "sess-reasoning",+ }]);+ });++ it("bounds recovery extensions and replenishes one only after semantic progress", () => {+ const silence: TurnSilencePolicy = {+ nativeIdleTimeoutMs: 100,+ daemonGraceMs: 20,+ recoveryGraceMs: 50,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 120,+ };+ let s = register(createInitialManagerState(), "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "retry" }, nowMs: 0 }).state;+ s = spawnRootWithSilence(s, 0, silence, "retry-turn");++ const recovery = (state: ManagerState, nowMs: number): ManagerState => reduceManager(state, {+ type: "runtime_signal",+ agentId: "a",+ sessionInstanceId: SESSION_INSTANCE,+ turnId: "retry-turn",+ kind: "recovery",+ phase: "recovery",+ nowMs,+ }).state;++ s = recovery(s, 110);+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 160, recoveryExtensionsUsed: 1 });+ s = recovery(s, 150);+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 160, recoveryExtensionsUsed: 1 });+ expect(reduceManager(s, { type: "tick", nowMs: 159 }).effects).toEqual([]);++ s = workRoot(s, "retry-turn", 159);+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 279, recoveryExtensionsUsed: 0 });+ s = recovery(s, 270);+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 320, recoveryExtensionsUsed: 1 });+ s = recovery(s, 310);+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 320, recoveryExtensionsUsed: 1 });+ expect(reduceManager(s, { type: "tick", nowMs: 319 }).effects).toEqual([]);+ expect(reduceManager(s, { type: "tick", nowMs: 320 }).effects).toEqual([{+ type: "terminate_stalled",+ agentId: "a",+ }]);+ });+it("fences tool blockers to the current root, clears them on reset, and restores a full stale window", () => {let s = createInitialManagerState(100);s = register(s, "a", PERSISTENT_GATED);@@ -293,6 +558,8 @@ describe("reduceManager — tick: stall + idle hibernation", () => {state: "active",identity: { sessionInstanceId: SESSION_INSTANCE, turnId: "root-turn" },lastWorkAt: 10,+ nativeDeadlineAt: 110,+ recoveryExtensionsUsed: 0,});const resetting = s;s = reduceManager(s, {@@ -331,6 +598,8 @@ describe("reduceManager — tick: stall + idle hibernation", () => {state: "active",identity: { sessionInstanceId: SESSION_INSTANCE, turnId: "root-turn" },lastWorkAt: 60,+ nativeDeadlineAt: 160,+ recoveryExtensionsUsed: 0,});expect(reduceManager(finished, { type: "tick", nowMs: 159 }).effects).toEqual([]);expect(reduceManager(finished, { type: "tick", nowMs: 160 }).effects).toEqual([diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.tsindex 159eaa5bf..c4f69cfe0 100644--- a/src/daemon/src/manager/managerPolicy.ts+++ b/src/daemon/src/manager/managerPolicy.ts@@ -17,6 +17,27 @@ export interface RootTerminal {at: number;}+export type NativeActivityKind =+ | "turn_started"+ | "backend_turn_started"+ | "thinking"+ | "text"+ | "tool_call"+ | "tool_output"+ | "internal_progress"+ | "recovery"+ | "turn_end";++export type RuntimePhase = "idle" | "admission" | "inference" | "tool" | "recovery" | "terminal";++export interface TurnSilencePolicy {+ nativeIdleTimeoutMs: number;+ daemonGraceMs: number;+ recoveryGraceMs: number;+ maxRecoveryExtensions: number;+ normalBudgetMs: number;+}+export interface PendingAdmission {sessionInstanceId: string;commandId: string;@@ -30,11 +51,20 @@ export interface PendingAdmission {export type RootLease =| { state: "detached" }| { state: "none"; lastTerminal: RootTerminal | null }- | { state: "active"; identity: TurnIdentity; lastWorkAt: number; outstandingToolUses?: number }+ | {+ state: "active";+ identity: TurnIdentity;+ lastWorkAt: number;+ nativeDeadlineAt: number;+ recoveryExtensionsUsed: number;+ outstandingToolUses?: number;+ }| {state: "suspect_active";identity: TurnIdentity;lastWorkAt: number;+ nativeDeadlineAt: number;+ recoveryExtensionsUsed: number;outstandingToolUses?: number;reason: "work_after_terminal";};@@ -48,11 +78,18 @@ export interface AgentState {status: AgentStatus;inbox: AgentMsg[];sessionId: string | null;+ /** Backend session that has already consumed its one same-session stall recovery. */+ stalledSessionId: string | null;execution: ExecutionEpoch;pendingAdmissions: PendingAdmission[];turnId: string | null;turnActive: boolean;lastProgressAt: number;+ lastNativeActivityAt: number;+ lastNativeActivityKind: NativeActivityKind | null;+ runtimePhase: RuntimePhase;+ backendTurnId: string | null;+ turnSilence: TurnSilencePolicy;lastDeliverAt: number | null;idleSince: number | null;stoppingSince: number | null;@@ -77,8 +114,14 @@ export type ManagerEvent =| { type: "register"; agentId: string }| { type: "wake"; agentId: string; message: AgentMsg; nowMs: number }| { type: "spawned"; agentId: string; nowMs: number }- | { type: "backend_session"; agentId: string; sessionId: string }- | { type: "attach_session"; agentId: string; sessionInstanceId: string; nowMs: number }+ | { type: "backend_session"; agentId: string; sessionId: string; stalledBefore?: boolean }+ | {+ type: "attach_session";+ agentId: string;+ sessionInstanceId: string;+ nowMs: number;+ turnSilence?: TurnSilencePolicy;+ }| {type: "admission_started";agentId: string;@@ -137,7 +180,23 @@ export type ManagerEvent =| { type: "reset_session"; agentId: string }| { type: "begin_reset"; agentId: string; nowMs: number }| { type: "rewake_after_reset"; agentId: string; message: AgentMsg }- | { type: "runtime_signal"; agentId: string; kind: string; nowMs: number }+ | {+ type: "runtime_signal";+ agentId: string;+ sessionInstanceId: string;+ turnId: string;+ kind: NativeActivityKind;+ phase: RuntimePhase;+ nowMs: number;+ backendTurnId?: string;+ recoveryStage?: "retrying" | "recovered";+ }+ | {+ type: "stall_control_failed";+ agentId: string;+ sessionId: string;+ transition: "attempt" | "fence" | "clear";+ }| {type: "delivery_rejected";agentId: string;@@ -149,12 +208,20 @@ export type ManagerEffect =| { type: "spawn"; agentId: string; messages: AgentMsg[]; resumeSessionId: string | null }| { type: "send"; agentId: string; message: AgentMsg; mode: "busy" | "idle" }| { type: "stop"; agentId: string; reason: string }- | { type: "terminate_stalled"; agentId: string }+ | { type: "terminate_stalled"; agentId: string; recordSessionId?: string; forgetSessionId?: string }+ | { type: "clear_stall_recovery"; agentId: string; sessionId: string }| { type: "expire_admission"; agentId: string; sessionInstanceId: string; commandIds: string[] }| { type: "requeue_delivery"; agentId: string; message: AgentMsg; mode: "busy" | "idle" }| { type: "force_exit"; agentId: string; reason: string };export const DEFAULT_STALE_THRESHOLD_MS = 120_000;+export const DEFAULT_TURN_SILENCE_POLICY: TurnSilencePolicy = {+ nativeIdleTimeoutMs: 300_000,+ daemonGraceMs: 60_000,+ recoveryGraceMs: 60_000,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 360_000,+};export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;export const DEFAULT_STOPPING_STUCK_THRESHOLD_MS = 30_000;@@ -189,6 +256,11 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRcase "backend_session":return mutate(state, event.agentId, (a) => {+ if (event.stalledBefore) {+ a.stalledSessionId = event.sessionId;+ } else if (a.stalledSessionId !== null && a.stalledSessionId !== event.sessionId) {+ a.stalledSessionId = null;+ }a.sessionId = event.sessionId;});@@ -201,7 +273,17 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR{const a = agent;a.execution = { sessionInstanceId: event.sessionInstanceId, lease: { state: "none", lastTerminal: null } };+ a.turnSilence = event.turnSilence ?? {+ ...DEFAULT_TURN_SILENCE_POLICY,+ nativeIdleTimeoutMs: state.staleThresholdMs,+ daemonGraceMs: 0,+ normalBudgetMs: state.staleThresholdMs,+ };a.lastProgressAt = event.nowMs;+ a.lastNativeActivityAt = event.nowMs;+ a.lastNativeActivityKind = null;+ a.runtimePhase = "admission";+ a.backendTurnId = null;a.idleSince = null;syncExecutionProjection(a);}@@ -264,6 +346,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRif (!state.agents[event.agentId]) return { state, effects: [] };return mutate(state, event.agentId, (a) => {a.sessionId = null;+ a.stalledSessionId = null;});case "begin_reset":@@ -302,8 +385,18 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRif ((lease.state === "active" || lease.state === "suspect_active") && !sameIdentity(lease.identity, identity)) return;const startedCommands = new Set(event.commandIds);a.pendingAdmissions = a.pendingAdmissions.filter((entry) => !startedCommands.has(entry.commandId));- a.execution.lease = { state: "active", identity, lastWorkAt: event.nowMs };+ a.execution.lease = {+ state: "active",+ identity,+ lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,+ };a.lastProgressAt = event.nowMs;+ a.lastNativeActivityAt = event.nowMs;+ a.lastNativeActivityKind = "turn_started";+ a.runtimePhase = "inference";+ a.backendTurnId = null;a.idleSince = null;syncExecutionProjection(a);});@@ -325,7 +418,12 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRconst lease = a.execution.lease;const identity = identityOf(event);if ((lease.state === "active" || lease.state === "suspect_active") && sameIdentity(lease.identity, identity)) {- a.execution.lease = { ...lease, lastWorkAt: event.nowMs };+ a.execution.lease = {+ ...lease,+ lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,+ };} else {const terminal = lease.state === "none" ? lease.lastTerminal : null;if (!terminal || !sameIdentity(terminal.identity, identity)) return;@@ -333,6 +431,8 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRstate: "suspect_active",identity,lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,reason: "work_after_terminal",};}@@ -349,7 +449,14 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRreturn onTurnToolLifecycle(state, event, "finished");case "turn_completed":- return onTurnCompleted(state, event.agentId, event.sessionInstanceId, event.nowMs, event.turnId);+ return onTurnCompleted(+ state,+ event.agentId,+ event.sessionInstanceId,+ event.nowMs,+ event.turnId,+ event.endReason,+ );case "session_closed":if (state.agents[event.agentId]?.execution.sessionInstanceId !== event.sessionInstanceId) {@@ -371,8 +478,55 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRcase "tick":return onTick(state, event.nowMs);- case "runtime_signal":- return { state, effects: [] };+ case "runtime_signal": {+ const existing = state.agents[event.agentId];+ if (!existing || existing.execution.sessionInstanceId !== event.sessionInstanceId) return { state, effects: [] };+ const lease = existing.execution.lease;+ const identity = { sessionInstanceId: event.sessionInstanceId, turnId: event.turnId };+ if ((lease.state !== "active" && lease.state !== "suspect_active") || !sameIdentity(lease.identity, identity)) {+ return { state, effects: [] };+ }+ return mutate(state, event.agentId, (a) => {+ const active = a.execution.lease;+ if ((active.state !== "active" && active.state !== "suspect_active") || !sameIdentity(active.identity, identity)) return;+ if (event.kind === "recovery" && event.recoveryStage !== "recovered") {+ if (active.recoveryExtensionsUsed < a.turnSilence.maxRecoveryExtensions) {+ a.execution.lease = {+ ...active,+ nativeDeadlineAt: Math.max(+ active.nativeDeadlineAt,+ event.nowMs + a.turnSilence.recoveryGraceMs,+ ),+ recoveryExtensionsUsed: active.recoveryExtensionsUsed + 1,+ };+ }+ } else {+ a.execution.lease = {+ ...active,+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,+ };+ }+ a.lastNativeActivityAt = event.nowMs;+ a.lastNativeActivityKind = event.kind;+ a.runtimePhase = event.phase;+ if (event.backendTurnId) a.backendTurnId = event.backendTurnId;+ });+ }++ case "stall_control_failed":+ return mutate(state, event.agentId, (a) => {+ if (event.transition === "clear") {+ if (a.sessionId === event.sessionId) a.stalledSessionId = event.sessionId;+ return;+ }+ if (a.status !== "stopping") return;+ const lease = a.execution.lease;+ if (lease.state !== "active" && lease.state !== "suspect_active") return;+ a.status = "running";+ a.stoppingSince = null;+ a.sessionId = event.sessionId;+ a.stalledSessionId = event.transition === "fence" ? event.sessionId : null;+ });case "delivery_rejected":return mutate(state, event.agentId, (a) => {@@ -424,6 +578,7 @@ function onTurnCompleted(sessionInstanceId: string,nowMs: number,turnId: string,+ endReason: "errored" | undefined,): ReduceResult {const existing = state.agents[agentId];if (!existing) return { state, effects: [] };@@ -436,14 +591,25 @@ function onTurnCompleted(const lastTerminal = { identity, at: nowMs };agent.execution.lease = { state: "none", lastTerminal };agent.lastProgressAt = nowMs;+ agent.lastNativeActivityAt = nowMs;+ agent.lastNativeActivityKind = "turn_end";+ agent.runtimePhase = "terminal";+ const clearedStallSessionId = endReason === undefined ? agent.stalledSessionId : null;+ if (clearedStallSessionId !== null) agent.stalledSessionId = null;syncExecutionProjection(agent);+ const clearEffects: ManagerEffect[] = clearedStallSessionId === null+ ? []+ : [{ type: "clear_stall_recovery", agentId, sessionId: clearedStallSessionId }];if (agent.inbox.length > 0) {const messages = drainInbox(agent);- return commit(state, agent, messages.map((queued) => ({ type: "send", agentId, message: queued, mode: "idle" })));+ return commit(state, agent, [+ ...clearEffects,+ ...messages.map((queued) => ({ type: "send" as const, agentId, message: queued, mode: "idle" as const })),+ ]);}agent.idleSince = nowMs;- return commit(state, agent, []);+ return commit(state, agent, clearEffects);}function onTurnToolLifecycle(@@ -475,11 +641,22 @@ function onTurnToolLifecycle(if ((lease.state === "active" || lease.state === "suspect_active") && sameIdentity(lease.identity, identity)) {const outstandingToolUses = (lease.outstandingToolUses ?? 0) + (lifecycle === "started" ? 1 : -1);if (outstandingToolUses > 0) {- agent.execution.lease = { ...lease, lastWorkAt: event.nowMs, outstandingToolUses };+ agent.execution.lease = {+ ...lease,+ lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,+ outstandingToolUses,+ };} else {const unblockedLease = { ...lease };delete unblockedLease.outstandingToolUses;- agent.execution.lease = { ...unblockedLease, lastWorkAt: event.nowMs };+ agent.execution.lease = {+ ...unblockedLease,+ lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,+ };}} else {const terminal = lease.state === "none" ? lease.lastTerminal : null;@@ -488,11 +665,16 @@ function onTurnToolLifecycle(state: "suspect_active",identity,lastWorkAt: event.nowMs,+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,+ recoveryExtensionsUsed: 0,outstandingToolUses: 1,reason: "work_after_terminal",};}agent.lastProgressAt = event.nowMs;+ agent.lastNativeActivityAt = event.nowMs;+ agent.lastNativeActivityKind = lifecycle === "started" ? "tool_call" : "tool_output";+ agent.runtimePhase = lifecycle === "started" ? "tool" : "inference";agent.idleSince = null;syncExecutionProjection(agent);});@@ -505,6 +687,8 @@ function onExit(state: ManagerState, agentId: string): ReduceResult {const effects = recoveryEffects(agent, agent.pendingAdmissions);agent.pendingAdmissions = [];agent.execution = { sessionInstanceId: null, lease: { state: "detached" } };+ agent.runtimePhase = "idle";+ agent.backendTurnId = null;agent.stoppingSince = null;syncExecutionProjection(agent);@@ -530,10 +714,24 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {const stalled = a.status === "running"&& (lease.state === "active" || lease.state === "suspect_active")&& (lease.outstandingToolUses ?? 0) === 0- && nowMs - lease.lastWorkAt >= state.staleThresholdMs;+ && nowMs - lease.lastWorkAt >= a.turnSilence.normalBudgetMs+ && nowMs >= lease.nativeDeadlineAt;if (stalled) {- agents[id] = { ...a, status: "stopping", idleSince: null, stoppingSince: nowMs };- effects.push({ type: "terminate_stalled", agentId: id });+ const repeatedSessionStall = a.sessionId !== null && a.stalledSessionId === a.sessionId;+ const forgetSessionId = repeatedSessionStall ? a.sessionId! : undefined;+ agents[id] = {+ ...a,+ status: "stopping",+ sessionId: repeatedSessionStall ? null : a.sessionId,+ stalledSessionId: repeatedSessionStall ? null : a.sessionId,+ idleSince: null,+ stoppingSince: nowMs,+ };+ effects.push(forgetSessionId+ ? { type: "terminate_stalled", agentId: id, forgetSessionId }+ : a.sessionId !== null+ ? { type: "terminate_stalled", agentId: id, recordSessionId: a.sessionId }+ : { type: "terminate_stalled", agentId: id });continue;}@@ -600,11 +798,17 @@ function freshAgent(agentId: string): AgentState {status: "idle",inbox: [],sessionId: null,+ stalledSessionId: null,execution: { sessionInstanceId: null, lease: { state: "detached" } },pendingAdmissions: [],turnId: null,turnActive: false,lastProgressAt: 0,+ lastNativeActivityAt: 0,+ lastNativeActivityKind: null,+ runtimePhase: "idle",+ backendTurnId: null,+ turnSilence: DEFAULT_TURN_SILENCE_POLICY,lastDeliverAt: null,idleSince: null,stoppingSince: null,diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.tsindex af3b2cb4c..76e2ca21a 100644--- a/src/daemon/src/manager/managerRuntime.test.ts+++ b/src/daemon/src/manager/managerRuntime.test.ts@@ -2,6 +2,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";import { EventEmitter } from "events";import type { ChildProcess } from "child_process";import { PassThrough } from "stream";+import * as fs from "node:fs";+import * as os from "node:os";+import * as path from "node:path";import {AgentProcessManager,truncateThinking,@@ -24,6 +27,7 @@ import type { AgentBackend as Driver } from "../drivers/index.js";import type { HostLaunchContext as LaunchContext } from "./hostContext.js";import type { RuntimeConfig } from "../runtimeConfig.js";import type { Logger } from "../logger.js";+import { createTimelineRecorder } from "../timeline/recorder.js";/** Stub logger — records calls per level for assertions. */function stubLogger(): Logger & { calls: Record<"debug" | "info" | "warn" | "error", Array<[string, unknown[]]>> } {@@ -299,6 +303,681 @@ function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoreturn { mgr, session, onRuntimeSpawnFailed, onRuntimeSessionEstablished };}+describe("AgentProcessManager — repeated backend-session stall recovery", () => {+ it("rejects a backend session before publishing it when setSession cannot persist control", async () => {+ const session = fakeSession("set-session-control-failure");+ session.stop = vi.fn(session.stop.bind(session));+ const onAgentSession = vi.fn();+ const onRuntimeSessionEstablished = vi.fn();+ const onRuntimeSpawnFailed = vi.fn();+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: {+ setSession: () => false,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => null,+ recordSessionStall: () => true,+ clearSessionStall: () => true,+ forgetSession: () => true,+ },+ onAgentSession,+ onRuntimeSessionEstablished,+ onRuntimeSpawnFailed,+ });+ manager.register("a1");+ manager.deliver("a1", { id: "set-failure", text: "hello" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-uncommitted" });++ expect(session.stop).toHaveBeenCalledWith({ reason: "shutdown", forceAfterMs: 2_000 });+ expect(onAgentSession).not.toHaveBeenCalled();+ expect(onRuntimeSessionEstablished).not.toHaveBeenCalled();+ expect(onRuntimeSpawnFailed).toHaveBeenCalledWith("codex", "resume_control_update_failed");+ expect(manager.snapshot().agents.a1.sessionId).toBeNull();+ });++ it("defers a first stall termination when the durable attempt transition fails", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const session = fakeSession("attempt-control-failure");+ session.stop = vi.fn(session.stop.bind(session));+ const onBotAuditEvent = vi.fn();+ let controlWritable = false;+ const recordSessionStall = vi.fn(() => controlWritable);+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: {+ setSession: () => true,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => null,+ recordSessionStall,+ clearSessionStall: () => true,+ forgetSession: () => true,+ },+ onBotAuditEvent,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ manager.start();+ manager.register("a1");+ manager.deliver("a1", { id: "attempt-failure", text: "hello" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });++ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(session.stop).not.toHaveBeenCalled();+ expect(manager.snapshot().agents.a1).toMatchObject({+ status: "running",+ sessionId: "sess-poison",+ stalledSessionId: null,+ stoppingSince: null,+ });+ expect(onBotAuditEvent).toHaveBeenCalledWith(+ "a1",+ expect.objectContaining({+ kind: "error",+ payload: expect.objectContaining({ code: "resume_control_update_failed" }),+ }),+ expect.anything(),+ );++ controlWritable = true;+ now = 105;+ await vi.advanceTimersByTimeAsync(5);+ expect(recordSessionStall).toHaveBeenCalledTimes(2);+ expect(session.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });+ expect(manager.snapshot().agents.a1).toMatchObject({+ status: "stopping",+ stalledSessionId: "sess-poison",+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("defers a repeated stall termination and restores the exact session when fencing fails", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const session = fakeSession("fence-control-failure");+ session.stop = vi.fn(session.stop.bind(session));+ let controlWritable = false;+ const forgetSession = vi.fn(() => controlWritable);+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: {+ setSession: () => true,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => "sess-poison",+ resolveResumeSession: () => ({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ }),+ recordSessionStall: () => true,+ clearSessionStall: () => true,+ forgetSession,+ },+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ manager.start();+ manager.register("a1");+ manager.deliver("a1", { id: "fence-failure", text: "hello" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });++ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(session.stop).not.toHaveBeenCalled();+ expect(manager.snapshot().agents.a1).toMatchObject({+ status: "running",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ stoppingSince: null,+ });++ controlWritable = true;+ now = 105;+ await vi.advanceTimersByTimeAsync(5);+ expect(forgetSession).toHaveBeenCalledTimes(2);+ expect(session.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });+ expect(manager.snapshot().agents.a1).toMatchObject({+ status: "stopping",+ sessionId: null,+ stalledSessionId: null,+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("keeps the allowance consumed when its durable clear transition fails", async () => {+ const session = fakeSession("clear-control-failure");+ const clearSessionStall = vi.fn(() => false);+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: {+ setSession: () => true,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => "sess-poison",+ resolveResumeSession: () => ({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ }),+ recordSessionStall: () => true,+ clearSessionStall,+ forgetSession: () => true,+ },+ });+ manager.register("a1");+ manager.deliver("a1", { id: "clear-failure", text: "hello" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });+ await session.fire("runtime_event", { kind: "turn_end", sessionId: "sess-poison" });++ expect(clearSessionStall).toHaveBeenCalledWith("a1", "sess-poison");+ expect(manager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");+ });++ for (const barrierType of ["reset_session", "nap"] as const) {+ it(`aborts ${barrierType} before spawn/stop when its control transition fails`, async () => {+ const sessionFactory = vi.fn(() => fakeSession());+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory,+ timeline: {+ setSession: () => true,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => null,+ recordSessionStall: () => true,+ clearSessionStall: () => true,+ forgetSession: () => false,+ },+ });+ await expect(manager.resetSession("a1", {+ runtimeConfig: {+ version: 1,+ runtime: "codex",+ model: { kind: "default" },+ mode: { kind: "default" },+ },+ launchId: "control-failure",+ rewakePrompt: "rewake",+ barrierType,+ })).rejects.toThrow("resume control could not be persisted");+ expect(sessionFactory).not.toHaveBeenCalled();+ expect(manager.snapshot().agents.a1).toBeUndefined();+ });+ }++ it("allows a genuinely different server session after an exact-session fence", async () => {+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-new-session-"));+ try {+ const writer = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ });+ writer.forgetSession("a1", "stall_recovery", "sess-poison");+ const reader = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ });+ let launch: LaunchContext | undefined;+ const session = fakeSession("different-session");+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: { sessionId: "sess-poison" } as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => {+ launch = hooks.ctx;+ return bindFactorySession(hooks, session);+ },+ timeline: reader,+ });+ manager.register("a1", { sessionId: "sess-fresh" });+ manager.deliver("a1", { id: "fresh-candidate", seq: 1, text: "fresh" });+ expect(launch?.config.sessionId).toBe("sess-fresh");+ await manager.stopAll();+ } finally {+ fs.rmSync(timelineDir, { recursive: true, force: true });+ }+ });++ it("rejects a stale fenced server candidate after a newer healthy timeline session", async () => {+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-stale-after-fresh-"));+ try {+ const writer = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ });+ writer.forgetSession("a1", "stall_recovery", "sess-poison");+ writer.setSession("a1", "sess-healthy");+ writer.appendEntryForAgent("a1", [{+ seq: "#1",+ channel: "/test/general",+ sender: "@tester#0001",+ content: { text: "healthy" },+ time: new Date().toISOString(),+ }]);+ const reader = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ });+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-healthy",+ stalledSessionId: null,+ fencedSessionId: "sess-poison",+ });++ let launch: LaunchContext | undefined;+ const session = fakeSession("fresh-after-stale");+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: { sessionId: "sess-poison" } as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => {+ launch = hooks.ctx;+ return bindFactorySession(hooks, session);+ },+ timeline: reader,+ });+ manager.register("a1", { sessionId: "sess-poison" });+ manager.deliver("a1", { id: "stale-candidate", seq: 2, text: "must fresh-start" });+ expect(launch?.config.sessionId).toBeUndefined();+ await manager.stopAll();+ } finally {+ fs.rmSync(timelineDir, { recursive: true, force: true });+ }+ });++ it("a durable clean marker replenishes the allowance after manager reconstruction", async () => {+ vi.useFakeTimers();+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-clean-restart-"));+ try {+ let now = 0;+ const writer = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ now: () => new Date(now),+ });+ writer.setSession("a1", "sess-healthy");+ writer.appendEntryForAgent("a1", [{+ seq: "#1",+ channel: "/test/general",+ sender: "@tester#0001",+ content: { text: "healthy" },+ time: new Date(now).toISOString(),+ }]);+ writer.recordSessionStall("a1", "sess-healthy");+ writer.clearSessionStall("a1", "sess-healthy");++ const reader = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ now: () => new Date(now),+ });+ const session = fakeSession("clean-restart");+ session.stop = vi.fn(session.stop.bind(session));+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: reader,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ manager.start();+ manager.register("a1", { sessionId: "sess-healthy" });+ manager.deliver("a1", { id: "healthy-retry", seq: 2, text: "next" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-healthy" });+ expect(manager.snapshot().agents.a1.stalledSessionId).toBeNull();++ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(manager.snapshot().agents.a1.sessionId).toBe("sess-healthy");+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-healthy",+ stalledSessionId: "sess-healthy",+ fencedSessionId: null,+ });+ await manager.stopAll();+ } finally {+ fs.rmSync(timelineDir, { recursive: true, force: true });+ vi.useRealTimers();+ }+ });++ it("restores a marker-only allowance against a stale server/base session candidate", async () => {+ vi.useFakeTimers();+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-marker-only-"));+ try {+ let now = 0;+ const writer = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ now: () => new Date(now),+ });+ writer.recordSessionStall("a1", "sess-poison");++ // Rebuild the recorder with no in-memory map and no ordinary session row.+ const reader = createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ now: () => new Date(now),+ });+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({+ kind: "none",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });+ const session = fakeSession("marker-only-restart");+ session.stop = vi.fn(session.stop.bind(session));+ const manager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: { sessionId: "sess-poison" } as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ timeline: reader,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ manager.start();+ manager.register("a1", { sessionId: "sess-poison" });+ manager.deliver("a1", { id: "marker-only-wake", seq: 1, text: "resume" });+ session.startResolver?.();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });+ expect(manager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");++ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(manager.snapshot().agents.a1.sessionId).toBeNull();+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({+ kind: "barrier",+ type: "stall_recovery",+ forgottenSessionId: "sess-poison",+ fencedSessionId: "sess-poison",+ });+ await manager.stopAll();+ } finally {+ fs.rmSync(timelineDir, { recursive: true, force: true });+ vi.useRealTimers();+ }+ });++ it("restores the consumed allowance after rebuilding both manager and timeline recorder", async () => {+ vi.useFakeTimers();+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-restart-"));+ try {+ let now = 0;+ const makeRecorder = () => createTimelineRecorder({+ timelineDirFor: () => timelineDir,+ providerFor: () => "codex",+ now: () => new Date(now),+ });+ const firstSession = fakeSession("before-daemon-restart");+ firstSession.stop = vi.fn(firstSession.stop.bind(firstSession));+ const firstRecorder = makeRecorder();+ const firstManager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, firstSession),+ timeline: firstRecorder,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ firstManager.start();+ firstManager.register("a1", { sessionId: "sess-poison" });+ firstManager.deliver("a1", { id: "wake-before-restart", seq: 1, text: "first" });+ firstSession.startResolver?.();+ await firstSession.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });+ firstRecorder.appendEntryForAgent("a1", [{+ seq: "#1",+ channel: "/test/general",+ sender: "@tester#0001",+ content: { text: "first" },+ time: new Date(now).toISOString(),+ }]);+ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(firstRecorder.resolveResumeSession("a1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });+ await firstManager.stopAll();++ // New objects model a real daemon process restart: no AgentState or+ // recorder map survives; only the local timeline directory is reused.+ const secondSession = fakeSession("after-daemon-restart");+ secondSession.stop = vi.fn(secondSession.stop.bind(secondSession));+ const secondRecorder = makeRecorder();+ const secondManager = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: { sessionId: "sess-poison" } as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, secondSession),+ timeline: secondRecorder,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ secondManager.start();+ secondManager.register("a1", { sessionId: "sess-poison" });+ secondManager.deliver("a1", { id: "wake-after-restart", seq: 2, text: "second" });+ secondSession.startResolver?.();+ await secondSession.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });+ expect(secondManager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");++ now = 200;+ await vi.advanceTimersByTimeAsync(5);+ expect(secondSession.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });+ expect(secondManager.snapshot().agents.a1.sessionId).toBeNull();+ expect(secondRecorder.resolveResumeSession("a1", "codex")).toEqual({+ kind: "barrier",+ type: "stall_recovery",+ forgottenSessionId: "sess-poison",+ fencedSessionId: "sess-poison",+ });+ await secondManager.stopAll();+ } finally {+ fs.rmSync(timelineDir, { recursive: true, force: true });+ vi.useRealTimers();+ }+ });++ it("resumes once, then durably fences the poisoned session across every fallback source", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const sessions: FakeSession[] = [];+ const launches: LaunchContext[] = [];+ let resolution = {+ kind: "session" as const,+ sessionId: "sess-poison",+ stalledSessionId: null,+ fencedSessionId: null,+ } as+ | {+ kind: "session";+ sessionId: string;+ stalledSessionId: string | null;+ fencedSessionId: string | null;+ }+ | {+ kind: "barrier";+ type: "stall_recovery";+ forgottenSessionId: string | null;+ fencedSessionId: string | null;+ };+ const setSession = vi.fn();+ const recordSessionStall = vi.fn((_agentId: string, sessionId: string) => {+ resolution = { kind: "session", sessionId, stalledSessionId: sessionId, fencedSessionId: null };+ });+ const forgetSession = vi.fn((_agentId: string, type?: string, forgottenSessionId?: string) => {+ resolution = {+ kind: "barrier",+ type: "stall_recovery",+ forgottenSessionId: forgottenSessionId ?? null,+ fencedSessionId: forgottenSessionId ?? null,+ };+ });+ const mgr = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: { sessionId: "sess-poison" } as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => {+ launches.push(hooks.ctx);+ const session = fakeSession(`instance-${sessions.length + 1}`);+ session.stop = vi.fn(session.stop.bind(session));+ sessions.push(session);+ return bindFactorySession(hooks, session);+ },+ timeline: {+ setSession,+ appendResponseToLatest: vi.fn(),+ resumeSessionId: () => resolution.kind === "session" ? resolution.sessionId : null,+ resolveResumeSession: () => resolution,+ recordSessionStall,+ clearSessionStall: vi.fn(),+ forgetSession,+ },+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ });+ mgr.start();+ mgr.register("a1", { sessionId: "sess-poison" });+ mgr.deliver("a1", { id: "wake-1", seq: 1, text: "first" });+ sessions[0]!.startResolver?.();+ await sessions[0]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });++ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(sessions[0]!.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });+ expect(recordSessionStall).toHaveBeenCalledWith("a1", "sess-poison");+ expect(forgetSession).not.toHaveBeenCalled();++ await sessions[0]!.fire("runtime_event", { kind: "turn_end" });+ mgr.deliver("a1", { id: "wake-2", seq: 2, text: "second" });+ await sessions[0]!.fire("exit", { code: 0, reason: "requested" });+ expect(launches[1]!.config.sessionId).toBe("sess-poison");+ sessions[1]!.startResolver?.();+ await sessions[1]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });++ now = 200;+ await vi.advanceTimersByTimeAsync(5);+ expect(forgetSession).toHaveBeenCalledWith("a1", "stall_recovery", "sess-poison");+ expect(mgr.snapshot().agents.a1.sessionId).toBeNull();++ // A death-rattle session event from the fenced owner must not repopulate+ // either the runtime cache or timeline after the barrier was written.+ await sessions[1]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });+ expect(setSession).toHaveBeenCalledTimes(2);++ // Model the next server wake still carrying its stale session id. The+ // durable barrier must also beat register(), timeline, and base config.+ mgr.register("a1", { sessionId: "sess-poison" });+ mgr.deliver("a1", { id: "wake-3", seq: 3, text: "third" });+ await sessions[1]!.fire("exit", { code: 0, reason: "requested" });+ expect(launches[2]!.config.sessionId).toBeUndefined();+ expect(sessions).toHaveLength(3);+ } finally {+ vi.useRealTimers();+ }+ });+});+describe("AgentProcessManager — runtime health callbacks", () => {it("ENOENT `error` followed by `exit` reports the failure ONCE with the specific code", async () => {const { mgr, session, onRuntimeSpawnFailed } = makeManager();@@ -751,6 +1430,159 @@ describe("AgentProcessManager — session race conditions", () => {}});+ it("does not let turn-scoped telemetry or diagnostics refresh the native silence deadline", async () => {+ vi.useFakeTimers();+ try {+ let currentTime = 0;+ const { mgr, session } = makeManager({ now: () => currentTime, tickIntervalMs: 5 });+ const baseSnapshot = session.snapshot.bind(session);+ session.snapshot = () => ({+ ...baseSnapshot(),+ diagnostics: {+ deliveryPhase: "working",+ turnSilence: {+ nativeIdleTimeoutMs: 80,+ daemonGraceMs: 20,+ recoveryGraceMs: 50,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 100,+ },+ metrics: {+ physicalOpenCount: 1,+ turnCount: 1,+ commandAdmissionCount: 1,+ commandAdmissionLatencyTotalMs: 0,+ queueDwellCount: 0,+ queueDwellTotalMs: 0,+ sseReconnectCount: 0,+ resumeOutcome: "not_requested",+ terminalOwnerKind: "transport_request",+ },+ },+ });+ const stop = vi.spyOn(session, "stop");+ mgr.start();+ mgr.deliver("a1", { id: "telemetry-stall", seq: 1, text: "hello" });+ session.startResolver?.();+ await Promise.resolve();+ await Promise.resolve();++ currentTime = 90;+ await session.pushAgentEvent({+ type: "diagnostic",+ turnId: "test-turn",+ severity: "info",+ source: "heartbeat",+ message: "still here",+ } as never);+ await session.pushAgentEvent({+ type: "rate_limits",+ turnId: "test-turn",+ source: "account",+ details: { remaining: 1 },+ } as never);+ await session.pushAgentEvent({+ type: "token_usage",+ turnId: "test-turn",+ source: "account",+ usage: {},+ details: { sampled: true },+ } as never);++ expect(mgr.snapshot().agents.a1).toMatchObject({+ lastNativeActivityAt: 0,+ lastNativeActivityKind: "turn_started",+ });+ currentTime = 100;+ await vi.advanceTimersByTimeAsync(5);+ expect(stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });+ expect(mgr.snapshot().agents.a1.status).toBe("stopping");+ } finally {+ vi.useRealTimers();+ }+ });++ it("maps backend retry activity to one bounded recovery extension", async () => {+ let currentTime = 0;+ const { mgr, session } = makeManager({ now: () => currentTime });+ const baseSnapshot = session.snapshot.bind(session);+ session.snapshot = () => ({+ ...baseSnapshot(),+ diagnostics: {+ deliveryPhase: "working",+ turnSilence: {+ nativeIdleTimeoutMs: 80,+ daemonGraceMs: 20,+ recoveryGraceMs: 50,+ maxRecoveryExtensions: 1,+ normalBudgetMs: 100,+ },+ metrics: {+ physicalOpenCount: 1,+ turnCount: 1,+ commandAdmissionCount: 1,+ commandAdmissionLatencyTotalMs: 0,+ queueDwellCount: 0,+ queueDwellTotalMs: 0,+ sseReconnectCount: 0,+ resumeOutcome: "not_requested",+ terminalOwnerKind: "transport_request",+ },+ },+ });+ mgr.deliver("a1", { id: "retry", seq: 1, text: "hello" });+ session.startResolver?.();+ await Promise.resolve();+ await Promise.resolve();++ currentTime = 90;+ await session.pushAgentEvent({+ type: "recovery",+ turnId: "test-turn",+ stage: "retrying",+ source: "codex_stream",+ } as never);+ expect(mgr.snapshot().agents.a1).toMatchObject({+ lastNativeActivityAt: 90,+ lastNativeActivityKind: "recovery",+ runtimePhase: "recovery",+ });+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({+ nativeDeadlineAt: 140,+ recoveryExtensionsUsed: 1,+ });++ currentTime = 135;+ await session.pushAgentEvent({+ type: "recovery",+ turnId: "test-turn",+ stage: "recovered",+ source: "pi_auto_retry",+ } as never);+ expect(mgr.snapshot().agents.a1).toMatchObject({+ lastNativeActivityAt: 135,+ lastNativeActivityKind: "recovery",+ runtimePhase: "inference",+ });+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({+ nativeDeadlineAt: 235,+ recoveryExtensionsUsed: 1,+ });++ currentTime = 200;+ await session.pushAgentEvent({+ type: "recovery",+ turnId: "test-turn",+ stage: "retrying",+ source: "codex_stream",+ } as never);+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({+ nativeDeadlineAt: 235,+ recoveryExtensionsUsed: 1,+ });+ await mgr.stopAll();+ });+it("turn-correlated internal progress renews the execution lease after a false terminal", async () => {vi.useFakeTimers();try {@@ -961,6 +1793,63 @@ describe("AgentProcessManager — session race conditions", () => {});describe("AgentProcessManager — onAgentActivity (derived activity reporting)", () => {+ it.each(["turn-before-spawn", "spawn-before-turn"] as const)(+ "publishes one running edge without a transient idle when admission order is %s",+ async (order) => {+ const session = fakeSession(`activity-${order}`);+ const onAgentActivity = vi.fn();+ const mgr = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory: (hooks) => bindFactorySession(hooks, session),+ onAgentActivity,+ });+ const dispatch = (event: Record<string, unknown>) => (+ mgr as unknown as { dispatch(value: unknown): void }+ ).dispatch(event);++ mgr.register("a1");+ mgr.deliver("a1", { id: "root", text: "hello" });+ expect(onAgentActivity.mock.calls.map((call) => call[0])).toEqual([+ { agentId: "a1", state: "starting" },+ ]);++ const turnStarted = {+ type: "turn_started",+ agentId: "a1",+ sessionInstanceId: session.sessionInstanceId,+ turnId: "root-turn",+ commandIds: ["root"],+ nowMs: 1,+ };+ if (order === "turn-before-spawn") {+ dispatch(turnStarted);+ dispatch({ type: "spawned", agentId: "a1", nowMs: 2 });+ } else {+ dispatch({ type: "spawned", agentId: "a1", nowMs: 1 });+ dispatch({+ type: "admission_settled",+ agentId: "a1",+ sessionInstanceId: session.sessionInstanceId,+ commandId: "root",+ outcome: "accepted",+ });+ dispatch(turnStarted);+ }++ expect(onAgentActivity.mock.calls.map((call) => call[0])).toEqual([+ { agentId: "a1", state: "starting" },+ { agentId: "a1", state: "running" },+ ]);+ },+ );+it("fires exactly once per real derived transition — the turn_end→idle transition fires once, not re-fired while the FSM stays running until hibernation", async () => {vi.useFakeTimers();try {@@ -3434,7 +4323,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {expect(closeFailureReported).toHaveBeenCalledWith("codex", "session_closed_rejected");});- it("maps all public session event families into backend-neutral runtime signals", async () => {+ it("maps only turn activity into runtime signals and excludes telemetry/diagnostics", async () => {const rows: B1TraceRow[] = [];const session = b1Session([]);const { mgr } = b1Manager({ sessions: [session], trace: (row) => rows.push(row) });@@ -3458,6 +4347,8 @@ describe("B1 red gate — exact-once terminal matrix", () => {publish({ type: "diagnostic", severity: "warning", source: "codex", message: "warning" });publish({ type: "token_usage", turnId: "test-turn", source: "codex", usage: {}, details: {} });publish({ type: "rate_limits", turnId: "test-turn", source: "codex", details: {} });+ publish({ type: "thinking_delta", turnId: "test-turn", text: "" });+ publish({ type: "text_delta", turnId: "test-turn", text: "" });publish({ type: "command_queued", commandId: "queued", reason: "runtime_busy" });publish({ type: "command_accepted", commandId: "accepted", turnId: "test-turn", delivery: "steer" });publish({@@ -3467,7 +4358,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {});publish({ type: "turn_started", turnId: "test-turn", commandIds: ["accepted"] });- expect(rows.filter((row) => row.event === "runtime_signal")).toHaveLength(baselineSignals + 9);+ expect(rows.filter((row) => row.event === "runtime_signal")).toHaveLength(baselineSignals + 4);});it("keeps tail telemetry observational while turn-correlated work can recover a false terminal", async () => {diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex 2c76cf8b5..8b30ad2e7 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -8,6 +8,8 @@ import {type AgentMsg,type AgentState,type AgentStatus,+ type NativeActivityKind,+ type RuntimePhase,isActivelyWorking,} from "./managerPolicy.js";import type {@@ -25,6 +27,8 @@ import { scrubRuntimeErrorDiagnosticText } from "../runtime/errorDiagnostics.js"import { buildCliSystemPrompt } from "../drivers/systemPrompt.js";import { createLogger, type Logger } from "../logger.js";import { nowLocalISO } from "../util/localTime.js";+import type { ResumeSessionResolution } from "../timeline/timeline.js";+import type { SystemEntryType } from "../timeline/types.js";import { randomUUID } from "node:crypto";const SESSION_STOP_GRACE_MS = 2_000;export type AgentActivityState = "idle" | "starting" | "running" | "stopping";@@ -71,6 +75,8 @@ interface ActiveSpawnState {handshakeTimer: ReturnType<typeof setTimeout> | null;torndown: boolean;superseded: boolean;+ discardEvents: boolean;+ stalledSessionIdAtLaunch: string | null;spawnFailureReason: string | null;terminationSemantics: string | null;spawnOrdinal: number;@@ -101,6 +107,13 @@ interface FsmTraceRecord extends TraceRecordBase, Partial<TraceSpanMetadata> {inbox: number;lastDeliverAt: number | null;lastProgressAt: number;+ lastNativeActivityAt: number;+ lastNativeActivityKind: NativeActivityKind | null;+ runtimePhase: RuntimePhase;+ backendTurnId: string | null;+ turnSilenceBudgetMs: number;+ nativeDeadlineAt: number | null;+ recoveryExtensionsUsed: number;idleSince: number | null;resetting: boolean;resettingSince: number | null;@@ -116,6 +129,7 @@ interface FsmTraceRecord extends TraceRecordBase, Partial<TraceSpanMetadata> {resumeOutcome?: "not_requested" | "pending" | "resumed" | "reset_required" | "failed";terminalOwnerKind?: "transport_request" | "vendor_message" | "prompt_invocation" | "lane_generation";sinceProgressMs: number;+ sinceNativeActivityMs: number;sinceDeliverMs: number | null;sinceStoppingMs: number | null;endReason?: "errored";@@ -232,10 +246,13 @@ export interface ManagerRuntimeOpts {logger?: Logger;}export interface TimelineRecorder {- setSession(agentId: string, sessionId: string): void;+ setSession(agentId: string, sessionId: string): boolean;appendResponseToLatest(agentId: string, text: string): void;resumeSessionId(agentId: string, provider: string | null): string | null;- forgetSession(agentId: string, barrierType?: "reset_session" | "nap"): void;+ resolveResumeSession?(agentId: string, provider: string | null): ResumeSessionResolution;+ recordSessionStall?(agentId: string, sessionId: string): boolean;+ clearSessionStall?(agentId: string, sessionId: string): boolean;+ forgetSession(agentId: string, barrierType?: SystemEntryType, forgottenSessionId?: string): boolean;}const THINKING_MAX_BYTES = 4096;const AUDIT_ERROR_MESSAGE_MAX_LEN = 2000;@@ -407,6 +424,7 @@ export class AgentProcessManager {private readonly liveSessions = new Map<string, string>();private readonly thinkingBuffers = new Map<string, string>();private readonly activeSpawnState = new Map<string, ActiveSpawnState>();+ private readonly publishedAgentActivity = new Map<string, AgentActivityState>();private readonly traceProcessNonce = randomUUID();private nextSpawnOrdinal = 1;private nextDaemonTurnOrdinal = 1;@@ -492,11 +510,21 @@ export class AgentProcessManager {const effects = this.dispatch({ type: "wake", agentId, message: normalized, nowMs: this.now() });return effects.length > 0;}- forgetSession(agentId: string, barrierType: "reset_session" | "nap" = "reset_session"): void {+ forgetSession(agentId: string, barrierType: "reset_session" | "nap" = "reset_session"): boolean {+ if (!this.forgetSessionSources(agentId, barrierType)) return false;+ this.dispatch({ type: "reset_session", agentId });+ return true;+ }+ private forgetSessionSources(+ agentId: string,+ barrierType: SystemEntryType,+ forgottenSessionId?: string,+ ): boolean {+ const persisted = this.opts.timeline?.forgetSession(agentId, barrierType, forgottenSessionId);+ if (persisted === false) return false;this.resumeSessions.delete(agentId);this.liveSessions.delete(agentId);- this.dispatch({ type: "reset_session", agentId });- this.opts.timeline?.forgetSession(agentId, barrierType);+ return true;}enqueueRewake(agentId: string, message: AgentMsg): void {this.dispatch({ type: "rewake_after_reset", agentId, message });@@ -537,8 +565,12 @@ export class AgentProcessManager {opName: string;},): Promise<void> {+ if (opts.forgetSession && !this.forgetSession(agentId, opts.barrierType ?? "reset_session")) {+ this.log.error("resume control transition failed; reset aborted", { agentId, barrierType: opts.barrierType });+ this.emitErrorAudit(agentId, "reset", "resume_control_update_failed", "Reset aborted because resume control could not be persisted");+ throw new Error("Reset aborted because resume control could not be persisted");+ }this.register(agentId, { runtimeConfig: opts.runtimeConfig, launchId: opts.launchId });- if (opts.forgetSession) this.forgetSession(agentId, opts.barrierType ?? "reset_session");this.abortCurrentTurn(agentId, opts.abortCause);this.markResetting(agentId);const status = this.state.agents[agentId]?.status;@@ -753,6 +785,19 @@ export class AgentProcessManager {inbox: a.inbox.length,lastDeliverAt: a.lastDeliverAt,lastProgressAt: a.lastProgressAt,+ lastNativeActivityAt: a.lastNativeActivityAt,+ lastNativeActivityKind: a.lastNativeActivityKind,+ runtimePhase: a.runtimePhase,+ backendTurnId: a.backendTurnId,+ turnSilenceBudgetMs: a.turnSilence.normalBudgetMs,+ nativeDeadlineAt:+ a.execution.lease.state === "active" || a.execution.lease.state === "suspect_active"+ ? a.execution.lease.nativeDeadlineAt+ : null,+ recoveryExtensionsUsed:+ a.execution.lease.state === "active" || a.execution.lease.state === "suspect_active"+ ? a.execution.lease.recoveryExtensionsUsed+ : 0,idleSince: a.idleSince,resetting: a.resetting,resettingSince: a.resettingSince,@@ -763,6 +808,7 @@ export class AgentProcessManager {timeIso: new Date(nowMs).toISOString(),...(activeSpan ? activeSpan : {}),sinceProgressMs: nowMs - a.lastProgressAt,+ sinceNativeActivityMs: nowMs - a.lastNativeActivityAt,sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,sinceStoppingMs: a.stoppingSince === null ? null : nowMs - a.stoppingSince,...(event.type === "turn_completed" && (event as { endReason?: "errored" }).endReason === "errored"@@ -820,14 +866,20 @@ export class AgentProcessManager {}if (this.opts.onAgentActivity- && event.type !== "spawned"&& event.type !== "admission_started"&& event.type !== "admission_settled") {const after = this.deriveActivitySnapshot(state);for (const [agentId, activity] of Object.entries(after)) {- if (agentId in before && before[agentId] !== activity) {+ if (!(agentId in before)) {+ this.publishedAgentActivity.set(agentId, activity);+ continue;+ }+ const previouslyPublished = this.publishedAgentActivity.get(agentId) ?? before[agentId];+ const publishable = event.type !== "spawned" || activity === "running";+ if (publishable && previouslyPublished !== activity) {this.opts.onAgentActivity({ agentId, state: activity });+ this.publishedAgentActivity.set(agentId, activity);}}}@@ -1022,6 +1074,56 @@ export class AgentProcessManager {case "terminate_stalled": {const session = this.sessions.get(effect.agentId);const spawnState = this.activeSpawnState.get(effect.agentId);+ const endedSessionId = this.liveSessions.get(effect.agentId) ?? "";+ if (effect.type === "terminate_stalled" && effect.recordSessionId) {+ const persisted = this.opts.timeline?.recordSessionStall?.(effect.agentId, effect.recordSessionId);+ if (persisted === false) {+ this.dispatch({+ type: "stall_control_failed",+ agentId: effect.agentId,+ sessionId: effect.recordSessionId,+ transition: "attempt",+ });+ this.log.error("stall recovery attempt was not persisted; termination deferred", {+ agentId: effect.agentId,+ sessionId: effect.recordSessionId,+ });+ this.emitErrorAudit(+ effect.agentId,+ "runtime",+ "resume_control_update_failed",+ "Stall termination deferred because the recovery attempt could not be persisted",+ );+ break;+ }+ }+ if (effect.type === "terminate_stalled" && effect.forgetSessionId) {+ const persisted = this.forgetSessionSources(effect.agentId, "stall_recovery", effect.forgetSessionId);+ if (!persisted) {+ this.dispatch({+ type: "stall_control_failed",+ agentId: effect.agentId,+ sessionId: effect.forgetSessionId,+ transition: "fence",+ });+ this.log.error("repeated-session fence was not persisted; termination deferred", {+ agentId: effect.agentId,+ sessionId: effect.forgetSessionId,+ });+ this.emitErrorAudit(+ effect.agentId,+ "runtime",+ "resume_control_update_failed",+ "Stall termination deferred because the exact session fence could not be persisted",+ );+ break;+ }+ if (spawnState) spawnState.discardEvents = true;+ this.log.warn("repeatedly stalled backend session fenced", {+ agentId: effect.agentId,+ sessionId: effect.forgetSessionId,+ });+ }if (effect.type === "terminate_stalled" && spawnState) {this.closeTurn(spawnState, spawnState.activeSpan, {event: "turn_abort",@@ -1039,10 +1141,36 @@ export class AgentProcessManager {if (spawnState) {spawnState.terminationSemantics = effect.type === "terminate_stalled" ? "killed_stalled" : "idle_stop";}- this.logSessionEnded(effect.agentId, effect.type === "stop" ? "stopped" : "terminate_stalled");+ this.logSessionEnded(+ effect.agentId,+ effect.type === "stop" ? "stopped" : "terminate_stalled",+ endedSessionId,+ );this.opts.onAgentLocallyStopped?.({ agentId: effect.agentId, reason: effect.type });break;}+ case "clear_stall_recovery": {+ const persisted = this.opts.timeline?.clearSessionStall?.(effect.agentId, effect.sessionId);+ if (persisted === false) {+ this.dispatch({+ type: "stall_control_failed",+ agentId: effect.agentId,+ sessionId: effect.sessionId,+ transition: "clear",+ });+ this.log.error("stall recovery clear was not persisted; allowance remains consumed", {+ agentId: effect.agentId,+ sessionId: effect.sessionId,+ });+ this.emitErrorAudit(+ effect.agentId,+ "runtime",+ "resume_control_update_failed",+ "Stall recovery allowance remains consumed because its clear could not be persisted",+ );+ }+ break;+ }case "expire_admission": {const owner = this.activeSpawnState.get(effect.agentId);if (!owner || owner.sessionInstanceId !== effect.sessionInstanceId) break;@@ -1095,8 +1223,12 @@ export class AgentProcessManager {}}}- private logSessionEnded(agentId: string, reason: "turn_end" | "stopped" | "terminate_stalled" | "exit"): void {- this.log.info("agent session ended", { agentId, sessionId: this.liveSessions.get(agentId) ?? "", reason });+ private logSessionEnded(+ agentId: string,+ reason: "turn_end" | "stopped" | "terminate_stalled" | "exit",+ sessionId = this.liveSessions.get(agentId) ?? "",+ ): void {+ this.log.info("agent session ended", { agentId, sessionId, reason });}private doSpawn(agentId: string, messages: AgentMsg[], resumeSessionId: string | null): void {const [first, ...pending] = messages;@@ -1120,11 +1252,31 @@ export class AgentProcessManager {mode: { kind: "default" },} as RuntimeConfig);const provider = runtimeConfig.runtime;- const sessionId =+ const timelineResolution = this.opts.timeline?.resolveResumeSession?.(agentId, provider);+ const timelineSessionId = timelineResolution?.kind === "session"+ ? timelineResolution.sessionId+ : timelineResolution === undefined+ ? this.opts.timeline?.resumeSessionId(agentId, provider)+ : null;+ const candidateSessionId =resumeSessionId ??this.resumeSessions.get(agentId) ??- this.opts.timeline?.resumeSessionId(agentId, provider) ??+ timelineSessionId ??base.config?.sessionId;+ const blockedByBarrier = timelineResolution?.kind === "barrier"+ && (+ timelineResolution.type !== "stall_recovery"+ || timelineResolution.forgottenSessionId === null+ || timelineResolution.forgottenSessionId === candidateSessionId+ );+ const blockedByExactFence = candidateSessionId !== undefined+ && timelineResolution?.fencedSessionId === candidateSessionId;+ const sessionId = blockedByBarrier || blockedByExactFence ? undefined : candidateSessionId;+ const stalledSessionIdAtLaunch = timelineResolution !== undefined+ && (timelineResolution.kind === "session" || timelineResolution.kind === "none")+ && timelineResolution.stalledSessionId === sessionId+ ? timelineResolution.stalledSessionId+ : null;const description = runtimeConfig.instruction ?? base.config?.description ?? runtimeConfig.agentName;const agentName = runtimeConfig.agentName ?? base.config?.agentName;const agentHandle = runtimeConfig.agentHandle ?? base.config?.agentHandle;@@ -1155,6 +1307,8 @@ export class AgentProcessManager {handshakeTimer: null,torndown: false,superseded: false,+ discardEvents: false,+ stalledSessionIdAtLaunch,spawnFailureReason: null,terminationSemantics: null,spawnOrdinal: this.nextSpawnOrdinal++,@@ -1237,10 +1391,23 @@ export class AgentProcessManager {};const onEvent = (event: ManagedEvent) => {if (state.torndown) return;+ if (state.discardEvents) {+ this.log.warn("ignored event from discarded backend session owner", { agentId, event: event.type });+ return;+ }if (event.type === "session_failed" && !state.hasEstablished) {reportSpawnFailure(event.error.code || "failed_to_start", { message: event.error.message });}if (event.type === "session_started") {+ const persisted = this.opts.timeline?.setSession(agentId, event.backendSessionId);+ if (persisted === false) {+ state.discardEvents = true;+ reportSpawnFailure("resume_control_update_failed", {+ message: "Backend session rejected because resume control could not be persisted",+ });+ void state.session?.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });+ return;+ }state.hasEstablished = true;clearHandshakeTimer();this.opts.onRuntimeSessionEstablished?.(driver.id);@@ -1260,6 +1427,7 @@ export class AgentProcessManager {agentId,sessionInstanceId: session.sessionInstanceId,nowMs: this.now(),+ turnSilence: session.snapshot().diagnostics?.turnSilence,}, state);previousOwner?.pendingDeliverySpans.clear();void (async () => {@@ -1501,9 +1669,13 @@ export class AgentProcessManager {}}if (event.type === "session_started") {- this.dispatch({ type: "backend_session", agentId, sessionId: event.backendSessionId }, owner);+ this.dispatch({+ type: "backend_session",+ agentId,+ sessionId: event.backendSessionId,+ stalledBefore: owner.stalledSessionIdAtLaunch === event.backendSessionId,+ }, owner);this.liveSessions.set(agentId, event.backendSessionId);- this.opts.timeline?.setSession(agentId, event.backendSessionId);this.opts.onAgentSession?.({agentId,sessionId: event.backendSessionId,@@ -1560,26 +1732,59 @@ export class AgentProcessManager {}, owner);if (!wasActive && this.state.agents[agentId]?.turnActive && !owner.activeSpan) this.openTurn(owner);}- const signalKind = (() => {+ const nativeSignal = (() => {switch (event.type) {- case "session_started": return "session_init";- case "thinking_delta": return "thinking";- case "text_delta": return "text";- case "tool_started": return "tool_call";- case "tool_finished": return "tool_output";- case "diagnostic": return "runtime_diagnostic";- case "token_usage":- case "rate_limits": return "telemetry";- case "turn_completed": return "turn_end";- case "session_failed": return "error";- case "command_queued":- case "command_accepted":- case "command_failed":- case "turn_started": return "internal_progress";- default: return event.type;+ case "turn_started":+ return { kind: "turn_started" as const, phase: "inference" as const, turnId: event.turnId };+ case "backend_turn_started":+ return {+ kind: "backend_turn_started" as const,+ phase: "inference" as const,+ turnId: event.turnId,+ backendTurnId: event.backendTurnId,+ };+ case "thinking_delta":+ return { kind: "thinking" as const, phase: "inference" as const, turnId: event.turnId };+ case "text_delta":+ return event.text.length > 0+ ? { kind: "text" as const, phase: "inference" as const, turnId: event.turnId }+ : null;+ case "tool_started":+ return { kind: "tool_call" as const, phase: "tool" as const, turnId: event.turnId };+ case "tool_finished":+ return { kind: "tool_output" as const, phase: "inference" as const, turnId: event.turnId };+ case "compaction_started":+ case "compaction_finished":+ case "review_started":+ case "review_finished":+ case "internal_progress":+ return event.turnId+ ? { kind: "internal_progress" as const, phase: "inference" as const, turnId: event.turnId }+ : null;+ case "recovery":+ return event.turnId+ ? {+ kind: "recovery" as const,+ phase: event.stage === "retrying" ? "recovery" as const : "inference" as const,+ recoveryStage: event.stage,+ turnId: event.turnId,+ }+ : null;+ case "turn_completed":+ return { kind: "turn_end" as const, phase: "terminal" as const, turnId: event.turnId };+ default:+ return null;}})();- this.dispatch({ type: "runtime_signal", agentId, kind: signalKind, nowMs: this.now() }, owner);+ if (nativeSignal) {+ this.dispatch({+ type: "runtime_signal",+ agentId,+ sessionInstanceId: event.sessionInstanceId,+ ...nativeSignal,+ nowMs: this.now(),+ }, owner);+ }if (event.type === "turn_completed") {this.logSessionEnded(agentId, "turn_end");const marker = this.nonCleanEndMarker.get(agentId);diff --git a/src/daemon/src/manager/managerTimeline.test.ts b/src/daemon/src/manager/managerTimeline.test.tsindex 6e9f712ef..830ad4043 100644--- a/src/daemon/src/manager/managerTimeline.test.ts+++ b/src/daemon/src/manager/managerTimeline.test.ts@@ -150,7 +150,7 @@ describe("manager.resetSession", () => {mode: { kind: "default" as const },};- it("idle path: register → forgetSession → deliver rewake; spawn fires with sessionId=undefined and rewake prompt", async () => {+ it("idle path: durable forget → register → deliver rewake; spawn fires with sessionId=undefined and rewake prompt", async () => {const { rec, calls } = fakeRecorder();const cap: { ctx?: LaunchContext } = {};const { mgr } = manager(rec, cap);diff --git a/src/daemon/src/timeline/recorder.test.ts b/src/daemon/src/timeline/recorder.test.tsindex f8fd868df..006fe88e0 100644--- a/src/daemon/src/timeline/recorder.test.ts+++ b/src/daemon/src/timeline/recorder.test.ts@@ -1,5 +1,7 @@import { describe, it, expect, afterEach, vi } from "vitest";import * as fs from "fs";+import nodeFs from "node:fs";+import { syncBuiltinESMExports } from "node:module";import * as os from "os";import * as path from "path";import { createTimelineRecorder } from "./recorder";@@ -146,6 +148,276 @@ describe("createTimelineRecorder (append-only, 4-field schema)", () => {});describe("forgetSession — inline system row", () => {+ it("keeps resume resolution at a fixed open-file bound with 1,000 historical day files", () => {+ const dir = mkDir();+ const emptyTurn = `${JSON.stringify(createTimelineEntry({ messages: [] }))}\n`;+ const historicalStart = new Date("2000-01-01T12:00:00Z");+ for (let index = 0; index < 1_000; index++) {+ const day = new Date(historicalStart);+ day.setUTCDate(day.getUTCDate() + index);+ fs.writeFileSync(path.join(dir, filenameForDate(day)), emptyTurn);+ }+ for (let index = 0; index < 7; index++) {+ const day = new Date(NOW());+ day.setDate(day.getDate() - index);+ fs.writeFileSync(path.join(dir, filenameForDate(day)), emptyTurn);+ }+ fs.writeFileSync(path.join(dir, ".resume-control.json"), JSON.stringify({+ version: 1,+ attemptedSessionId: null,+ fencedSessionId: null,+ fullBarrier: null,+ }) + "\n");++ const originalOpenSync = nodeFs.openSync;+ const open = vi.fn(originalOpenSync);+ nodeFs.openSync = open as typeof nodeFs.openSync;+ syncBuiltinESMExports();+ try {+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "none",+ stalledSessionId: null,+ fencedSessionId: null,+ });+ expect(open).toHaveBeenCalledTimes(8);+ } finally {+ nodeFs.openSync = originalOpenSync;+ syncBuiltinESMExports();+ }+ });++ it("uses recent history only when control state is missing and fails closed for corrupt control inputs", () => {+ const makeSessionHistory = (): { dir: string; controlPath: string } => {+ const dir = mkDir();+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ rec.setSession("agent_1", "sess-existing");+ rec.appendEntryForAgent("agent_1", [msg("#1", "existing")]);+ return { dir, controlPath: path.join(dir, ".resume-control.json") };+ };+ const resolve = (dir: string) => createTimelineRecorder({+ timelineDirFor: () => dir,+ providerFor: () => "codex",+ now: NOW,+ }).resolveResumeSession("agent_1", "codex");++ const missing = makeSessionHistory();+ fs.unlinkSync(missing.controlPath);+ expect(resolve(missing.dir)).toEqual({+ kind: "session",+ sessionId: "sess-existing",+ stalledSessionId: null,+ fencedSessionId: null,+ });++ const corrupt = makeSessionHistory();+ fs.writeFileSync(corrupt.controlPath, "{not-json}\n");+ expect(resolve(corrupt.dir)).toEqual({+ kind: "barrier",+ type: "reset_session",+ forgottenSessionId: null,+ fencedSessionId: null,+ });++ const oversized = makeSessionHistory();+ fs.writeFileSync(oversized.controlPath, "x".repeat(4_097));+ expect(resolve(oversized.dir)).toEqual({+ kind: "barrier",+ type: "reset_session",+ forgottenSessionId: null,+ fencedSessionId: null,+ });++ const nonRegular = makeSessionHistory();+ fs.unlinkSync(nonRegular.controlPath);+ fs.mkdirSync(nonRegular.controlPath);+ expect(resolve(nonRegular.dir)).toEqual({+ kind: "barrier",+ type: "reset_session",+ forgottenSessionId: null,+ fencedSessionId: null,+ });++ if (process.platform !== "win32") {+ const linked = makeSessionHistory();+ const outside = path.join(mkDir(), "outside-control.json");+ fs.writeFileSync(outside, JSON.stringify({+ version: 1,+ attemptedSessionId: null,+ fencedSessionId: null,+ fullBarrier: null,+ }));+ fs.unlinkSync(linked.controlPath);+ fs.symlinkSync(outside, linked.controlPath);+ expect(resolve(linked.dir)).toEqual({+ kind: "barrier",+ type: "reset_session",+ forgottenSessionId: null,+ fencedSessionId: null,+ });+ }+ });++ it("persists and clears the one-stall recovery allowance", () => {+ const dir = mkDir();+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ rec.setSession("agent_1", "sess-poison");+ rec.appendEntryForAgent("agent_1", [msg("#1", "before stall")]);++ rec.recordSessionStall("agent_1", "sess-poison");+ const controlPath = path.join(dir, ".resume-control.json");+ const controlStat = fs.statSync(controlPath);+ expect(controlStat.isFile()).toBe(true);+ if (process.platform !== "win32") expect(controlStat.mode & 0o777).toBe(0o600);+ expect(fs.readdirSync(dir).filter((name) => name.includes("resume-control") && name.endsWith(".tmp"))).toEqual([]);+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });++ rec.clearSessionStall("agent_1", "sess-poison");+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: null,+ fencedSessionId: null,+ });+ });++ for (const failureMode of ["lock", "rename"] as const) {+ for (const transition of ["attempt", "fence", "reset_session", "nap", "clear"] as const) {+ it(`does not append a ${transition} forensic row when the control ${failureMode} transition fails`, () => {+ const dir = mkDir();+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ expect(rec.setSession("agent_1", "sess-poison")).toBe(true);+ rec.appendEntryForAgent("agent_1", [msg("#1", "before transition")]);+ if (transition === "fence" || transition === "clear") {+ expect(rec.recordSessionStall("agent_1", "sess-poison")).toBe(true);+ }++ const controlPath = path.join(dir, ".resume-control.json");+ const beforeControl = fs.readFileSync(controlPath, "utf8");+ const beforeSystemRows = readRecentEntries(dir, { now: NOW() })+ .filter((row) => row.system)+ .map((row) => row.system);+ const lockPath = lockPathFor(dir, ".resume-control.json");+ const originalRenameSync = nodeFs.renameSync;+ if (failureMode === "lock") expect(acquireLock(lockPath)).toBe(true);+ else {+ nodeFs.renameSync = ((oldPath: fs.PathLike, newPath: fs.PathLike) => {+ if (String(newPath) === controlPath) throw new Error("injected control rename failure");+ return originalRenameSync(oldPath, newPath);+ }) as typeof nodeFs.renameSync;+ syncBuiltinESMExports();+ }++ try {+ const result = transition === "attempt"+ ? rec.recordSessionStall("agent_1", "sess-poison")+ : transition === "fence"+ ? rec.forgetSession("agent_1", "stall_recovery", "sess-poison")+ : transition === "clear"+ ? rec.clearSessionStall("agent_1", "sess-poison")+ : rec.forgetSession("agent_1", transition);+ expect(result).toBe(false);+ } finally {+ if (failureMode === "lock") releaseLock(lockPath);+ else {+ nodeFs.renameSync = originalRenameSync;+ syncBuiltinESMExports();+ }+ }++ expect(fs.readFileSync(controlPath, "utf8")).toBe(beforeControl);+ expect(readRecentEntries(dir, { now: NOW() })+ .filter((row) => row.system)+ .map((row) => row.system)).toEqual(beforeSystemRows);+ const rebuilt = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ expect(rebuilt.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: transition === "fence" || transition === "clear" ? "sess-poison" : null,+ fencedSessionId: null,+ });+ });+ }++ it(`keeps the prior in-memory session when setSession hits a control ${failureMode} failure`, () => {+ const dir = mkDir();+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ expect(rec.setSession("agent_1", "sess-old")).toBe(true);+ const controlPath = path.join(dir, ".resume-control.json");+ const beforeControl = fs.readFileSync(controlPath, "utf8");+ const lockPath = lockPathFor(dir, ".resume-control.json");+ const originalRenameSync = nodeFs.renameSync;+ if (failureMode === "lock") expect(acquireLock(lockPath)).toBe(true);+ else {+ nodeFs.renameSync = ((oldPath: fs.PathLike, newPath: fs.PathLike) => {+ if (String(newPath) === controlPath) throw new Error("injected control rename failure");+ return originalRenameSync(oldPath, newPath);+ }) as typeof nodeFs.renameSync;+ syncBuiltinESMExports();+ }+ try {+ expect(rec.setSession("agent_1", "sess-new")).toBe(false);+ } finally {+ if (failureMode === "lock") releaseLock(lockPath);+ else {+ nodeFs.renameSync = originalRenameSync;+ syncBuiltinESMExports();+ }+ }+ expect(fs.readFileSync(controlPath, "utf8")).toBe(beforeControl);+ rec.appendEntryForAgent("agent_1", [msg("#1", "after failed set")]);+ expect(readRecentEntries(dir, { now: NOW() }).at(-1)?.session_id).toBe("sess-old");+ });+ }++ it("resolves an attempt marker outside the seven-day turn-history window", () => {+ const dir = mkDir();+ const oldRecorder = createTimelineRecorder({+ timelineDirFor: () => dir,+ providerFor: () => "codex",+ now: () => new Date("2026-06-01T12:00:00Z"),+ });+ oldRecorder.recordSessionStall("agent_1", "sess-poison");++ const rebuiltRecorder = createTimelineRecorder({+ timelineDirFor: () => dir,+ providerFor: () => "codex",+ now: () => new Date("2026-06-25T12:00:00Z"),+ });+ expect(rebuiltRecorder.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "none",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });+ });++ it("persists the exact repeatedly stalled session in a stall_recovery barrier", () => {+ const dir = mkDir();+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });+ rec.setSession("agent_1", "sess-poison");++ rec.forgetSession("agent_1", "stall_recovery", "sess-poison");++ const rows = readRecentEntries(dir, { now: NOW() });+ expect(rows).toHaveLength(1);+ expect(rows[0].system).toEqual({+ type: "stall_recovery",+ time: NOW().toISOString(),+ backend_session_id: "sess-poison",+ });+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({+ kind: "barrier",+ type: "stall_recovery",+ forgottenSessionId: "sess-poison",+ fencedSessionId: "sess-poison",+ });+ });+it("appends a bare reset_session system row (no forgot_session_id) and clears the map", () => {const dir = mkDir();const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "claude", now: NOW });diff --git a/src/daemon/src/timeline/recorder.ts b/src/daemon/src/timeline/recorder.tsindex fbefcb58f..464306cd0 100644--- a/src/daemon/src/timeline/recorder.ts+++ b/src/daemon/src/timeline/recorder.ts@@ -24,14 +24,17 @@ import {appendOrMergeEntry,updateLatestEntryResult,readRecentEntries,+ readResumeControlState,+ updateResumeControlState,createTimelineEntry,createSystemEntry,- findResumableSession,+ resolveResumableSession,appendEntry,prepareTimelineDirectory,} from "./timeline.js";import type { Message } from "../server/contract.js";import type { SystemEntryType } from "./types.js";+import type { ResumeSessionResolution } from "./timeline.js";const MAX_AGENT_RESPONSES = 5;@@ -44,24 +47,32 @@ function appendAgentResponse(entry: { agent_responses: string[] }, text: string)/** Manager/daemon-facing recorder interface (structural, avoids a cyclic import). */export interface TimelineRecorderLike {- /** CONTROL plane: remember the runtime session id (baked into new entries). */- setSession(agentId: string, sessionId: string): void;+ /** CONTROL plane: remember the runtime session id; false means no state may advance. */+ setSession(agentId: string, sessionId: string): boolean;/** DATA plane: a successful inbox pull opens a new entry of what the agent saw. */appendEntryForAgent(agentId: string, messages: Message[]): void;/** CONTROL plane: append the agent's text output to its latest open entry. */appendResponseToLatest(agentId: string, text: string): void;/** Latest session id recorded for this agent (resume target), or null. */resumeSessionId(agentId: string, provider: string | null): string | null;+ /** Latest resume target or the durable barrier that supersedes older sources. */+ resolveResumeSession(agentId: string, provider: string | null): ResumeSessionResolution;+ /** Persist the consumed allowance; false means the caller must defer termination. */+ recordSessionStall(agentId: string, sessionId: string): boolean;+ /** Persist a clean replenishment; false leaves the allowance consumed. */+ clearSessionStall(agentId: string, sessionId: string): boolean;/*** Reset: append a `system` barrier row to the timeline (`reset_session` for* an owner reset, `nap` for an agent self-reset — default `reset_session`).- * The resume walker treats either as a barrier — every turn at or before it+ * The resume walker treats reset/nap as full barriers and stall recovery as+ * an exact-session barrier — every superseded turn* becomes invisible to `resumeSessionId` — and the agent's own history read* (for recall) sees it in-line with turns. Also clears the in-memory session* cache so a racing `appendEntryForAgent` can't bake the stale id into a- * fresh row.+ * fresh row. Returns false unless the authoritative control transition was+ * committed; the forensic row is appended only after that precondition.*/- forgetSession(agentId: string, barrierType?: SystemEntryType): void;+ forgetSession(agentId: string, barrierType?: SystemEntryType, forgottenSessionId?: string): boolean;}export interface TimelineRecorderOptions {@@ -79,10 +90,59 @@ export function createTimelineRecorder(opts: TimelineRecorderOptions): TimelineR// session_init (control plane) lands before the agent's first pull (data plane)// opens an entry, so hold the latest session id here and bake it into new rows.const sessionByAgent = new Map<string, string>();+ const resolveForAgent = (agentId: string, provider: string | null): ResumeSessionResolution => {+ const dir = dirFor(agentId);+ const recent = resolveResumableSession(+ readRecentEntries(dir, { now: now() }),+ provider ?? undefined,+ );+ const control = readResumeControlState(dir);+ if (control.kind === "missing") return recent;+ if (control.kind === "invalid") {+ return {+ kind: "barrier",+ type: "reset_session",+ forgottenSessionId: null,+ fencedSessionId: null,+ };+ }+ const { attemptedSessionId, fencedSessionId, fullBarrier } = control.state;+ if (fullBarrier !== null) {+ return {+ kind: "barrier",+ type: fullBarrier,+ forgottenSessionId: null,+ fencedSessionId,+ };+ }+ if (recent.kind === "session") {+ return {+ kind: "session",+ sessionId: recent.sessionId,+ stalledSessionId: attemptedSessionId === recent.sessionId ? attemptedSessionId : null,+ fencedSessionId,+ };+ }+ if (recent.kind === "barrier") {+ if (recent.type !== "stall_recovery" || recent.forgottenSessionId === fencedSessionId) {+ return { ...recent, fencedSessionId };+ }+ }+ return { kind: "none", stalledSessionId: attemptedSessionId, fencedSessionId };+ };return {setSession(agentId, sessionId) {+ const dir = dirFor(agentId);+ if (!prepareTimelineDirectory(dir)) return false;+ const persisted = updateResumeControlState(dir, (state) => ({+ ...state,+ fullBarrier: null,+ attemptedSessionId: state.attemptedSessionId === sessionId ? state.attemptedSessionId : null,+ }));+ if (!persisted) return false;sessionByAgent.set(agentId, sessionId);+ return true;},appendEntryForAgent(agentId, messages) {if (messages.length === 0) return;@@ -120,20 +180,62 @@ export function createTimelineRecorder(opts: TimelineRecorderOptions): TimelineRappendEntry(dir, entry, now());},resumeSessionId(agentId, provider) {- const rows = readRecentEntries(dirFor(agentId), { now: now() });- return findResumableSession(rows, provider ?? undefined);+ const resolution = resolveForAgent(agentId, provider);+ return resolution.kind === "session" ? resolution.sessionId : null;+ },+ resolveResumeSession(agentId, provider) {+ return resolveForAgent(agentId, provider);+ },+ recordSessionStall(agentId, sessionId) {+ return appendStallMarker(agentId, "stall_recovery_attempt", sessionId);+ },+ clearSessionStall(agentId, sessionId) {+ return appendStallMarker(agentId, "stall_recovery_clear", sessionId);},- forgetSession(agentId, barrierType = "reset_session") {+ forgetSession(agentId, barrierType = "reset_session", forgottenSessionId) {const dir = dirFor(agentId);- // Kill happens BEFORE this call (see `AgentProcessManager.resetSession`),- // so no race — clear the in-memory session map and append the barrier.+ // Persist the authoritative transition before clearing the in-memory+ // session map or appending its best-effort forensic row.// One `now()` sample so the system-row `time` and the day-file the row// is written into can't disagree on the boundary between two consecutive// clock reads.- sessionByAgent.delete(agentId);- if (!prepareTimelineDirectory(dir)) return;+ if (!prepareTimelineDirectory(dir)) return false;const stamp = now();- appendEntry(dir, createSystemEntry(barrierType, stamp.toISOString()), stamp);+ let persisted = false;+ if (barrierType === "reset_session" || barrierType === "nap") {+ persisted = updateResumeControlState(dir, (state) => ({+ ...state,+ attemptedSessionId: null,+ fencedSessionId: null,+ fullBarrier: barrierType,+ }));+ } else if (barrierType === "stall_recovery") {+ persisted = updateResumeControlState(dir, (state) => ({+ ...state,+ attemptedSessionId:+ state.attemptedSessionId === forgottenSessionId ? null : state.attemptedSessionId,+ fencedSessionId: forgottenSessionId ?? null,+ fullBarrier: null,+ }));+ }+ if (!persisted) return false;+ sessionByAgent.delete(agentId);+ appendEntry(dir, createSystemEntry(barrierType, stamp.toISOString(), forgottenSessionId), stamp);+ return true;},};++ function appendStallMarker(agentId: string, type: SystemEntryType, sessionId: string): boolean {+ const dir = dirFor(agentId);+ if (!prepareTimelineDirectory(dir)) return false;+ const stamp = now();+ const persisted = updateResumeControlState(dir, (state) => ({+ ...state,+ attemptedSessionId: type === "stall_recovery_attempt" ? sessionId : null,+ fullBarrier: null,+ }));+ if (!persisted) return false;+ appendEntry(dir, createSystemEntry(type, stamp.toISOString(), sessionId), stamp);+ return true;+ }}diff --git a/src/daemon/src/timeline/timeline.test.ts b/src/daemon/src/timeline/timeline.test.tsindex aa2dcfe1a..afe717c56 100644--- a/src/daemon/src/timeline/timeline.test.ts+++ b/src/daemon/src/timeline/timeline.test.ts@@ -11,6 +11,7 @@ import {createTimelineEntry,createSystemEntry,findResumableSession,+ resolveResumableSession,filenameForDate,localISOString,sweepTimelineHistory,@@ -584,6 +585,74 @@ describe("findResumableSession", () => {expect(findResumableSession(rows)).toBeNull();});+ it("returns the exact backend session fenced by a stall_recovery barrier", () => {+ const rows = [+ { ...createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }) },+ createSystemEntry("stall_recovery", "2026-06-25T12:00:00Z", "sess-poison"),+ ];+ expect(resolveResumableSession(rows, "codex")).toEqual({+ kind: "barrier",+ type: "stall_recovery",+ forgottenSessionId: "sess-poison",+ fencedSessionId: "sess-poison",+ });+ expect(findResumableSession(rows, "codex")).toBeNull();+ });++ it("carries an un-cleared first-stall attempt across later rows for the same session", () => {+ const rows = [+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),+ ];+ expect(resolveResumableSession(rows, "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });+ });++ it("retains the consumed attempt even when no timeline session row remains", () => {+ const rows = [+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),+ ];+ expect(resolveResumableSession(rows, "codex")).toEqual({+ kind: "none",+ stalledSessionId: "sess-poison",+ fencedSessionId: null,+ });+ });++ it("a clean-completion marker clears the durable first-stall attempt", () => {+ const rows = [+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),+ createSystemEntry("stall_recovery_clear", "2026-06-25T12:01:00Z", "sess-poison"),+ ];+ expect(resolveResumableSession(rows, "codex")).toEqual({+ kind: "session",+ sessionId: "sess-poison",+ stalledSessionId: null,+ fencedSessionId: null,+ });+ });++ it("carries an exact fence alongside a newer healthy session", () => {+ const rows = [+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),+ createSystemEntry("stall_recovery", "2026-06-25T12:00:00Z", "sess-poison"),+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-healthy" }),+ ];+ expect(resolveResumableSession(rows, "codex")).toEqual({+ kind: "session",+ sessionId: "sess-healthy",+ stalledSessionId: null,+ fencedSessionId: "sess-poison",+ });+ });+it("returns a session id from a row appended AFTER the barrier", () => {const rows = [{ ...createTimelineEntry({ messages: [], provider: "claude", sessionId: "s-old" }) },diff --git a/src/daemon/src/timeline/timeline.ts b/src/daemon/src/timeline/timeline.tsindex ad42dfbd3..9dc00f763 100644--- a/src/daemon/src/timeline/timeline.ts+++ b/src/daemon/src/timeline/timeline.ts@@ -24,6 +24,27 @@ export { localISOString };export const TIMELINE_MAX_BYTES = 1_048_576;export const TIMELINE_READ_CHUNK_BYTES = 65_536;const DATE_FILENAME_PATTERN = /^\d{4}-\d{2}-\d{2}\.jsonl$/;+const RESUME_CONTROL_FILENAME = ".resume-control.json";+const RESUME_CONTROL_MAX_BYTES = 4_096;++export interface ResumeControlState {+ version: 1;+ attemptedSessionId: string | null;+ fencedSessionId: string | null;+ fullBarrier: "reset_session" | "nap" | null;+}++export type ResumeControlReadResult =+ | { kind: "missing" }+ | { kind: "invalid" }+ | { kind: "state"; state: ResumeControlState };++const EMPTY_RESUME_CONTROL: ResumeControlState = {+ version: 1,+ attemptedSessionId: null,+ fencedSessionId: null,+ fullBarrier: null,+};interface TimelineLine {text: string;@@ -33,7 +54,7 @@ interface TimelineLine {}function isBarrier(entry: ContextTimelineEntry): boolean {- return entry.system?.type === "reset_session" || entry.system?.type === "nap";+ return entry.system !== undefined;}function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {@@ -41,10 +62,24 @@ function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {const entry = value as Partial<ContextTimelineEntry>;if (entry.system) {if (- (entry.system.type !== "reset_session" && entry.system.type !== "nap") ||- typeof entry.system.time !== "string"+ (+ entry.system.type !== "reset_session"+ && entry.system.type !== "nap"+ && entry.system.type !== "stall_recovery_attempt"+ && entry.system.type !== "stall_recovery_clear"+ && entry.system.type !== "stall_recovery"+ )+ || typeof entry.system.time !== "string"+ || (+ entry.system.backend_session_id !== undefined+ && typeof entry.system.backend_session_id !== "string"+ )) return null;- return createSystemEntry(entry.system.type, entry.system.time);+ return createSystemEntry(+ entry.system.type,+ entry.system.time,+ entry.system.backend_session_id,+ );}if (entry.session_id !== null && typeof entry.session_id !== "string") return null;if (entry.provider !== null && typeof entry.provider !== "string") return null;@@ -60,7 +95,7 @@ function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {function timelineLine(entry: ContextTimelineEntry): TimelineLine | null {const boundedEntry = entry.system- ? createSystemEntry(entry.system.type, entry.system.time)+ ? createSystemEntry(entry.system.type, entry.system.time, entry.system.backend_session_id): { ...entry, agent_responses: entry.agent_responses.slice(-5) };const text = JSON.stringify(boundedEntry);const bytes = Buffer.byteLength(text, "utf8") + 1;@@ -341,6 +376,114 @@ export function readRecentEntries(timelineDir: string, opts: ReadRecentOptions =return entries;}+/** Latest durable resume-control row across the complete retained timeline. */+export function readResumeControlState(timelineDir: string): ResumeControlReadResult {+ if (timelineDirectoryState(timelineDir) !== "safe") return { kind: "missing" };+ const filePath = join(timelineDir, RESUME_CONTROL_FILENAME);+ let source: fs.Stats;+ try {+ source = fs.lstatSync(filePath);+ } catch (error) {+ return (error as NodeJS.ErrnoException).code === "ENOENT"+ ? { kind: "missing" }+ : { kind: "invalid" };+ }+ if (!source.isFile() || source.size <= 0 || source.size > RESUME_CONTROL_MAX_BYTES) {+ return { kind: "invalid" };+ }+ let fd: number | null = null;+ try {+ fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));+ const stat = fs.fstatSync(fd);+ if (!stat.isFile() || stat.size <= 0 || stat.size > RESUME_CONTROL_MAX_BYTES) {+ return { kind: "invalid" };+ }+ const bounded = Buffer.allocUnsafe(stat.size + 1);+ let bytesRead = 0;+ while (bytesRead < bounded.length) {+ const count = fs.readSync(fd, bounded, bytesRead, bounded.length - bytesRead, bytesRead);+ if (count <= 0) break;+ bytesRead += count;+ }+ if (bytesRead !== stat.size) return { kind: "invalid" };+ const raw = bounded.subarray(0, bytesRead).toString("utf8");+ const value = JSON.parse(raw) as Partial<ResumeControlState>;+ const validSessionId = (candidate: unknown): candidate is string | null =>+ candidate === null || (typeof candidate === "string" && candidate.length > 0 && candidate.length <= 512);+ if (+ value.version !== 1+ || !validSessionId(value.attemptedSessionId)+ || !validSessionId(value.fencedSessionId)+ || (+ value.fullBarrier !== null+ && value.fullBarrier !== "reset_session"+ && value.fullBarrier !== "nap"+ )+ ) return { kind: "invalid" };+ return {+ kind: "state",+ state: {+ version: 1,+ attemptedSessionId: value.attemptedSessionId,+ fencedSessionId: value.fencedSessionId,+ fullBarrier: value.fullBarrier,+ },+ };+ } catch {+ return { kind: "invalid" };+ } finally {+ if (fd !== null) {+ try { fs.closeSync(fd); } catch { /* best effort */ }+ }+ }+}++export function updateResumeControlState(+ timelineDir: string,+ update: (state: ResumeControlState) => ResumeControlState,+): boolean {+ if (timelineDirectoryState(timelineDir) !== "safe") return false;+ const lockPath = lockPathFor(timelineDir, RESUME_CONTROL_FILENAME);+ if (!acquireLock(lockPath)) return false;+ try {+ const current = readResumeControlState(timelineDir);+ const base = current.kind === "state" ? current.state : EMPTY_RESUME_CONTROL;+ const next = update({ ...base });+ const canonical: ResumeControlState = {+ version: 1,+ attemptedSessionId: next.attemptedSessionId,+ fencedSessionId: next.fencedSessionId,+ fullBarrier: next.fullBarrier,+ };+ const body = JSON.stringify(canonical) + "\n";+ if (Buffer.byteLength(body, "utf8") > RESUME_CONTROL_MAX_BYTES) return false;+ const filePath = join(timelineDir, RESUME_CONTROL_FILENAME);+ const tempPath = join(+ timelineDir,+ `.${RESUME_CONTROL_FILENAME}.${process.pid}.${randomBytes(12).toString("hex")}.tmp`,+ );+ let fd: number | null = null;+ try {+ fd = fs.openSync(tempPath, "wx", 0o600);+ fs.writeFileSync(fd, body, "utf8");+ fs.fsyncSync(fd);+ fs.closeSync(fd);+ fd = null;+ fs.renameSync(tempPath, filePath);+ return true;+ } finally {+ if (fd !== null) {+ try { fs.closeSync(fd); } catch { /* best effort */ }+ }+ try { fs.unlinkSync(tempPath); } catch { /* best effort */ }+ }+ } catch {+ return false;+ } finally {+ releaseLock(lockPath);+ }+}+/* ------------------------------------------------------------------ *//* Write (lock-guarded) *//* ------------------------------------------------------------------ */@@ -588,19 +731,23 @@ export function createTimelineEntry(fields: NewEntryFields): ContextTimelineEntr/*** Build a system entry. System rows are inlined in the JSONL alongside turns- * so the resume walker and any agent-facing reader see them in place. The- * first (and only) type today is `reset_session`.+ * so the resume walker and any agent-facing reader see them in place.*/export function createSystemEntry(type: SystemEntryType,time: string,+ backendSessionId?: string,): ContextTimelineEntry {return {session_id: null,messages: [],agent_responses: [],provider: null,- system: { type, time },+ system: {+ type,+ time,+ ...(backendSessionId ? { backend_session_id: backendSessionId } : {}),+ },};}@@ -617,22 +764,85 @@ export function createSystemEntry(* a codex launch). One session per agent and each timeline lives in that* agent's own workdir, so there's no thread keying.*- * A `system: { type: "reset_session" }` row is a barrier: the walker returns- * null the moment it hits one going newest→oldest. Every turn at or before- * the reset becomes invisible to resume without editing those rows. Since- * kill happens BEFORE the barrier is written (see- * `AgentProcessManager.resetSession`), no `forgot_session_id` fallback is- * needed — no turn row can land after the barrier from the old session.+ * Reset/nap rows are full barriers. A `stall_recovery` row fences only its+ * exact backend session, while attempt/clear rows carry the bounded retry+ * state across daemon restarts without disabling healthy persistence.*/-export function findResumableSession(rows: ContextTimelineEntry[], provider?: string): string | null {+export type ResumeSessionResolution =+ | {+ kind: "session";+ sessionId: string;+ stalledSessionId: string | null;+ fencedSessionId: string | null;+ }+ | {+ kind: "barrier";+ type: SystemEntryType;+ forgottenSessionId: string | null;+ fencedSessionId: string | null;+ }+ | { kind: "none"; stalledSessionId: string | null; fencedSessionId: string | null };++export function resolveResumableSession(+ rows: ContextTimelineEntry[],+ provider?: string,+): ResumeSessionResolution {+ let candidateSessionId: string | null = null;+ let recoveryMarkerSeen = false;+ let stalledSessionId: string | null = null;+ let fencedSessionId: string | null = null;for (let i = rows.length - 1; i >= 0; i--) {const e = rows[i];- // Both reset_session (owner) and nap (agent self-reset) are resume- // barriers — a fresh session was deliberately started at this point.- if (e.system?.type === "reset_session" || e.system?.type === "nap") return null;+ if (e.system) {+ if (e.system.type === "stall_recovery_attempt") {+ if (!recoveryMarkerSeen) {+ recoveryMarkerSeen = true;+ stalledSessionId = e.system.backend_session_id ?? null;+ }+ continue;+ }+ if (e.system.type === "stall_recovery_clear") {+ if (!recoveryMarkerSeen) recoveryMarkerSeen = true;+ continue;+ }+ if (e.system.type === "stall_recovery" && fencedSessionId === null) {+ fencedSessionId = e.system.backend_session_id ?? null;+ }+ if (candidateSessionId !== null) {+ return {+ kind: "session",+ sessionId: candidateSessionId,+ stalledSessionId: stalledSessionId === candidateSessionId ? stalledSessionId : null,+ fencedSessionId,+ };+ }+ return {+ kind: "barrier",+ type: e.system.type,+ forgottenSessionId: e.system.backend_session_id ?? null,+ fencedSessionId,+ };+ }if (!e.session_id) continue;if (provider && e.provider !== provider) continue;- return e.session_id;+ if (candidateSessionId === null) {+ candidateSessionId = e.session_id;+ continue;+ }+ if (candidateSessionId !== e.session_id) break;+ }+ if (candidateSessionId !== null) {+ return {+ kind: "session",+ sessionId: candidateSessionId,+ stalledSessionId: stalledSessionId === candidateSessionId ? stalledSessionId : null,+ fencedSessionId,+ };}- return null;+ return { kind: "none", stalledSessionId, fencedSessionId };+}++export function findResumableSession(rows: ContextTimelineEntry[], provider?: string): string | null {+ const resolution = resolveResumableSession(rows, provider);+ return resolution.kind === "session" ? resolution.sessionId : null;}diff --git a/src/daemon/src/timeline/types.ts b/src/daemon/src/timeline/types.tsindex 2800e5c1d..e8c0d03c7 100644--- a/src/daemon/src/timeline/types.ts+++ b/src/daemon/src/timeline/types.ts@@ -25,27 +25,33 @@ import type { Message } from "../server/contract.js";* message carries its own `time`.* - **System** (`system` set): an out-of-band event the daemon needs to* record in-line with turns so both the resume walker AND the agent's- * own history read see it. First (and only) type today is- * `reset_session`: the owner asked to forget prior conversation. On- * resume, `findResumableSession` walks newest→oldest and STOPS if it- * hits a `reset_session` row before finding a session id — so every- * row at or before the reset is invisible for resume without touching- * the rows themselves.+ * own history read see it. `reset_session` and `nap` deliberately forget+ * all prior conversation; stall-recovery rows persist the one allowed+ * recovery attempt, its clean reset, and the final exact-session fence. On+ * resume, the resolver walks newest→oldest: reset/nap rows hide older+ * candidates, while an exact stall fence remains attached to any newer+ * healthy candidate so stale external resume sources cannot resurrect it.** On a system row, `session_id`/`provider` are null and `messages`/* `agent_responses` are empty. The system row is not mergeable with a* following turn (see `appendOrMergeEntry`).*/-// `reset_session` — owner-triggered reset. `nap` — the agent reset its OWN-// session (see agent:nap). Both are resume barriers with identical walker-// semantics; the distinct type just records who initiated it.-export type SystemEntryType = "reset_session" | "nap";+// Stall recovery uses durable attempt/clear markers plus a final barrier so a+// daemon restart cannot replenish the same poisoned session's retry allowance.+export type SystemEntryType =+ | "reset_session"+ | "nap"+ | "stall_recovery_attempt"+ | "stall_recovery_clear"+ | "stall_recovery";export interface SystemEntry {/** Event kind. Extend by adding to `SystemEntryType`. */type: SystemEntryType;/** ISO timestamp when the event landed — the row is otherwise untimed. */time: string;+ /** Exact backend session associated with a stall-recovery row. */+ backend_session_id?: string;}export interface ContextTimelineEntry {