Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): make stall recovery backend-aware (#543)

Gustavo Ye648c9fd5f09c
diff --git a/src/daemon/agent-driver/src/adapters/codex/index.ts b/src/daemon/agent-driver/src/adapters/codex/index.ts
index e54d85871..d48cdfa44 100644
--- a/src/daemon/agent-driver/src/adapters/codex/index.ts
+++ b/src/daemon/agent-driver/src/adapters/codex/index.ts
@@ -44,6 +44,12 @@ export class CodexDriver implements BackendAdapter {
transport: { kind: "stdio_rpc", protocol: "codex.app-server.v1" },
wakeStart: "immediate",
terminalOwnership: "transport_request",
+ turnSilence: {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ },
} as const;
private readonly eventNormalizer = new CodexEventNormalizer();
diff --git a/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts b/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts
index 053c55dce..768d35bad 100644
--- a/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts
+++ b/src/daemon/agent-driver/src/adapters/codex/normalizer.test.ts
@@ -201,13 +201,36 @@ describe("CodexEventNormalizer — turn id tracking (for turn/steer expectedTurn
threadId: "root-thread",
turn: { id: "next-turn", status: "inProgress" },
}))).toEqual([
- { kind: "turn_owner", receipt: "codex:root-thread:next-turn" },
+ {
+ kind: "turn_owner",
+ receipt: "codex:root-thread:next-turn",
+ nativeTurnId: "next-turn",
+ },
{ kind: "thinking", text: "" },
]);
expect(n.normalizeLine(completed)).toEqual([]);
expect(n.currentTurnId).toBe("next-turn");
});
+ it("classifies retrying stream errors as recovery and preserves nested fatal messages", () => {
+ const n = new CodexEventNormalizer();
+ adoptRootTurn(n, "root-thread", "root-turn");
+
+ expect(n.normalizeLine(notify("error", {
+ threadId: "root-thread",
+ turnId: "root-turn",
+ willRetry: true,
+ error: { message: "Reconnecting... 1/5", additionalDetails: "sensitive transport detail" },
+ }))).toEqual([{ kind: "runtime_recovery", stage: "retrying", source: "codex_stream" }]);
+
+ expect(n.normalizeLine(notify("error", {
+ threadId: "root-thread",
+ turnId: "root-turn",
+ willRetry: false,
+ error: { message: "request failed" },
+ }))).toEqual([{ kind: "error", message: "request failed" }]);
+ });
+
it("clears terminal ownership when a new thread is explicitly adopted and rejects an unowned terminal", () => {
const n = new CodexEventNormalizer();
adoptRootTurn(n, "old-thread", "old-turn");
diff --git a/src/daemon/agent-driver/src/adapters/codex/normalizer.ts b/src/daemon/agent-driver/src/adapters/codex/normalizer.ts
index ef96b1907..0803a82d6 100644
--- a/src/daemon/agent-driver/src/adapters/codex/normalizer.ts
+++ b/src/daemon/agent-driver/src/adapters/codex/normalizer.ts
@@ -103,7 +103,11 @@ export class CodexEventNormalizer {
this.turnId = params.turn.id;
this.terminalTurn = null;
return [
- { kind: "turn_owner", receipt: this.turnReceipt(params.threadId, params.turn.id) },
+ {
+ kind: "turn_owner",
+ receipt: this.turnReceipt(params.threadId, params.turn.id),
+ nativeTurnId: params.turn.id,
+ },
{ kind: "thinking", text: "" },
];
@@ -145,7 +149,10 @@ export class CodexEventNormalizer {
return [{ kind: "turn_end", sessionId: this.threadId ?? undefined, turnOwner: this.turnReceipt(params.threadId, params.turn.id) }];
case "error":
- return [{ kind: "error", message: params?.message ?? "Codex error" }];
+ if (params?.willRetry === true) {
+ return [{ kind: "runtime_recovery", stage: "retrying", source: "codex_stream" }];
+ }
+ return [{ kind: "error", message: params?.error?.message ?? params?.message ?? "Codex error" }];
case "thread/tokenUsage/updated":
case "account/rateLimits/updated":
diff --git a/src/daemon/agent-driver/src/adapters/pi/index.test.ts b/src/daemon/agent-driver/src/adapters/pi/index.test.ts
index bf93e4707..768dafbc8 100644
--- a/src/daemon/agent-driver/src/adapters/pi/index.test.ts
+++ b/src/daemon/agent-driver/src/adapters/pi/index.test.ts
@@ -50,6 +50,15 @@ function fakeDeps() {
}
describe("PiDriver.openLane — AGENTS.md packing", () => {
+ it("maps SDK retry lifecycle into bounded recovery activity", () => {
+ expect(mapPiSdkEvent({ type: "auto_retry_start" }, "sess_1", { sawTextDelta: false }))
+ .toEqual([{ kind: "runtime_recovery", stage: "retrying", source: "pi_auto_retry" }]);
+ expect(mapPiSdkEvent({ type: "auto_retry_end" }, "sess_1", { sawTextDelta: false }))
+ .toEqual([{ kind: "runtime_recovery", stage: "recovered", source: "pi_auto_retry" }]);
+ expect(PI_IGNORED_EVENT_TYPES).not.toContain("auto_retry_start");
+ expect(PI_IGNORED_EVENT_TYPES).not.toContain("auto_retry_end");
+ });
+
it("exposes SDK-only parser and encoder no-ops", () => {
const driver = new PiDriver(() => fakeDeps());
expect(driver.normalizeLine()).toEqual([]);
diff --git a/src/daemon/agent-driver/src/adapters/pi/index.ts b/src/daemon/agent-driver/src/adapters/pi/index.ts
index ba2f44ec5..85071be52 100644
--- a/src/daemon/agent-driver/src/adapters/pi/index.ts
+++ b/src/daemon/agent-driver/src/adapters/pi/index.ts
@@ -152,8 +152,6 @@ export const PI_IGNORED_EVENT_TYPES = [
"queue_update",
"session_info_changed",
"thinking_level_changed",
- "auto_retry_start",
- "auto_retry_end",
"agent_end",
] as const;
@@ -176,6 +174,10 @@ export function mapPiSdkEvent(event: any, sessionId: string, state: { sawTextDel
}
}
switch (event?.type) {
+ case "auto_retry_start":
+ return [{ kind: "runtime_recovery", stage: "retrying", source: "pi_auto_retry" }];
+ case "auto_retry_end":
+ return [{ kind: "runtime_recovery", stage: "recovered", source: "pi_auto_retry" }];
case "tool_execution_start":
return [{ kind: "tool_call", name: event.toolName ?? "unknown_tool", input: event.args ?? {} }];
case "tool_execution_end":
diff --git a/src/daemon/agent-driver/src/contract.ts b/src/daemon/agent-driver/src/contract.ts
index ad074dfd5..5c395757c 100644
--- a/src/daemon/agent-driver/src/contract.ts
+++ b/src/daemon/agent-driver/src/contract.ts
@@ -291,6 +291,7 @@ export type CoreAgentEventPayload =
readonly error: AgentDriverError;
}
| { readonly type: "turn_started"; readonly turnId: string; readonly commandIds: readonly string[] }
+ | { readonly type: "backend_turn_started"; readonly turnId: string; readonly backendTurnId: string }
| { readonly type: "thinking_delta"; readonly turnId: string; readonly text: string }
| { readonly type: "text_delta"; readonly turnId: string; readonly text: string }
| {
@@ -325,6 +326,12 @@ export type CoreAgentEventPayload =
readonly source?: string;
readonly message: string;
}
+ | {
+ readonly type: "recovery";
+ readonly turnId?: string;
+ readonly stage: "retrying" | "recovered";
+ readonly source?: string;
+ }
| {
readonly type: "token_usage";
readonly turnId?: string;
@@ -383,6 +390,13 @@ export interface AgentSessionSnapshot {
| "reviewing"
| "tool_wait"
| "working";
+ readonly turnSilence: {
+ readonly nativeIdleTimeoutMs: number;
+ readonly daemonGraceMs: number;
+ readonly recoveryGraceMs: number;
+ readonly maxRecoveryExtensions: number;
+ readonly normalBudgetMs: number;
+ };
readonly metrics: {
readonly physicalOpenCount: number;
readonly turnCount: number;
diff --git a/src/daemon/agent-driver/src/controller/logical-session.test.ts b/src/daemon/agent-driver/src/controller/logical-session.test.ts
index 0dc9d06ed..6fbeaa385 100644
--- a/src/daemon/agent-driver/src/controller/logical-session.test.ts
+++ b/src/daemon/agent-driver/src/controller/logical-session.test.ts
@@ -268,6 +268,41 @@ async function take(
}
describe.each(["claude", "codex", "cursor", "opencode", "pi"] as const)("%s logical-session conformance", (backend) => {
+ it("projects the public bounded turn-silence defaults", async () => {
+ const { session } = makeSession(backend);
+ expect(session.snapshot().diagnostics.turnSilence).toEqual({
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 360_000,
+ });
+ await session.stop({ reason: "shutdown", forceAfterMs: 10 });
+ });
+
+ it("projects adapter recovery as exact-turn public recovery events", async () => {
+ const { session, driver } = makeSession(backend);
+ const iterator = session.events[Symbol.asyncIterator]();
+ const started = await session.start({ id: "recovery-root", kind: "user", text: "start" });
+ expect(started.status).toBe("accepted");
+ const turnId = started.status === "accepted" ? started.turnId : "unreachable";
+
+ await emit(driver, { kind: "runtime_recovery", stage: "retrying", source: `${backend}.retry` });
+ await emit(driver, { kind: "runtime_recovery", stage: "recovered", source: `${backend}.retry` });
+
+ const recoveryEvents: Array<AgentEvent<BuiltinBackendSpecs, BuiltinBackendId>> = [];
+ while (recoveryEvents.length < 2) {
+ const next = await iterator.next();
+ if (next.done) break;
+ if (next.value.type === "recovery") recoveryEvents.push(next.value);
+ }
+ expect(recoveryEvents).toEqual([
+ expect.objectContaining({ type: "recovery", turnId, stage: "retrying", source: `${backend}.retry` }),
+ expect.objectContaining({ type: "recovery", turnId, stage: "recovered", source: `${backend}.retry` }),
+ ]);
+ await session.stop({ reason: "shutdown", forceAfterMs: 10 });
+ });
+
it("passes the exported black-box conformance suite", async () => {
const { session, driver } = makeSession(backend);
await runAgentDriverConformance(async () => ({
@@ -430,6 +465,35 @@ describe("runtime-lane admission state machine", () => {
await session.stop({ reason: "shutdown", forceAfterMs: 10 });
});
+ it("publishes only bounded opaque native turn ids", async () => {
+ const validLane = new ControlledRuntimeLane();
+ const validSession = makeSession("claude", { lane: validLane }).session;
+ const validIterator = validSession.events[Symbol.asyncIterator]();
+ await validSession.start({ id: "one", kind: "user", text: "start" });
+ validLane.emit({ kind: "turn_owner", receipt: "claude:test:1", nativeTurnId: "native-turn_7" });
+ expect((await take(validIterator as never, 3)).map((event) => event.type)).toEqual([
+ "command_accepted",
+ "turn_started",
+ "backend_turn_started",
+ ]);
+ await validSession.stop({ reason: "shutdown", forceAfterMs: 10 });
+
+ const hostileLane = new ControlledRuntimeLane();
+ const hostileSession = makeSession("claude", { lane: hostileLane }).session;
+ const hostileIterator = hostileSession.events[Symbol.asyncIterator]();
+ await hostileSession.start({ id: "one", kind: "user", text: "start" });
+ hostileLane.emit({
+ kind: "turn_owner",
+ receipt: "claude:test:1",
+ nativeTurnId: "OPENAI_API_KEY=native-secret /Users/private",
+ });
+ hostileLane.emit({ kind: "text", text: "next safe event" });
+ const hostileEvents = await take(hostileIterator as never, 3);
+ expect(hostileEvents.map((event) => event.type)).toEqual(["command_accepted", "turn_started", "text_delta"]);
+ expect(JSON.stringify(hostileEvents)).not.toMatch(/native-secret|\/Users\/private/);
+ await hostileSession.stop({ reason: "shutdown", forceAfterMs: 10 });
+ });
+
it("ignores a forged terminal owner and completes only for the admission receipt", async () => {
const lane = new ControlledRuntimeLane();
const { session } = makeSession("claude", { lane });
@@ -590,6 +654,13 @@ describe("logical delivery diagnostics", () => {
expect(session.snapshot().diagnostics).toEqual({
deliveryPhase: "idle",
+ turnSilence: {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 360_000,
+ },
metrics: {
physicalOpenCount: 0,
turnCount: 0,
diff --git a/src/daemon/agent-driver/src/controller/logical-session.ts b/src/daemon/agent-driver/src/controller/logical-session.ts
index a190f9a1e..61987822f 100644
--- a/src/daemon/agent-driver/src/controller/logical-session.ts
+++ b/src/daemon/agent-driver/src/controller/logical-session.ts
@@ -27,6 +27,12 @@ import type {
LaneAdmission,
RuntimeLane,
} from "../internal/adapter.js";
+import {
+ DEFAULT_DAEMON_GRACE_MS,
+ DEFAULT_MAX_RECOVERY_EXTENSIONS,
+ DEFAULT_NATIVE_IDLE_TIMEOUT_MS,
+ DEFAULT_RECOVERY_GRACE_MS,
+} from "../internal/adapter.js";
import { BufferedEventQueue } from "./event-queue.js";
import { writeAgentFile } from "../internal/agentFile.js";
import { scrubDriverErrorMessage } from "../internal/errors.js";
@@ -162,6 +168,7 @@ implements AgentSession<Specs, Id> {
private readonly eventQueue: BufferedEventQueue<AgentEvent<Specs, Id>>;
private readonly behavior;
+ private readonly turnSilence: AgentSessionSnapshot["diagnostics"]["turnSilence"];
constructor(
readonly backend: Id,
@@ -180,6 +187,18 @@ implements AgentSession<Specs, Id> {
) {
this.capabilities = capabilities;
this.behavior = this.capabilities as import("../contract.js").BackendCapabilities;
+ const declaredSilence = adapter.execution.turnSilence;
+ const nativeIdleTimeoutMs = declaredSilence?.nativeIdleTimeoutMs ?? DEFAULT_NATIVE_IDLE_TIMEOUT_MS;
+ const daemonGraceMs = declaredSilence?.daemonGraceMs ?? DEFAULT_DAEMON_GRACE_MS;
+ const recoveryGraceMs = declaredSilence?.recoveryGraceMs ?? DEFAULT_RECOVERY_GRACE_MS;
+ const maxRecoveryExtensions = declaredSilence?.maxRecoveryExtensions ?? DEFAULT_MAX_RECOVERY_EXTENSIONS;
+ this.turnSilence = {
+ nativeIdleTimeoutMs,
+ daemonGraceMs,
+ recoveryGraceMs,
+ maxRecoveryExtensions,
+ normalBudgetMs: nativeIdleTimeoutMs + daemonGraceMs,
+ };
this.resumeOutcome = launch.resumeSessionId ? "pending" : "not_requested";
this.sessionInstanceId = host.createId();
this.closed = new Promise((resolve) => { this.resolveClosed = resolve; });
@@ -296,6 +315,7 @@ implements AgentSession<Specs, Id> {
lastEventSequence: this.eventSequence,
diagnostics: {
deliveryPhase: this.deliveryPhase(),
+ turnSilence: this.turnSilence,
metrics: {
physicalOpenCount: this.metricValue(this.physicalOpenCount),
turnCount: this.metricValue(this.turnCount),
@@ -617,6 +637,14 @@ implements AgentSession<Specs, Id> {
return;
}
this.activeTurn.terminalOwner = event.receipt;
+ const nativeTurnId = event.nativeTurnId?.trim();
+ if (nativeTurnId && nativeTurnId.length <= 512 && /^[A-Za-z0-9._:-]+$/.test(nativeTurnId)) {
+ this.emit({
+ type: "backend_turn_started",
+ turnId: this.activeTurn.turnId,
+ backendTurnId: nativeTurnId,
+ });
+ }
return;
case "thinking":
if (turnId) this.emit({ type: "thinking_delta", turnId, text: event.text });
@@ -658,8 +686,19 @@ implements AgentSession<Specs, Id> {
message: scrubDriverErrorMessage(event.message, "Runtime diagnostic"),
});
return;
+ case "runtime_recovery":
+ this.emit({
+ type: "recovery",
+ turnId,
+ stage: event.stage,
+ source: event.source,
+ });
+ return;
case "runtime_metric":
- if (event.name === "sse_reconnect" && event.increment === 1) this.sseReconnectCount += 1;
+ if (event.name === "sse_reconnect" && event.increment === 1) {
+ this.sseReconnectCount += 1;
+ this.emit({ type: "recovery", turnId, stage: "retrying", source: "transport_reconnect" });
+ }
return;
case "telemetry": {
const details = jsonValue(event.attrs) as JsonObject;
diff --git a/src/daemon/agent-driver/src/internal/adapter.ts b/src/daemon/agent-driver/src/internal/adapter.ts
index a9040caa3..de4593520 100644
--- a/src/daemon/agent-driver/src/internal/adapter.ts
+++ b/src/daemon/agent-driver/src/internal/adapter.ts
@@ -35,8 +35,19 @@ export interface BackendExecution {
readonly transport: AdapterTransport;
readonly wakeStart: "immediate" | "deferred";
readonly terminalOwnership: TerminalOwnership;
+ readonly turnSilence?: {
+ readonly nativeIdleTimeoutMs: number;
+ readonly daemonGraceMs: number;
+ readonly recoveryGraceMs: number;
+ readonly maxRecoveryExtensions: number;
+ };
}
+export const DEFAULT_NATIVE_IDLE_TIMEOUT_MS = 300_000;
+export const DEFAULT_DAEMON_GRACE_MS = 60_000;
+export const DEFAULT_RECOVERY_GRACE_MS = 60_000;
+export const DEFAULT_MAX_RECOVERY_EXTENSIONS = 1;
+
export type InputMode = "busy" | "idle";
export interface EncodeMessageOptions { mode?: InputMode }
@@ -52,8 +63,9 @@ export type AdapterEvent =
| { kind: "review_finished" }
| { kind: "internal_progress"; source?: string; itemType?: string; payloadBytes?: number }
| { kind: "runtime_diagnostic"; severity?: string; source?: string; message: string }
+ | { kind: "runtime_recovery"; stage: "retrying" | "recovered"; source?: string }
| { kind: "runtime_metric"; name: "sse_reconnect"; increment: 1 }
- | { kind: "turn_owner"; receipt: string }
+ | { kind: "turn_owner"; receipt: string; nativeTurnId?: string }
| { kind: "turn_end"; sessionId?: string; turnOwner?: string }
| { kind: "error"; message: string }
| { kind: "telemetry"; name: "token_usage" | "rate_limits"; source: string; usageKind?: string; attrs: Record<string, unknown> };
diff --git a/src/daemon/agent-driver/src/registry.test.ts b/src/daemon/agent-driver/src/registry.test.ts
index 876fa114b..472220852 100644
--- a/src/daemon/agent-driver/src/registry.test.ts
+++ b/src/daemon/agent-driver/src/registry.test.ts
@@ -40,6 +40,12 @@ describe("driver.capabilities", () => {
transport: { kind: "stdio_rpc", protocol: "codex.app-server.v1" },
wakeStart: "immediate",
terminalOwnership: "transport_request",
+ turnSilence: {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ },
},
cursor: {
lifetime: "session",
@@ -155,4 +161,40 @@ describe("adapter registration runtime boundary", () => {
adapter,
)).toThrow("delivery conflicts with its execution lifetime");
});
+
+ it("rejects malformed turn-silence policies instead of disabling stall detection", () => {
+ const adapter = {
+ id: "sixth",
+ instructionDelivery: { kind: "native" },
+ execution: {
+ lifetime: "session",
+ transport: { kind: "stdio_stream", protocol: "sixth.test.v1" },
+ wakeStart: "immediate",
+ terminalOwnership: "vendor_message",
+ turnSilence: {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ },
+ },
+ probe() {},
+ openLane() {},
+ };
+ const invalidPolicies = [
+ null,
+ { ...adapter.execution.turnSilence, nativeIdleTimeoutMs: 0 },
+ { ...adapter.execution.turnSilence, daemonGraceMs: -1 },
+ { ...adapter.execution.turnSilence, recoveryGraceMs: Number.POSITIVE_INFINITY },
+ { ...adapter.execution.turnSilence, maxRecoveryExtensions: 0.5 },
+ { ...adapter.execution.turnSilence, nativeIdleTimeoutMs: Number.MAX_SAFE_INTEGER, daemonGraceMs: 1 },
+ ];
+ for (const turnSilence of invalidPolicies) {
+ expect(() => assertAdapterCompatibility(
+ "sixth",
+ EXPECTED.claude,
+ { ...adapter, execution: { ...adapter.execution, turnSilence } },
+ )).toThrow("invalid turnSilence declaration");
+ }
+ });
});
diff --git a/src/daemon/agent-driver/src/registry.ts b/src/daemon/agent-driver/src/registry.ts
index fb10dc8fe..6219e94cb 100644
--- a/src/daemon/agent-driver/src/registry.ts
+++ b/src/daemon/agent-driver/src/registry.ts
@@ -146,6 +146,23 @@ export function assertAdapterCompatibility(
) {
throw new Error(`Adapter ${registrationId} has an invalid transport declaration`);
}
+ if (execution.turnSilence !== undefined) {
+ const silence = execution.turnSilence as Record<string, unknown> | undefined;
+ const safeInteger = (value: unknown, minimum: number): value is number =>
+ typeof value === "number" && Number.isSafeInteger(value) && value >= minimum;
+ if (
+ !silence
+ || typeof silence !== "object"
+ || Array.isArray(silence)
+ || !safeInteger(silence.nativeIdleTimeoutMs, 1)
+ || !safeInteger(silence.daemonGraceMs, 0)
+ || !safeInteger(silence.recoveryGraceMs, 0)
+ || !safeInteger(silence.maxRecoveryExtensions, 0)
+ || !Number.isSafeInteger(silence.nativeIdleTimeoutMs + silence.daemonGraceMs)
+ ) {
+ throw new Error(`Adapter ${registrationId} has an invalid turnSilence declaration`);
+ }
+ }
const capabilities = registeredCapabilities as Record<string, unknown> | undefined;
const declaredLifetime = capabilities?.sessionLifetime === "persistent" ? "session" : "turn";
if (execution.lifetime !== declaredLifetime) {
diff --git a/src/daemon/src/cli/daemonRunner.test.ts b/src/daemon/src/cli/daemonRunner.test.ts
index d55b5eb0b..c836a355f 100644
--- a/src/daemon/src/cli/daemonRunner.test.ts
+++ b/src/daemon/src/cli/daemonRunner.test.ts
@@ -289,7 +289,7 @@ describe("daemon runner logger", () => {
}
});
- it("preserves projected FSM timestamps through the runner bundle seam", async () => {
+ it("exports projected native-liveness diagnostics and FSM timestamps through the runner bundle seam", async () => {
resetDiagnosticHarness();
vi.useFakeTimers();
const listenersBefore = processListenerSnapshot();
@@ -316,12 +316,19 @@ describe("daemon runner logger", () => {
traceSink.write(JSON.stringify({
recordKind: "fsm",
agentId: "bot_1",
- event: "register",
+ event: "runtime_signal",
status: "idle",
turnActive: false,
inbox: 0,
lastDeliverAt: null,
lastProgressAt: eventTime,
+ lastNativeActivityAt: eventTime - 10,
+ lastNativeActivityKind: "backend_turn_started",
+ runtimePhase: "inference",
+ backendTurnId: "native-turn-7",
+ turnSilenceBudgetMs: 360_000,
+ nativeDeadlineAt: eventTime + 360_000,
+ recoveryExtensionsUsed: 1,
idleSince: eventTime,
resetting: false,
resettingSince: null,
@@ -331,6 +338,7 @@ describe("daemon runner logger", () => {
nowMs: eventTime,
timeIso: new Date(eventTime).toISOString(),
sinceProgressMs: 0,
+ sinceNativeActivityMs: 10,
sinceDeliverMs: null,
sinceStoppingMs: null,
}));
@@ -388,9 +396,17 @@ describe("daemon runner logger", () => {
expect.objectContaining({
recordKind: "fsm",
agentId: "bot_1",
- event: "register",
+ event: "runtime_signal",
nowMs: eventTime,
timeMs: eventTime,
+ lastNativeActivityAt: eventTime - 10,
+ lastNativeActivityKind: "backend_turn_started",
+ runtimePhase: "inference",
+ backendTurnId: "native-turn-7",
+ turnSilenceBudgetMs: 360_000,
+ nativeDeadlineAt: eventTime + 360_000,
+ recoveryExtensionsUsed: 1,
+ sinceNativeActivityMs: 10,
}),
expect.objectContaining({
recordKind: "turn_span",
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index b17ced323..afb25d074 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -1,6 +1,6 @@
import { describe, it, expect, vi, afterEach } from "vitest";
import { EventEmitter } from "events";
-import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
@@ -453,9 +453,9 @@ for await (const line of createInterface({ input: process.stdin })) {
messageReminderClock: {
now: () => now,
setTimer: (callback) => {
- const timer = { callback, cancelled: false };
+ const timer = { callback, cancelled: false, unref() {} };
timers.push(timer);
- return { unref() {} } as ReturnType<typeof setTimeout>;
+ return timer as unknown as ReturnType<typeof setTimeout>;
},
clearTimer: (handle) => {
const index = timers.findIndex((timer) => timer === (handle as unknown));
@@ -474,6 +474,13 @@ for await (const line of createInterface({ input: process.stdin })) {
headers: { authorization: "Bearer vch_test", "content-type": "application/json" },
body: JSON.stringify({ channel: "/demo#1234/general", sentSeq, remindAfterMs }),
});
+ const sentFrames = () => sockets[0]!.sent.map((frame) => JSON.parse(frame) as Record<string, unknown>);
+ const wakeAcked = (launchId: string) => sentFrames().some((frame) =>
+ frame.type === "agent_wake_ack" && frame.launchId === launchId && frame.status === "ok");
+ const stopAcked = () => sentFrames().some((frame) =>
+ frame.type === "agent_stopped_ack" && frame.agentId === "bot_1" && frame.status === "ok");
+ const reminderDeliveries = () => deliver.mock.calls.filter(([, message]) =>
+ typeof message.id === "string" && message.id.includes(":reminder:"));
try {
sockets[0]!.emit("open");
@@ -484,20 +491,20 @@ for await (const line of createInterface({ input: process.stdin })) {
launchId: "launch_1",
unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 1 },
}));
- await vi.waitFor(() => expect(deliver).toHaveBeenCalledTimes(1));
+ await vi.waitFor(() => expect(wakeAcked("launch_1")).toBe(true));
now = 2_000;
expect(await (await arm(7)).json()).toEqual({ armed: true, dueAt: 62_000 });
runTimer(0);
- expect(deliver).toHaveBeenCalledTimes(2);
- expect(deliver).toHaveBeenLastCalledWith("bot_1", expect.objectContaining({
+ expect(reminderDeliveries()).toHaveLength(1);
+ expect(reminderDeliveries().at(-1)).toEqual(["bot_1", expect.objectContaining({
text: expect.stringContaining("/demo#1234/general#7"),
- }));
+ })]);
await arm(8);
expect(await (await arm(9, 0)).json()).toEqual({ armed: false, reason: "disabled" });
runTimer(1);
- expect(deliver).toHaveBeenCalledTimes(2);
+ expect(reminderDeliveries()).toHaveLength(1);
await arm(10);
sockets[0]!.emit("message", JSON.stringify({
@@ -507,9 +514,10 @@ for await (const line of createInterface({ input: process.stdin })) {
launchId: "launch_2",
unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 11 },
}));
- await new Promise((resolve) => setTimeout(resolve, 0));
+ await vi.waitFor(() => expect(wakeAcked("launch_2")).toBe(true));
+ await vi.waitFor(() => expect(timers[2]?.cancelled).toBe(true));
runTimer(2);
- expect(deliver).toHaveBeenCalledTimes(2);
+ expect(reminderDeliveries()).toHaveLength(1);
await arm(12);
sockets[0]!.emit("message", JSON.stringify({
@@ -519,27 +527,32 @@ for await (const line of createInterface({ input: process.stdin })) {
launchId: "launch_duplicate",
unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 11 },
}));
- await new Promise((resolve) => setTimeout(resolve, 0));
+ await vi.waitFor(() => expect(wakeAcked("launch_duplicate")).toBe(true));
+ expect(timers[3]?.cancelled).toBe(false);
runTimer(3);
- expect(deliver).toHaveBeenCalledTimes(3);
- expect(deliver).toHaveBeenLastCalledWith("bot_1", expect.objectContaining({
+ expect(reminderDeliveries()).toHaveLength(2);
+ expect(reminderDeliveries().at(-1)).toEqual(["bot_1", expect.objectContaining({
text: expect.stringContaining("/demo#1234/general#12"),
- }));
+ })]);
await arm(13);
sockets[0]!.emit("message", JSON.stringify({ type: "agent:stop", agentId: "bot_1" }));
+ await vi.waitFor(() => expect(stopAcked()).toBe(true));
+ await vi.waitFor(() => expect(timers[4]?.cancelled).toBe(true));
runTimer(4);
- expect(deliver).toHaveBeenCalledTimes(3);
+ expect(reminderDeliveries()).toHaveLength(2);
await arm(14);
sockets[0]!.emit("message", JSON.stringify({ type: "bot:removed", botId: "bot_1" }));
+ await vi.waitFor(() => expect(timers[5]?.cancelled).toBe(true));
runTimer(5);
- expect(deliver).toHaveBeenCalledTimes(3);
+ expect(reminderDeliveries()).toHaveLength(2);
await arm(15);
await daemon.stop();
+ expect(timers[6]?.cancelled).toBe(true);
runTimer(6);
- expect(deliver).toHaveBeenCalledTimes(3);
+ expect(reminderDeliveries()).toHaveLength(2);
} finally {
// stop is idempotent enough for the failure path and keeps the loopback
// server from leaking if an assertion above throws early.
@@ -550,6 +563,11 @@ for await (const line of createInterface({ input: process.stdin })) {
it("resets self-sleep only for newer wakes and suspends it while an agent works", async () => {
const sockets: FakeSocket[] = [];
const sessions: DaemonFakeSession[] = [];
+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "daemon-self-sleep-"));
+ startupSweepDirs.push(workingDirectoryBase);
+ // The real driver SDK prepares the agent directory before session_started.
+ // This test uses a fake session, so reproduce that precondition explicitly.
+ mkdirSync(join(workingDirectoryBase, "bot_1"));
const timers: Array<{
callback: () => void;
delayMs: number;
@@ -590,6 +608,7 @@ for await (const line of createInterface({ input: process.stdin })) {
return session;
},
capabilities: [],
+ workingDirectoryBase,
onSelfSleep,
selfSleepClock: clock,
});
@@ -600,6 +619,13 @@ for await (const line of createInterface({ input: process.stdin })) {
launchId: `launch_${seq}`,
unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: seq },
}));
+ const sentFrames = () => sockets[0]!.sent.map((frame) => JSON.parse(frame) as Record<string, unknown>);
+ const wakeAckCount = (seq: number) => sentFrames()
+ .filter((frame) => frame.type === "agent_wake_ack" && frame.launchId === `launch_${seq}` && frame.status === "ok")
+ .length;
+ const activityCount = (state: "running" | "idle") => sentFrames()
+ .filter((frame) => frame.type === "agent_activity" && frame.agentId === "bot_1" && frame.state === state)
+ .length;
try {
expect(timers).toHaveLength(1);
@@ -607,20 +633,25 @@ for await (const line of createInterface({ input: process.stdin })) {
sockets[0]!.emit("open");
wake(1);
await vi.waitFor(() => expect(sessions).toHaveLength(1));
+ await vi.waitFor(() => expect(wakeAckCount(1)).toBe(1));
+ await vi.waitFor(() => expect(activityCount("running")).toBe(1), { timeout: 5_000 });
await vi.waitFor(() => expect(timers).toHaveLength(2));
- expect(timers.every((timer) => timer.cancelled)).toBe(true);
+ expect(timers.slice(0, 2).every((timer) => timer.cancelled)).toBe(true);
await sessions[0]!.fire("runtime_event", { kind: "turn_end", sessionId: "test-session" });
+ await vi.waitFor(() => expect(activityCount("idle")).toBe(1), { timeout: 5_000 });
await vi.waitFor(() => expect(timers).toHaveLength(3));
expect(timers[2]?.cancelled).toBe(false);
wake(1);
- await new Promise<void>((resolve) => setImmediate(resolve));
+ await vi.waitFor(() => expect(wakeAckCount(1)).toBe(2));
expect(timers).toHaveLength(3);
wake(2);
+ await vi.waitFor(() => expect(wakeAckCount(2)).toBe(1));
await vi.waitFor(() => expect(timers.length).toBeGreaterThanOrEqual(4));
expect(timers[2]?.cancelled).toBe(true);
+ expect(timers[3]?.cancelled).toBe(false);
timers[2]?.callback();
expect(onSelfSleep).not.toHaveBeenCalled();
} finally {
@@ -1151,6 +1182,11 @@ describe("createDaemon — logging", () => {
}) as unknown as typeof fetch;
const channel = "/demo#1234/general";
+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "working-unread-coverage-"));
+ startupSweepDirs.push(workingDirectoryBase);
+ // The real driver SDK prepares this directory before session_started. This
+ // test uses a fake session, so reproduce that precondition explicitly.
+ mkdirSync(join(workingDirectoryBase, "bot_working"));
const starts: Array<{ id: string; text: string }> = [];
const sends: Array<{ id: string; text: string; sequence?: number }> = [];
const sockets: FakeSocket[] = [];
@@ -1160,6 +1196,7 @@ describe("createDaemon — logging", () => {
serverWsUrl: "ws://x",
webSocketFactory: factory(sockets) as any,
runtimeReport: [{ id: "codex" }],
+ workingDirectoryBase,
driverFor: () => fullFakeDriver("codex"),
sessionFactory: daemonSessionFactory({
onStart: (input) => starts.push(input),
diff --git a/src/daemon/src/diagnostics/privacyPolicy.test.ts b/src/daemon/src/diagnostics/privacyPolicy.test.ts
index 734c874a5..92acc6f8a 100644
--- a/src/daemon/src/diagnostics/privacyPolicy.test.ts
+++ b/src/daemon/src/diagnostics/privacyPolicy.test.ts
@@ -49,11 +49,12 @@ const EXPECTED_MANAGER_EVENTS = [
"begin_reset",
"rewake_after_reset",
"runtime_signal",
+ "stall_control_failed",
"admission_started",
"admission_settled",
] as const;
-const EXPECTED_MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "force_exit", "gated_hold"] as const;
+const EXPECTED_MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "clear_stall_recovery", "force_exit", "gated_hold"] as const;
const EXPECTED_AGENT_STATUSES = ["idle", "starting", "running", "stopping"] as const;
const EXPECTED_DERIVED_ACTIVITIES = ["idle", "starting", "running", "stopping"] as const;
const EXPECTED_TURN_OUTCOMES = ["clean", "errored"] as const;
@@ -77,6 +78,18 @@ const EXPECTED_ABORT_CAUSES = [
"force_exit",
"other",
] as const;
+const EXPECTED_NATIVE_ACTIVITY_KINDS = [
+ "turn_started",
+ "backend_turn_started",
+ "thinking",
+ "text",
+ "tool_call",
+ "tool_output",
+ "internal_progress",
+ "recovery",
+ "turn_end",
+] as const;
+const EXPECTED_RUNTIME_PHASES = ["idle", "admission", "inference", "tool", "recovery", "terminal"] as const;
const EXPECTED_LOG_LEVELS = ["debug", "info", "warn", "error"] as const;
const EXPECTED_EXIT_SIGNALS = [
"SIGABRT",
@@ -137,6 +150,13 @@ const FSM_FIELD_RULES: Readonly<Record<string, ProjectionFieldRule>> = {
inbox: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },
lastDeliverAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },
lastProgressAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },
+ lastNativeActivityAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },
+ lastNativeActivityKind: { type: "enum", values: EXPECTED_NATIVE_ACTIVITY_KINDS, nullable: true, optional: true },
+ runtimePhase: { type: "enum", values: EXPECTED_RUNTIME_PHASES, optional: true },
+ backendTurnId: { type: "string", maxChars: 512, nullable: true, optional: true },
+ turnSilenceBudgetMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },
+ nativeDeadlineAt: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true, optional: true },
+ recoveryExtensionsUsed: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },
idleSince: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },
resetting: { type: "boolean" },
resettingSince: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },
@@ -155,6 +175,7 @@ const FSM_FIELD_RULES: Readonly<Record<string, ProjectionFieldRule>> = {
nowMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },
timeIso: { type: "timestamp" },
sinceProgressMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER },
+ sinceNativeActivityMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, optional: true },
sinceDeliverMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },
sinceStoppingMs: { type: "integer", min: 0, max: Number.MAX_SAFE_INTEGER, nullable: true },
traceTurnId: { type: "string", maxChars: 1_024, optional: true },
@@ -375,6 +396,13 @@ function baseFsm(overrides: Record<string, unknown> = {}): Record<string, unknow
inbox: 2,
lastDeliverAt: 100,
lastProgressAt: 110,
+ lastNativeActivityAt: 115,
+ lastNativeActivityKind: "thinking",
+ runtimePhase: "inference",
+ backendTurnId: "native-turn-7",
+ turnSilenceBudgetMs: 360_000,
+ nativeDeadlineAt: 475_000,
+ recoveryExtensionsUsed: 0,
idleSince: null,
resetting: false,
resettingSince: null,
@@ -384,6 +412,7 @@ function baseFsm(overrides: Record<string, unknown> = {}): Record<string, unknow
nowMs: 120,
timeIso: "1970-01-01T00:00:00.120Z",
sinceProgressMs: 10,
+ sinceNativeActivityMs: 5,
sinceDeliverMs: 20,
sinceStoppingMs: null,
traceTurnId: "launch_1:1",
@@ -518,6 +547,13 @@ describe("B2c privacy policy", () => {
inbox: 2,
lastDeliverAt: 100,
lastProgressAt: 110,
+ lastNativeActivityAt: 115,
+ lastNativeActivityKind: "thinking",
+ runtimePhase: "inference",
+ backendTurnId: "native-turn-7",
+ turnSilenceBudgetMs: 360_000,
+ nativeDeadlineAt: 475_000,
+ recoveryExtensionsUsed: 0,
idleSince: null,
resetting: false,
resettingSince: null,
@@ -527,6 +563,7 @@ describe("B2c privacy policy", () => {
nowMs: 120,
timeIso: "1970-01-01T00:00:00.120Z",
sinceProgressMs: 10,
+ sinceNativeActivityMs: 5,
sinceDeliverMs: 20,
sinceStoppingMs: null,
traceTurnId: "launch_1:1",
@@ -537,6 +574,12 @@ describe("B2c privacy policy", () => {
});
expect(JSON.stringify(projected)).not.toMatch(/PROMPT_LEAK|SEND_LEAK|RESPONSE_LEAK|THINKING_LEAK|TOOL_LEAK|ERROR_LEAK|RECENT_LEAK|UNKNOWN_LEAK/);
+ const credentialBearingNativeId = api.projectFsmTraceRow(baseFsm({
+ backendTurnId: "OPENAI_API_KEY=native-turn-secret /Users/private/native-turn",
+ }), "target-agent");
+ expect(credentialBearingNativeId).not.toBeNull();
+ expect(JSON.stringify(credentialBearingNativeId)).not.toMatch(/native-turn-secret|\/Users\/private/);
+
for (const deliveryPhase of EXPECTED_DELIVERY_PHASES) {
expect(api.projectFsmTraceRow(baseFsm({ deliveryPhase }), "target-agent"), deliveryPhase).toMatchObject({ deliveryPhase });
}
diff --git a/src/daemon/src/diagnostics/privacyPolicy.ts b/src/daemon/src/diagnostics/privacyPolicy.ts
index 454d8cbc2..4ba27e1d1 100644
--- a/src/daemon/src/diagnostics/privacyPolicy.ts
+++ b/src/daemon/src/diagnostics/privacyPolicy.ts
@@ -164,8 +164,8 @@ export function projectDaemonLogRow(value: unknown, targetAgentId: string): Reco
};
}
-const MANAGER_EVENTS = ["register", "wake", "spawned", "session", "root_work", "turn_end", "exit", "tick", "reset_session", "begin_reset", "rewake_after_reset", "runtime_signal", "admission_started", "admission_settled"] as const;
-const MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "force_exit", "gated_hold"] as const;
+const MANAGER_EVENTS = ["register", "wake", "spawned", "session", "root_work", "turn_end", "exit", "tick", "reset_session", "begin_reset", "rewake_after_reset", "runtime_signal", "stall_control_failed", "admission_started", "admission_settled"] as const;
+const MANAGER_EFFECTS = ["spawn", "send", "stop", "terminate_stalled", "clear_stall_recovery", "force_exit", "gated_hold"] as const;
const AGENT_STATUSES = ["idle", "starting", "running", "stopping"] as const;
const DELIVERY_PHASES = ["idle", "admission_wait", "steering", "next_turn_queued", "compacting", "reviewing", "tool_wait", "working"] as const;
const RESUME_OUTCOMES = ["not_requested", "pending", "resumed", "reset_required", "failed"] as const;
@@ -174,6 +174,8 @@ const TERMINATION_CAUSES = ["runtime_error", "killed_stalled", "other"] as const
const SPAWN_FAILURE_REASONS = ["ENOENT", "handshake_timeout", "pre_handshake_exit", "spawn_threw", "other"] as const;
const TERMINATION_SEMANTICS = ["killed_stalled", "idle_stop", "force_exit", "other"] as const;
const ABORT_CAUSES = ["start_threw", "start_rejected", "send_threw", "spawn_failure", "handshake_timeout", "reset", "nap", "model_switch", "requested_stop", "shutdown", "physical_exit", "terminate_stalled", "force_exit", "other"] as const;
+const NATIVE_ACTIVITY_KINDS = ["turn_started", "backend_turn_started", "thinking", "text", "tool_call", "tool_output", "internal_progress", "recovery", "turn_end"] as const;
+const RUNTIME_PHASES = ["idle", "admission", "inference", "tool", "recovery", "terminal"] as const;
const EXIT_SIGNALS = new Set(["SIGABRT", "SIGALRM", "SIGBREAK", "SIGBUS", "SIGCHLD", "SIGCONT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINFO", "SIGINT", "SIGIO", "SIGIOT", "SIGKILL", "SIGLOST", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", "SIGQUIT", "SIGSEGV", "SIGSTKFLT", "SIGSTOP", "SIGSYS", "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ", "other"]);
function enumValue(value: unknown, values: readonly string[], bucket = false, maxChars = 64): string | null {
@@ -204,6 +206,12 @@ function copyString(row: Record<string, unknown>, output: Record<string, unknown
return true;
}
+function copyScrubbedString(row: Record<string, unknown>, output: Record<string, unknown>, key: string, maxChars: number, optional = false, nullable = false): boolean {
+ if (!copyString(row, output, key, maxChars, optional, nullable)) return false;
+ if (typeof output[key] === "string") output[key] = scrubDiagnosticText(output[key] as string);
+ return true;
+}
+
function projectTraceBase(row: Record<string, unknown>, targetAgentId: string): Record<string, unknown> | null {
if (row.agentId !== targetAgentId || !boundedString(row.agentId, 128) || !canonicalTime(row.timeIso)) return null;
const output: Record<string, unknown> = { recordType: "fsm", recordKind: row.recordKind, agentId: row.agentId };
@@ -233,11 +241,30 @@ function projectFsm(row: Record<string, unknown>, targetAgentId: string): Record
|| !copyInteger(row, output, "lastDeliverAt", { nullable: true })
|| !copyInteger(row, output, "lastProgressAt")
|| !copyInteger(row, output, "idleSince", { nullable: true })) return null;
+ if (!copyInteger(row, output, "lastNativeActivityAt", { optional: true })
+ || !copyScrubbedString(row, output, "backendTurnId", 512, true, true)
+ || !copyInteger(row, output, "turnSilenceBudgetMs", { optional: true })
+ || !copyInteger(row, output, "nativeDeadlineAt", { optional: true, nullable: true })
+ || !copyInteger(row, output, "recoveryExtensionsUsed", { optional: true })) return null;
+ if ("lastNativeActivityKind" in row && row.lastNativeActivityKind !== undefined) {
+ if (row.lastNativeActivityKind === null) output.lastNativeActivityKind = null;
+ else {
+ const kind = enumValue(row.lastNativeActivityKind, NATIVE_ACTIVITY_KINDS);
+ if (!kind) return null;
+ output.lastNativeActivityKind = kind;
+ }
+ }
+ if ("runtimePhase" in row && row.runtimePhase !== undefined) {
+ const phase = enumValue(row.runtimePhase, RUNTIME_PHASES);
+ if (!phase) return null;
+ output.runtimePhase = phase;
+ }
output.resetting = row.resetting;
if (!copyInteger(row, output, "resettingSince", { nullable: true })
|| !copyInteger(row, output, "stoppingSince", { nullable: true })) return null;
output.deliveryPhase = deliveryPhase;
if (!copyInteger(row, output, "sinceProgressMs")
+ || !copyInteger(row, output, "sinceNativeActivityMs", { optional: true })
|| !copyInteger(row, output, "sinceDeliverMs", { nullable: true })
|| !copyInteger(row, output, "sinceStoppingMs", { nullable: true })
|| !projectOptionalSpanMetadata(row, output)) return null;
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.ts
index c2d47a2aa..2842809b5 100644
--- a/src/daemon/src/manager/managerPolicy.test.ts
+++ b/src/daemon/src/manager/managerPolicy.test.ts
@@ -6,6 +6,7 @@ import {
type ManagerState,
type AgentState,
type AgentStatus,
+ type TurnSilencePolicy,
} from "./managerPolicy.js";
interface LegacyCaps {
@@ -54,6 +55,30 @@ function spawnRoot(state: ManagerState, nowMs: number, turnId = "turn-a"): Manag
return reduceManager(next, { type: "spawned", agentId: "a", nowMs }).state;
}
+function spawnRootWithSilence(
+ state: ManagerState,
+ nowMs: number,
+ turnSilence: TurnSilencePolicy,
+ turnId = "turn-a",
+): ManagerState {
+ let next = reduceManager(state, {
+ type: "attach_session",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ nowMs,
+ turnSilence,
+ }).state;
+ next = reduceManager(next, {
+ type: "turn_started",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ turnId,
+ commandIds: [],
+ nowMs,
+ }).state;
+ return reduceManager(next, { type: "spawned", agentId: "a", nowMs }).state;
+}
+
function completeRoot(state: ManagerState, turnId: string, nowMs: number) {
return reduceManager(state, {
type: "turn_completed",
@@ -232,6 +257,161 @@ describe("reduceManager — turn_end behavior", () => {
});
describe("reduceManager — tick: stall + idle hibernation", () => {
+ it("allows one same-session stall recovery, then forgets that backend session on a repeat stall", () => {
+ let s = createInitialManagerState(100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { id: "m1", text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0, "turn-1");
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;
+
+ const firstStall = reduceManager(s, { type: "tick", nowMs: 100 });
+ expect(firstStall.effects).toEqual([{
+ type: "terminate_stalled",
+ agentId: "a",
+ recordSessionId: "sess-poison",
+ }]);
+ expect(firstStall.state.agents.a).toMatchObject({
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ });
+
+ s = reduceManager(firstStall.state, {
+ type: "turn_completed",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ turnId: "turn-1",
+ nowMs: 101,
+ endReason: "errored",
+ terminationCause: "killed_stalled",
+ }).state;
+ expect(s.agents.a.stalledSessionId).toBe("sess-poison");
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { id: "m2", text: "m2" }, nowMs: 102 }).state;
+ const respawn = reduceManager(s, { type: "exit", agentId: "a" });
+ expect(respawn.effects).toContainEqual({
+ type: "spawn",
+ agentId: "a",
+ messages: [{ id: "m2", text: "m2" }],
+ resumeSessionId: "sess-poison",
+ });
+
+ s = reduceManager(respawn.state, {
+ type: "attach_session",
+ agentId: "a",
+ sessionInstanceId: "session-instance-2",
+ nowMs: 103,
+ }).state;
+ s = reduceManager(s, {
+ type: "turn_started",
+ agentId: "a",
+ sessionInstanceId: "session-instance-2",
+ turnId: "turn-2",
+ commandIds: [],
+ nowMs: 103,
+ }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 103 }).state;
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;
+
+ const secondStall = reduceManager(s, { type: "tick", nowMs: 203 });
+ expect(secondStall.effects).toEqual([{
+ type: "terminate_stalled",
+ agentId: "a",
+ forgetSessionId: "sess-poison",
+ }]);
+ expect(secondStall.state.agents.a.sessionId).toBeNull();
+ expect(secondStall.state.agents.a.stalledSessionId).toBeNull();
+
+ s = reduceManager(secondStall.state, {
+ type: "wake",
+ agentId: "a",
+ message: { id: "m3", text: "m3" },
+ nowMs: 204,
+ }).state;
+ expect(reduceManager(s, { type: "exit", agentId: "a" }).effects).toContainEqual({
+ type: "spawn",
+ agentId: "a",
+ messages: [{ id: "m3", text: "m3" }],
+ resumeSessionId: null,
+ });
+ });
+
+ it("resets the stall breaker after a clean completion or a different backend session", () => {
+ let s = createInitialManagerState(100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0, "turn-1");
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-1" }).state;
+ s = reduceManager(s, { type: "tick", nowMs: 100 }).state;
+ expect(s.agents.a.stalledSessionId).toBe("sess-1");
+
+ const clean = reduceManager(s, {
+ type: "turn_completed",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ turnId: "turn-1",
+ nowMs: 101,
+ });
+ expect(clean.effects).toEqual([{
+ type: "clear_stall_recovery",
+ agentId: "a",
+ sessionId: "sess-1",
+ }]);
+ s = clean.state;
+ expect(s.agents.a.stalledSessionId).toBeNull();
+
+ s = { ...s, agents: { ...s.agents, a: { ...s.agents.a, stalledSessionId: "sess-1" } } };
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-2" }).state;
+ expect(s.agents.a.stalledSessionId).toBeNull();
+ expect(s.agents.a.sessionId).toBe("sess-2");
+ });
+
+ it("rolls back attempt/fence transitions and keeps a failed clear fail-closed", () => {
+ let s = createInitialManagerState(100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0, "turn-1");
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-poison" }).state;
+
+ const first = reduceManager(s, { type: "tick", nowMs: 100 }).state;
+ s = reduceManager(first, {
+ type: "stall_control_failed",
+ agentId: "a",
+ sessionId: "sess-poison",
+ transition: "attempt",
+ }).state;
+ expect(s.agents.a).toMatchObject({
+ status: "running",
+ sessionId: "sess-poison",
+ stalledSessionId: null,
+ stoppingSince: null,
+ });
+
+ s = { ...s, agents: { ...s.agents, a: { ...s.agents.a, stalledSessionId: "sess-poison" } } };
+ const repeated = reduceManager(s, { type: "tick", nowMs: 101 }).state;
+ expect(repeated.agents.a).toMatchObject({ status: "stopping", sessionId: null, stalledSessionId: null });
+ s = reduceManager(repeated, {
+ type: "stall_control_failed",
+ agentId: "a",
+ sessionId: "sess-poison",
+ transition: "fence",
+ }).state;
+ expect(s.agents.a).toMatchObject({
+ status: "running",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ stoppingSince: null,
+ });
+
+ const completed = completeRoot(s, "turn-1", 102).state;
+ expect(completed.agents.a.stalledSessionId).toBeNull();
+ s = reduceManager(completed, {
+ type: "stall_control_failed",
+ agentId: "a",
+ sessionId: "sess-poison",
+ transition: "clear",
+ }).state;
+ expect(s.agents.a.stalledSessionId).toBe("sess-poison");
+ });
+
it("terminates a stalled per-turn agent past the stale threshold", () => {
let s = createInitialManagerState(100); // staleThresholdMs = 100
s = register(s, "a", PER_TURN);
@@ -251,6 +431,91 @@ describe("reduceManager — tick: stall + idle hibernation", () => {
expect(reduceManager(s, { type: "tick", nowMs: 50 }).effects).toEqual([]);
});
+ it("keeps a high-reasoning turn alive through 112s of semantic silence without consuming the breaker", () => {
+ const silence: TurnSilencePolicy = {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 360_000,
+ };
+ let s = register(createInitialManagerState(), "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "reason" }, nowMs: 0 }).state;
+ s = spawnRootWithSilence(s, 0, silence, "reasoning-turn");
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-reasoning" }).state;
+
+ expect(reduceManager(s, { type: "tick", nowMs: 112_000 }).effects).toEqual([]);
+ s = reduceManager(s, {
+ type: "runtime_signal",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ turnId: "reasoning-turn",
+ kind: "thinking",
+ phase: "inference",
+ nowMs: 112_000,
+ }).state;
+
+ expect(s.agents.a).toMatchObject({
+ status: "running",
+ stalledSessionId: null,
+ lastProgressAt: 0,
+ lastNativeActivityAt: 112_000,
+ lastNativeActivityKind: "thinking",
+ runtimePhase: "inference",
+ });
+ expect(s.agents.a.execution.lease).toMatchObject({
+ nativeDeadlineAt: 472_000,
+ recoveryExtensionsUsed: 0,
+ });
+ expect(reduceManager(s, { type: "tick", nowMs: 471_999 }).effects).toEqual([]);
+ expect(reduceManager(s, { type: "tick", nowMs: 472_000 }).effects).toEqual([{
+ type: "terminate_stalled",
+ agentId: "a",
+ recordSessionId: "sess-reasoning",
+ }]);
+ });
+
+ it("bounds recovery extensions and replenishes one only after semantic progress", () => {
+ const silence: TurnSilencePolicy = {
+ nativeIdleTimeoutMs: 100,
+ daemonGraceMs: 20,
+ recoveryGraceMs: 50,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 120,
+ };
+ let s = register(createInitialManagerState(), "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "retry" }, nowMs: 0 }).state;
+ s = spawnRootWithSilence(s, 0, silence, "retry-turn");
+
+ const recovery = (state: ManagerState, nowMs: number): ManagerState => reduceManager(state, {
+ type: "runtime_signal",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ turnId: "retry-turn",
+ kind: "recovery",
+ phase: "recovery",
+ nowMs,
+ }).state;
+
+ s = recovery(s, 110);
+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 160, recoveryExtensionsUsed: 1 });
+ s = recovery(s, 150);
+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 160, recoveryExtensionsUsed: 1 });
+ expect(reduceManager(s, { type: "tick", nowMs: 159 }).effects).toEqual([]);
+
+ s = workRoot(s, "retry-turn", 159);
+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 279, recoveryExtensionsUsed: 0 });
+ s = recovery(s, 270);
+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 320, recoveryExtensionsUsed: 1 });
+ s = recovery(s, 310);
+ expect(s.agents.a.execution.lease).toMatchObject({ nativeDeadlineAt: 320, recoveryExtensionsUsed: 1 });
+ expect(reduceManager(s, { type: "tick", nowMs: 319 }).effects).toEqual([]);
+ expect(reduceManager(s, { type: "tick", nowMs: 320 }).effects).toEqual([{
+ type: "terminate_stalled",
+ agentId: "a",
+ }]);
+ });
+
it("fences tool blockers to the current root, clears them on reset, and restores a full stale window", () => {
let s = createInitialManagerState(100);
s = register(s, "a", PERSISTENT_GATED);
@@ -293,6 +558,8 @@ describe("reduceManager — tick: stall + idle hibernation", () => {
state: "active",
identity: { sessionInstanceId: SESSION_INSTANCE, turnId: "root-turn" },
lastWorkAt: 10,
+ nativeDeadlineAt: 110,
+ recoveryExtensionsUsed: 0,
});
const resetting = s;
s = reduceManager(s, {
@@ -331,6 +598,8 @@ describe("reduceManager — tick: stall + idle hibernation", () => {
state: "active",
identity: { sessionInstanceId: SESSION_INSTANCE, turnId: "root-turn" },
lastWorkAt: 60,
+ nativeDeadlineAt: 160,
+ recoveryExtensionsUsed: 0,
});
expect(reduceManager(finished, { type: "tick", nowMs: 159 }).effects).toEqual([]);
expect(reduceManager(finished, { type: "tick", nowMs: 160 }).effects).toEqual([
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index 159eaa5bf..c4f69cfe0 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -17,6 +17,27 @@ export interface RootTerminal {
at: number;
}
+export type NativeActivityKind =
+ | "turn_started"
+ | "backend_turn_started"
+ | "thinking"
+ | "text"
+ | "tool_call"
+ | "tool_output"
+ | "internal_progress"
+ | "recovery"
+ | "turn_end";
+
+export type RuntimePhase = "idle" | "admission" | "inference" | "tool" | "recovery" | "terminal";
+
+export interface TurnSilencePolicy {
+ nativeIdleTimeoutMs: number;
+ daemonGraceMs: number;
+ recoveryGraceMs: number;
+ maxRecoveryExtensions: number;
+ normalBudgetMs: number;
+}
+
export interface PendingAdmission {
sessionInstanceId: string;
commandId: string;
@@ -30,11 +51,20 @@ export interface PendingAdmission {
export type RootLease =
| { state: "detached" }
| { state: "none"; lastTerminal: RootTerminal | null }
- | { state: "active"; identity: TurnIdentity; lastWorkAt: number; outstandingToolUses?: number }
+ | {
+ state: "active";
+ identity: TurnIdentity;
+ lastWorkAt: number;
+ nativeDeadlineAt: number;
+ recoveryExtensionsUsed: number;
+ outstandingToolUses?: number;
+ }
| {
state: "suspect_active";
identity: TurnIdentity;
lastWorkAt: number;
+ nativeDeadlineAt: number;
+ recoveryExtensionsUsed: number;
outstandingToolUses?: number;
reason: "work_after_terminal";
};
@@ -48,11 +78,18 @@ export interface AgentState {
status: AgentStatus;
inbox: AgentMsg[];
sessionId: string | null;
+ /** Backend session that has already consumed its one same-session stall recovery. */
+ stalledSessionId: string | null;
execution: ExecutionEpoch;
pendingAdmissions: PendingAdmission[];
turnId: string | null;
turnActive: boolean;
lastProgressAt: number;
+ lastNativeActivityAt: number;
+ lastNativeActivityKind: NativeActivityKind | null;
+ runtimePhase: RuntimePhase;
+ backendTurnId: string | null;
+ turnSilence: TurnSilencePolicy;
lastDeliverAt: number | null;
idleSince: number | null;
stoppingSince: number | null;
@@ -77,8 +114,14 @@ export type ManagerEvent =
| { type: "register"; agentId: string }
| { type: "wake"; agentId: string; message: AgentMsg; nowMs: number }
| { type: "spawned"; agentId: string; nowMs: number }
- | { type: "backend_session"; agentId: string; sessionId: string }
- | { type: "attach_session"; agentId: string; sessionInstanceId: string; nowMs: number }
+ | { type: "backend_session"; agentId: string; sessionId: string; stalledBefore?: boolean }
+ | {
+ type: "attach_session";
+ agentId: string;
+ sessionInstanceId: string;
+ nowMs: number;
+ turnSilence?: TurnSilencePolicy;
+ }
| {
type: "admission_started";
agentId: string;
@@ -137,7 +180,23 @@ export type ManagerEvent =
| { type: "reset_session"; agentId: string }
| { type: "begin_reset"; agentId: string; nowMs: number }
| { type: "rewake_after_reset"; agentId: string; message: AgentMsg }
- | { type: "runtime_signal"; agentId: string; kind: string; nowMs: number }
+ | {
+ type: "runtime_signal";
+ agentId: string;
+ sessionInstanceId: string;
+ turnId: string;
+ kind: NativeActivityKind;
+ phase: RuntimePhase;
+ nowMs: number;
+ backendTurnId?: string;
+ recoveryStage?: "retrying" | "recovered";
+ }
+ | {
+ type: "stall_control_failed";
+ agentId: string;
+ sessionId: string;
+ transition: "attempt" | "fence" | "clear";
+ }
| {
type: "delivery_rejected";
agentId: string;
@@ -149,12 +208,20 @@ export type ManagerEffect =
| { type: "spawn"; agentId: string; messages: AgentMsg[]; resumeSessionId: string | null }
| { type: "send"; agentId: string; message: AgentMsg; mode: "busy" | "idle" }
| { type: "stop"; agentId: string; reason: string }
- | { type: "terminate_stalled"; agentId: string }
+ | { type: "terminate_stalled"; agentId: string; recordSessionId?: string; forgetSessionId?: string }
+ | { type: "clear_stall_recovery"; agentId: string; sessionId: string }
| { type: "expire_admission"; agentId: string; sessionInstanceId: string; commandIds: string[] }
| { type: "requeue_delivery"; agentId: string; message: AgentMsg; mode: "busy" | "idle" }
| { type: "force_exit"; agentId: string; reason: string };
export const DEFAULT_STALE_THRESHOLD_MS = 120_000;
+export const DEFAULT_TURN_SILENCE_POLICY: TurnSilencePolicy = {
+ nativeIdleTimeoutMs: 300_000,
+ daemonGraceMs: 60_000,
+ recoveryGraceMs: 60_000,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 360_000,
+};
export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;
export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;
export const DEFAULT_STOPPING_STUCK_THRESHOLD_MS = 30_000;
@@ -189,6 +256,11 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
case "backend_session":
return mutate(state, event.agentId, (a) => {
+ if (event.stalledBefore) {
+ a.stalledSessionId = event.sessionId;
+ } else if (a.stalledSessionId !== null && a.stalledSessionId !== event.sessionId) {
+ a.stalledSessionId = null;
+ }
a.sessionId = event.sessionId;
});
@@ -201,7 +273,17 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
{
const a = agent;
a.execution = { sessionInstanceId: event.sessionInstanceId, lease: { state: "none", lastTerminal: null } };
+ a.turnSilence = event.turnSilence ?? {
+ ...DEFAULT_TURN_SILENCE_POLICY,
+ nativeIdleTimeoutMs: state.staleThresholdMs,
+ daemonGraceMs: 0,
+ normalBudgetMs: state.staleThresholdMs,
+ };
a.lastProgressAt = event.nowMs;
+ a.lastNativeActivityAt = event.nowMs;
+ a.lastNativeActivityKind = null;
+ a.runtimePhase = "admission";
+ a.backendTurnId = null;
a.idleSince = null;
syncExecutionProjection(a);
}
@@ -264,6 +346,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
if (!state.agents[event.agentId]) return { state, effects: [] };
return mutate(state, event.agentId, (a) => {
a.sessionId = null;
+ a.stalledSessionId = null;
});
case "begin_reset":
@@ -302,8 +385,18 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
if ((lease.state === "active" || lease.state === "suspect_active") && !sameIdentity(lease.identity, identity)) return;
const startedCommands = new Set(event.commandIds);
a.pendingAdmissions = a.pendingAdmissions.filter((entry) => !startedCommands.has(entry.commandId));
- a.execution.lease = { state: "active", identity, lastWorkAt: event.nowMs };
+ a.execution.lease = {
+ state: "active",
+ identity,
+ lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
+ };
a.lastProgressAt = event.nowMs;
+ a.lastNativeActivityAt = event.nowMs;
+ a.lastNativeActivityKind = "turn_started";
+ a.runtimePhase = "inference";
+ a.backendTurnId = null;
a.idleSince = null;
syncExecutionProjection(a);
});
@@ -325,7 +418,12 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
const lease = a.execution.lease;
const identity = identityOf(event);
if ((lease.state === "active" || lease.state === "suspect_active") && sameIdentity(lease.identity, identity)) {
- a.execution.lease = { ...lease, lastWorkAt: event.nowMs };
+ a.execution.lease = {
+ ...lease,
+ lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
+ };
} else {
const terminal = lease.state === "none" ? lease.lastTerminal : null;
if (!terminal || !sameIdentity(terminal.identity, identity)) return;
@@ -333,6 +431,8 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
state: "suspect_active",
identity,
lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
reason: "work_after_terminal",
};
}
@@ -349,7 +449,14 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
return onTurnToolLifecycle(state, event, "finished");
case "turn_completed":
- return onTurnCompleted(state, event.agentId, event.sessionInstanceId, event.nowMs, event.turnId);
+ return onTurnCompleted(
+ state,
+ event.agentId,
+ event.sessionInstanceId,
+ event.nowMs,
+ event.turnId,
+ event.endReason,
+ );
case "session_closed":
if (state.agents[event.agentId]?.execution.sessionInstanceId !== event.sessionInstanceId) {
@@ -371,8 +478,55 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
case "tick":
return onTick(state, event.nowMs);
- case "runtime_signal":
- return { state, effects: [] };
+ case "runtime_signal": {
+ const existing = state.agents[event.agentId];
+ if (!existing || existing.execution.sessionInstanceId !== event.sessionInstanceId) return { state, effects: [] };
+ const lease = existing.execution.lease;
+ const identity = { sessionInstanceId: event.sessionInstanceId, turnId: event.turnId };
+ if ((lease.state !== "active" && lease.state !== "suspect_active") || !sameIdentity(lease.identity, identity)) {
+ return { state, effects: [] };
+ }
+ return mutate(state, event.agentId, (a) => {
+ const active = a.execution.lease;
+ if ((active.state !== "active" && active.state !== "suspect_active") || !sameIdentity(active.identity, identity)) return;
+ if (event.kind === "recovery" && event.recoveryStage !== "recovered") {
+ if (active.recoveryExtensionsUsed < a.turnSilence.maxRecoveryExtensions) {
+ a.execution.lease = {
+ ...active,
+ nativeDeadlineAt: Math.max(
+ active.nativeDeadlineAt,
+ event.nowMs + a.turnSilence.recoveryGraceMs,
+ ),
+ recoveryExtensionsUsed: active.recoveryExtensionsUsed + 1,
+ };
+ }
+ } else {
+ a.execution.lease = {
+ ...active,
+ nativeDeadlineAt: event.nowMs + a.turnSilence.normalBudgetMs,
+ };
+ }
+ a.lastNativeActivityAt = event.nowMs;
+ a.lastNativeActivityKind = event.kind;
+ a.runtimePhase = event.phase;
+ if (event.backendTurnId) a.backendTurnId = event.backendTurnId;
+ });
+ }
+
+ case "stall_control_failed":
+ return mutate(state, event.agentId, (a) => {
+ if (event.transition === "clear") {
+ if (a.sessionId === event.sessionId) a.stalledSessionId = event.sessionId;
+ return;
+ }
+ if (a.status !== "stopping") return;
+ const lease = a.execution.lease;
+ if (lease.state !== "active" && lease.state !== "suspect_active") return;
+ a.status = "running";
+ a.stoppingSince = null;
+ a.sessionId = event.sessionId;
+ a.stalledSessionId = event.transition === "fence" ? event.sessionId : null;
+ });
case "delivery_rejected":
return mutate(state, event.agentId, (a) => {
@@ -424,6 +578,7 @@ function onTurnCompleted(
sessionInstanceId: string,
nowMs: number,
turnId: string,
+ endReason: "errored" | undefined,
): ReduceResult {
const existing = state.agents[agentId];
if (!existing) return { state, effects: [] };
@@ -436,14 +591,25 @@ function onTurnCompleted(
const lastTerminal = { identity, at: nowMs };
agent.execution.lease = { state: "none", lastTerminal };
agent.lastProgressAt = nowMs;
+ agent.lastNativeActivityAt = nowMs;
+ agent.lastNativeActivityKind = "turn_end";
+ agent.runtimePhase = "terminal";
+ const clearedStallSessionId = endReason === undefined ? agent.stalledSessionId : null;
+ if (clearedStallSessionId !== null) agent.stalledSessionId = null;
syncExecutionProjection(agent);
+ const clearEffects: ManagerEffect[] = clearedStallSessionId === null
+ ? []
+ : [{ type: "clear_stall_recovery", agentId, sessionId: clearedStallSessionId }];
if (agent.inbox.length > 0) {
const messages = drainInbox(agent);
- return commit(state, agent, messages.map((queued) => ({ type: "send", agentId, message: queued, mode: "idle" })));
+ return commit(state, agent, [
+ ...clearEffects,
+ ...messages.map((queued) => ({ type: "send" as const, agentId, message: queued, mode: "idle" as const })),
+ ]);
}
agent.idleSince = nowMs;
- return commit(state, agent, []);
+ return commit(state, agent, clearEffects);
}
function onTurnToolLifecycle(
@@ -475,11 +641,22 @@ function onTurnToolLifecycle(
if ((lease.state === "active" || lease.state === "suspect_active") && sameIdentity(lease.identity, identity)) {
const outstandingToolUses = (lease.outstandingToolUses ?? 0) + (lifecycle === "started" ? 1 : -1);
if (outstandingToolUses > 0) {
- agent.execution.lease = { ...lease, lastWorkAt: event.nowMs, outstandingToolUses };
+ agent.execution.lease = {
+ ...lease,
+ lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
+ outstandingToolUses,
+ };
} else {
const unblockedLease = { ...lease };
delete unblockedLease.outstandingToolUses;
- agent.execution.lease = { ...unblockedLease, lastWorkAt: event.nowMs };
+ agent.execution.lease = {
+ ...unblockedLease,
+ lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
+ };
}
} else {
const terminal = lease.state === "none" ? lease.lastTerminal : null;
@@ -488,11 +665,16 @@ function onTurnToolLifecycle(
state: "suspect_active",
identity,
lastWorkAt: event.nowMs,
+ nativeDeadlineAt: event.nowMs + agent.turnSilence.normalBudgetMs,
+ recoveryExtensionsUsed: 0,
outstandingToolUses: 1,
reason: "work_after_terminal",
};
}
agent.lastProgressAt = event.nowMs;
+ agent.lastNativeActivityAt = event.nowMs;
+ agent.lastNativeActivityKind = lifecycle === "started" ? "tool_call" : "tool_output";
+ agent.runtimePhase = lifecycle === "started" ? "tool" : "inference";
agent.idleSince = null;
syncExecutionProjection(agent);
});
@@ -505,6 +687,8 @@ function onExit(state: ManagerState, agentId: string): ReduceResult {
const effects = recoveryEffects(agent, agent.pendingAdmissions);
agent.pendingAdmissions = [];
agent.execution = { sessionInstanceId: null, lease: { state: "detached" } };
+ agent.runtimePhase = "idle";
+ agent.backendTurnId = null;
agent.stoppingSince = null;
syncExecutionProjection(agent);
@@ -530,10 +714,24 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
const stalled = a.status === "running"
&& (lease.state === "active" || lease.state === "suspect_active")
&& (lease.outstandingToolUses ?? 0) === 0
- && nowMs - lease.lastWorkAt >= state.staleThresholdMs;
+ && nowMs - lease.lastWorkAt >= a.turnSilence.normalBudgetMs
+ && nowMs >= lease.nativeDeadlineAt;
if (stalled) {
- agents[id] = { ...a, status: "stopping", idleSince: null, stoppingSince: nowMs };
- effects.push({ type: "terminate_stalled", agentId: id });
+ const repeatedSessionStall = a.sessionId !== null && a.stalledSessionId === a.sessionId;
+ const forgetSessionId = repeatedSessionStall ? a.sessionId! : undefined;
+ agents[id] = {
+ ...a,
+ status: "stopping",
+ sessionId: repeatedSessionStall ? null : a.sessionId,
+ stalledSessionId: repeatedSessionStall ? null : a.sessionId,
+ idleSince: null,
+ stoppingSince: nowMs,
+ };
+ effects.push(forgetSessionId
+ ? { type: "terminate_stalled", agentId: id, forgetSessionId }
+ : a.sessionId !== null
+ ? { type: "terminate_stalled", agentId: id, recordSessionId: a.sessionId }
+ : { type: "terminate_stalled", agentId: id });
continue;
}
@@ -600,11 +798,17 @@ function freshAgent(agentId: string): AgentState {
status: "idle",
inbox: [],
sessionId: null,
+ stalledSessionId: null,
execution: { sessionInstanceId: null, lease: { state: "detached" } },
pendingAdmissions: [],
turnId: null,
turnActive: false,
lastProgressAt: 0,
+ lastNativeActivityAt: 0,
+ lastNativeActivityKind: null,
+ runtimePhase: "idle",
+ backendTurnId: null,
+ turnSilence: DEFAULT_TURN_SILENCE_POLICY,
lastDeliverAt: null,
idleSince: null,
stoppingSince: null,
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index af3b2cb4c..76e2ca21a 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -2,6 +2,9 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { EventEmitter } from "events";
import type { ChildProcess } from "child_process";
import { PassThrough } from "stream";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
import {
AgentProcessManager,
truncateThinking,
@@ -24,6 +27,7 @@ import type { AgentBackend as Driver } from "../drivers/index.js";
import type { HostLaunchContext as LaunchContext } from "./hostContext.js";
import type { RuntimeConfig } from "../runtimeConfig.js";
import type { Logger } from "../logger.js";
+import { createTimelineRecorder } from "../timeline/recorder.js";
/** Stub logger — records calls per level for assertions. */
function stubLogger(): Logger & { calls: Record<"debug" | "info" | "warn" | "error", Array<[string, unknown[]]>> } {
@@ -299,6 +303,681 @@ function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeo
return { mgr, session, onRuntimeSpawnFailed, onRuntimeSessionEstablished };
}
+describe("AgentProcessManager — repeated backend-session stall recovery", () => {
+ it("rejects a backend session before publishing it when setSession cannot persist control", async () => {
+ const session = fakeSession("set-session-control-failure");
+ session.stop = vi.fn(session.stop.bind(session));
+ const onAgentSession = vi.fn();
+ const onRuntimeSessionEstablished = vi.fn();
+ const onRuntimeSpawnFailed = vi.fn();
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: {
+ setSession: () => false,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => null,
+ recordSessionStall: () => true,
+ clearSessionStall: () => true,
+ forgetSession: () => true,
+ },
+ onAgentSession,
+ onRuntimeSessionEstablished,
+ onRuntimeSpawnFailed,
+ });
+ manager.register("a1");
+ manager.deliver("a1", { id: "set-failure", text: "hello" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-uncommitted" });
+
+ expect(session.stop).toHaveBeenCalledWith({ reason: "shutdown", forceAfterMs: 2_000 });
+ expect(onAgentSession).not.toHaveBeenCalled();
+ expect(onRuntimeSessionEstablished).not.toHaveBeenCalled();
+ expect(onRuntimeSpawnFailed).toHaveBeenCalledWith("codex", "resume_control_update_failed");
+ expect(manager.snapshot().agents.a1.sessionId).toBeNull();
+ });
+
+ it("defers a first stall termination when the durable attempt transition fails", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const session = fakeSession("attempt-control-failure");
+ session.stop = vi.fn(session.stop.bind(session));
+ const onBotAuditEvent = vi.fn();
+ let controlWritable = false;
+ const recordSessionStall = vi.fn(() => controlWritable);
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: {
+ setSession: () => true,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => null,
+ recordSessionStall,
+ clearSessionStall: () => true,
+ forgetSession: () => true,
+ },
+ onBotAuditEvent,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ manager.start();
+ manager.register("a1");
+ manager.deliver("a1", { id: "attempt-failure", text: "hello" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(session.stop).not.toHaveBeenCalled();
+ expect(manager.snapshot().agents.a1).toMatchObject({
+ status: "running",
+ sessionId: "sess-poison",
+ stalledSessionId: null,
+ stoppingSince: null,
+ });
+ expect(onBotAuditEvent).toHaveBeenCalledWith(
+ "a1",
+ expect.objectContaining({
+ kind: "error",
+ payload: expect.objectContaining({ code: "resume_control_update_failed" }),
+ }),
+ expect.anything(),
+ );
+
+ controlWritable = true;
+ now = 105;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(recordSessionStall).toHaveBeenCalledTimes(2);
+ expect(session.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });
+ expect(manager.snapshot().agents.a1).toMatchObject({
+ status: "stopping",
+ stalledSessionId: "sess-poison",
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("defers a repeated stall termination and restores the exact session when fencing fails", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const session = fakeSession("fence-control-failure");
+ session.stop = vi.fn(session.stop.bind(session));
+ let controlWritable = false;
+ const forgetSession = vi.fn(() => controlWritable);
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: {
+ setSession: () => true,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => "sess-poison",
+ resolveResumeSession: () => ({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ }),
+ recordSessionStall: () => true,
+ clearSessionStall: () => true,
+ forgetSession,
+ },
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ manager.start();
+ manager.register("a1");
+ manager.deliver("a1", { id: "fence-failure", text: "hello" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(session.stop).not.toHaveBeenCalled();
+ expect(manager.snapshot().agents.a1).toMatchObject({
+ status: "running",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ stoppingSince: null,
+ });
+
+ controlWritable = true;
+ now = 105;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(forgetSession).toHaveBeenCalledTimes(2);
+ expect(session.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });
+ expect(manager.snapshot().agents.a1).toMatchObject({
+ status: "stopping",
+ sessionId: null,
+ stalledSessionId: null,
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("keeps the allowance consumed when its durable clear transition fails", async () => {
+ const session = fakeSession("clear-control-failure");
+ const clearSessionStall = vi.fn(() => false);
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: {
+ setSession: () => true,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => "sess-poison",
+ resolveResumeSession: () => ({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ }),
+ recordSessionStall: () => true,
+ clearSessionStall,
+ forgetSession: () => true,
+ },
+ });
+ manager.register("a1");
+ manager.deliver("a1", { id: "clear-failure", text: "hello" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+ await session.fire("runtime_event", { kind: "turn_end", sessionId: "sess-poison" });
+
+ expect(clearSessionStall).toHaveBeenCalledWith("a1", "sess-poison");
+ expect(manager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");
+ });
+
+ for (const barrierType of ["reset_session", "nap"] as const) {
+ it(`aborts ${barrierType} before spawn/stop when its control transition fails`, async () => {
+ const sessionFactory = vi.fn(() => fakeSession());
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory,
+ timeline: {
+ setSession: () => true,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => null,
+ recordSessionStall: () => true,
+ clearSessionStall: () => true,
+ forgetSession: () => false,
+ },
+ });
+ await expect(manager.resetSession("a1", {
+ runtimeConfig: {
+ version: 1,
+ runtime: "codex",
+ model: { kind: "default" },
+ mode: { kind: "default" },
+ },
+ launchId: "control-failure",
+ rewakePrompt: "rewake",
+ barrierType,
+ })).rejects.toThrow("resume control could not be persisted");
+ expect(sessionFactory).not.toHaveBeenCalled();
+ expect(manager.snapshot().agents.a1).toBeUndefined();
+ });
+ }
+
+ it("allows a genuinely different server session after an exact-session fence", async () => {
+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-new-session-"));
+ try {
+ const writer = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ });
+ writer.forgetSession("a1", "stall_recovery", "sess-poison");
+ const reader = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ });
+ let launch: LaunchContext | undefined;
+ const session = fakeSession("different-session");
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: { sessionId: "sess-poison" } as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => {
+ launch = hooks.ctx;
+ return bindFactorySession(hooks, session);
+ },
+ timeline: reader,
+ });
+ manager.register("a1", { sessionId: "sess-fresh" });
+ manager.deliver("a1", { id: "fresh-candidate", seq: 1, text: "fresh" });
+ expect(launch?.config.sessionId).toBe("sess-fresh");
+ await manager.stopAll();
+ } finally {
+ fs.rmSync(timelineDir, { recursive: true, force: true });
+ }
+ });
+
+ it("rejects a stale fenced server candidate after a newer healthy timeline session", async () => {
+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-stale-after-fresh-"));
+ try {
+ const writer = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ });
+ writer.forgetSession("a1", "stall_recovery", "sess-poison");
+ writer.setSession("a1", "sess-healthy");
+ writer.appendEntryForAgent("a1", [{
+ seq: "#1",
+ channel: "/test/general",
+ sender: "@tester#0001",
+ content: { text: "healthy" },
+ time: new Date().toISOString(),
+ }]);
+ const reader = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ });
+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-healthy",
+ stalledSessionId: null,
+ fencedSessionId: "sess-poison",
+ });
+
+ let launch: LaunchContext | undefined;
+ const session = fakeSession("fresh-after-stale");
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: { sessionId: "sess-poison" } as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => {
+ launch = hooks.ctx;
+ return bindFactorySession(hooks, session);
+ },
+ timeline: reader,
+ });
+ manager.register("a1", { sessionId: "sess-poison" });
+ manager.deliver("a1", { id: "stale-candidate", seq: 2, text: "must fresh-start" });
+ expect(launch?.config.sessionId).toBeUndefined();
+ await manager.stopAll();
+ } finally {
+ fs.rmSync(timelineDir, { recursive: true, force: true });
+ }
+ });
+
+ it("a durable clean marker replenishes the allowance after manager reconstruction", async () => {
+ vi.useFakeTimers();
+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-clean-restart-"));
+ try {
+ let now = 0;
+ const writer = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ now: () => new Date(now),
+ });
+ writer.setSession("a1", "sess-healthy");
+ writer.appendEntryForAgent("a1", [{
+ seq: "#1",
+ channel: "/test/general",
+ sender: "@tester#0001",
+ content: { text: "healthy" },
+ time: new Date(now).toISOString(),
+ }]);
+ writer.recordSessionStall("a1", "sess-healthy");
+ writer.clearSessionStall("a1", "sess-healthy");
+
+ const reader = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ now: () => new Date(now),
+ });
+ const session = fakeSession("clean-restart");
+ session.stop = vi.fn(session.stop.bind(session));
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: reader,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ manager.start();
+ manager.register("a1", { sessionId: "sess-healthy" });
+ manager.deliver("a1", { id: "healthy-retry", seq: 2, text: "next" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-healthy" });
+ expect(manager.snapshot().agents.a1.stalledSessionId).toBeNull();
+
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(manager.snapshot().agents.a1.sessionId).toBe("sess-healthy");
+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-healthy",
+ stalledSessionId: "sess-healthy",
+ fencedSessionId: null,
+ });
+ await manager.stopAll();
+ } finally {
+ fs.rmSync(timelineDir, { recursive: true, force: true });
+ vi.useRealTimers();
+ }
+ });
+
+ it("restores a marker-only allowance against a stale server/base session candidate", async () => {
+ vi.useFakeTimers();
+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-marker-only-"));
+ try {
+ let now = 0;
+ const writer = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ now: () => new Date(now),
+ });
+ writer.recordSessionStall("a1", "sess-poison");
+
+ // Rebuild the recorder with no in-memory map and no ordinary session row.
+ const reader = createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ now: () => new Date(now),
+ });
+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "none",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+ const session = fakeSession("marker-only-restart");
+ session.stop = vi.fn(session.stop.bind(session));
+ const manager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: { sessionId: "sess-poison" } as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ timeline: reader,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ manager.start();
+ manager.register("a1", { sessionId: "sess-poison" });
+ manager.deliver("a1", { id: "marker-only-wake", seq: 1, text: "resume" });
+ session.startResolver?.();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+ expect(manager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");
+
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(manager.snapshot().agents.a1.sessionId).toBeNull();
+ expect(reader.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "barrier",
+ type: "stall_recovery",
+ forgottenSessionId: "sess-poison",
+ fencedSessionId: "sess-poison",
+ });
+ await manager.stopAll();
+ } finally {
+ fs.rmSync(timelineDir, { recursive: true, force: true });
+ vi.useRealTimers();
+ }
+ });
+
+ it("restores the consumed allowance after rebuilding both manager and timeline recorder", async () => {
+ vi.useFakeTimers();
+ const timelineDir = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), "stall-restart-"));
+ try {
+ let now = 0;
+ const makeRecorder = () => createTimelineRecorder({
+ timelineDirFor: () => timelineDir,
+ providerFor: () => "codex",
+ now: () => new Date(now),
+ });
+ const firstSession = fakeSession("before-daemon-restart");
+ firstSession.stop = vi.fn(firstSession.stop.bind(firstSession));
+ const firstRecorder = makeRecorder();
+ const firstManager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, firstSession),
+ timeline: firstRecorder,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ firstManager.start();
+ firstManager.register("a1", { sessionId: "sess-poison" });
+ firstManager.deliver("a1", { id: "wake-before-restart", seq: 1, text: "first" });
+ firstSession.startResolver?.();
+ await firstSession.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+ firstRecorder.appendEntryForAgent("a1", [{
+ seq: "#1",
+ channel: "/test/general",
+ sender: "@tester#0001",
+ content: { text: "first" },
+ time: new Date(now).toISOString(),
+ }]);
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(firstRecorder.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+ await firstManager.stopAll();
+
+ // New objects model a real daemon process restart: no AgentState or
+ // recorder map survives; only the local timeline directory is reused.
+ const secondSession = fakeSession("after-daemon-restart");
+ secondSession.stop = vi.fn(secondSession.stop.bind(secondSession));
+ const secondRecorder = makeRecorder();
+ const secondManager = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: { sessionId: "sess-poison" } as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, secondSession),
+ timeline: secondRecorder,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ secondManager.start();
+ secondManager.register("a1", { sessionId: "sess-poison" });
+ secondManager.deliver("a1", { id: "wake-after-restart", seq: 2, text: "second" });
+ secondSession.startResolver?.();
+ await secondSession.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+ expect(secondManager.snapshot().agents.a1.stalledSessionId).toBe("sess-poison");
+
+ now = 200;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(secondSession.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });
+ expect(secondManager.snapshot().agents.a1.sessionId).toBeNull();
+ expect(secondRecorder.resolveResumeSession("a1", "codex")).toEqual({
+ kind: "barrier",
+ type: "stall_recovery",
+ forgottenSessionId: "sess-poison",
+ fencedSessionId: "sess-poison",
+ });
+ await secondManager.stopAll();
+ } finally {
+ fs.rmSync(timelineDir, { recursive: true, force: true });
+ vi.useRealTimers();
+ }
+ });
+
+ it("resumes once, then durably fences the poisoned session across every fallback source", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const sessions: FakeSession[] = [];
+ const launches: LaunchContext[] = [];
+ let resolution = {
+ kind: "session" as const,
+ sessionId: "sess-poison",
+ stalledSessionId: null,
+ fencedSessionId: null,
+ } as
+ | {
+ kind: "session";
+ sessionId: string;
+ stalledSessionId: string | null;
+ fencedSessionId: string | null;
+ }
+ | {
+ kind: "barrier";
+ type: "stall_recovery";
+ forgottenSessionId: string | null;
+ fencedSessionId: string | null;
+ };
+ const setSession = vi.fn();
+ const recordSessionStall = vi.fn((_agentId: string, sessionId: string) => {
+ resolution = { kind: "session", sessionId, stalledSessionId: sessionId, fencedSessionId: null };
+ });
+ const forgetSession = vi.fn((_agentId: string, type?: string, forgottenSessionId?: string) => {
+ resolution = {
+ kind: "barrier",
+ type: "stall_recovery",
+ forgottenSessionId: forgottenSessionId ?? null,
+ fencedSessionId: forgottenSessionId ?? null,
+ };
+ });
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: { sessionId: "sess-poison" } as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => {
+ launches.push(hooks.ctx);
+ const session = fakeSession(`instance-${sessions.length + 1}`);
+ session.stop = vi.fn(session.stop.bind(session));
+ sessions.push(session);
+ return bindFactorySession(hooks, session);
+ },
+ timeline: {
+ setSession,
+ appendResponseToLatest: vi.fn(),
+ resumeSessionId: () => resolution.kind === "session" ? resolution.sessionId : null,
+ resolveResumeSession: () => resolution,
+ recordSessionStall,
+ clearSessionStall: vi.fn(),
+ forgetSession,
+ },
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ });
+ mgr.start();
+ mgr.register("a1", { sessionId: "sess-poison" });
+ mgr.deliver("a1", { id: "wake-1", seq: 1, text: "first" });
+ sessions[0]!.startResolver?.();
+ await sessions[0]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(sessions[0]!.stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });
+ expect(recordSessionStall).toHaveBeenCalledWith("a1", "sess-poison");
+ expect(forgetSession).not.toHaveBeenCalled();
+
+ await sessions[0]!.fire("runtime_event", { kind: "turn_end" });
+ mgr.deliver("a1", { id: "wake-2", seq: 2, text: "second" });
+ await sessions[0]!.fire("exit", { code: 0, reason: "requested" });
+ expect(launches[1]!.config.sessionId).toBe("sess-poison");
+ sessions[1]!.startResolver?.();
+ await sessions[1]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+
+ now = 200;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(forgetSession).toHaveBeenCalledWith("a1", "stall_recovery", "sess-poison");
+ expect(mgr.snapshot().agents.a1.sessionId).toBeNull();
+
+ // A death-rattle session event from the fenced owner must not repopulate
+ // either the runtime cache or timeline after the barrier was written.
+ await sessions[1]!.fire("runtime_event", { kind: "session_init", sessionId: "sess-poison" });
+ expect(setSession).toHaveBeenCalledTimes(2);
+
+ // Model the next server wake still carrying its stale session id. The
+ // durable barrier must also beat register(), timeline, and base config.
+ mgr.register("a1", { sessionId: "sess-poison" });
+ mgr.deliver("a1", { id: "wake-3", seq: 3, text: "third" });
+ await sessions[1]!.fire("exit", { code: 0, reason: "requested" });
+ expect(launches[2]!.config.sessionId).toBeUndefined();
+ expect(sessions).toHaveLength(3);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
+
describe("AgentProcessManager — runtime health callbacks", () => {
it("ENOENT `error` followed by `exit` reports the failure ONCE with the specific code", async () => {
const { mgr, session, onRuntimeSpawnFailed } = makeManager();
@@ -751,6 +1430,159 @@ describe("AgentProcessManager — session race conditions", () => {
}
});
+ it("does not let turn-scoped telemetry or diagnostics refresh the native silence deadline", async () => {
+ vi.useFakeTimers();
+ try {
+ let currentTime = 0;
+ const { mgr, session } = makeManager({ now: () => currentTime, tickIntervalMs: 5 });
+ const baseSnapshot = session.snapshot.bind(session);
+ session.snapshot = () => ({
+ ...baseSnapshot(),
+ diagnostics: {
+ deliveryPhase: "working",
+ turnSilence: {
+ nativeIdleTimeoutMs: 80,
+ daemonGraceMs: 20,
+ recoveryGraceMs: 50,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 100,
+ },
+ metrics: {
+ physicalOpenCount: 1,
+ turnCount: 1,
+ commandAdmissionCount: 1,
+ commandAdmissionLatencyTotalMs: 0,
+ queueDwellCount: 0,
+ queueDwellTotalMs: 0,
+ sseReconnectCount: 0,
+ resumeOutcome: "not_requested",
+ terminalOwnerKind: "transport_request",
+ },
+ },
+ });
+ const stop = vi.spyOn(session, "stop");
+ mgr.start();
+ mgr.deliver("a1", { id: "telemetry-stall", seq: 1, text: "hello" });
+ session.startResolver?.();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ currentTime = 90;
+ await session.pushAgentEvent({
+ type: "diagnostic",
+ turnId: "test-turn",
+ severity: "info",
+ source: "heartbeat",
+ message: "still here",
+ } as never);
+ await session.pushAgentEvent({
+ type: "rate_limits",
+ turnId: "test-turn",
+ source: "account",
+ details: { remaining: 1 },
+ } as never);
+ await session.pushAgentEvent({
+ type: "token_usage",
+ turnId: "test-turn",
+ source: "account",
+ usage: {},
+ details: { sampled: true },
+ } as never);
+
+ expect(mgr.snapshot().agents.a1).toMatchObject({
+ lastNativeActivityAt: 0,
+ lastNativeActivityKind: "turn_started",
+ });
+ currentTime = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(stop).toHaveBeenCalledWith({ reason: "stalled", forceAfterMs: 2_000 });
+ expect(mgr.snapshot().agents.a1.status).toBe("stopping");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("maps backend retry activity to one bounded recovery extension", async () => {
+ let currentTime = 0;
+ const { mgr, session } = makeManager({ now: () => currentTime });
+ const baseSnapshot = session.snapshot.bind(session);
+ session.snapshot = () => ({
+ ...baseSnapshot(),
+ diagnostics: {
+ deliveryPhase: "working",
+ turnSilence: {
+ nativeIdleTimeoutMs: 80,
+ daemonGraceMs: 20,
+ recoveryGraceMs: 50,
+ maxRecoveryExtensions: 1,
+ normalBudgetMs: 100,
+ },
+ metrics: {
+ physicalOpenCount: 1,
+ turnCount: 1,
+ commandAdmissionCount: 1,
+ commandAdmissionLatencyTotalMs: 0,
+ queueDwellCount: 0,
+ queueDwellTotalMs: 0,
+ sseReconnectCount: 0,
+ resumeOutcome: "not_requested",
+ terminalOwnerKind: "transport_request",
+ },
+ },
+ });
+ mgr.deliver("a1", { id: "retry", seq: 1, text: "hello" });
+ session.startResolver?.();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ currentTime = 90;
+ await session.pushAgentEvent({
+ type: "recovery",
+ turnId: "test-turn",
+ stage: "retrying",
+ source: "codex_stream",
+ } as never);
+ expect(mgr.snapshot().agents.a1).toMatchObject({
+ lastNativeActivityAt: 90,
+ lastNativeActivityKind: "recovery",
+ runtimePhase: "recovery",
+ });
+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({
+ nativeDeadlineAt: 140,
+ recoveryExtensionsUsed: 1,
+ });
+
+ currentTime = 135;
+ await session.pushAgentEvent({
+ type: "recovery",
+ turnId: "test-turn",
+ stage: "recovered",
+ source: "pi_auto_retry",
+ } as never);
+ expect(mgr.snapshot().agents.a1).toMatchObject({
+ lastNativeActivityAt: 135,
+ lastNativeActivityKind: "recovery",
+ runtimePhase: "inference",
+ });
+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({
+ nativeDeadlineAt: 235,
+ recoveryExtensionsUsed: 1,
+ });
+
+ currentTime = 200;
+ await session.pushAgentEvent({
+ type: "recovery",
+ turnId: "test-turn",
+ stage: "retrying",
+ source: "codex_stream",
+ } as never);
+ expect(mgr.snapshot().agents.a1.execution.lease).toMatchObject({
+ nativeDeadlineAt: 235,
+ recoveryExtensionsUsed: 1,
+ });
+ await mgr.stopAll();
+ });
+
it("turn-correlated internal progress renews the execution lease after a false terminal", async () => {
vi.useFakeTimers();
try {
@@ -961,6 +1793,63 @@ describe("AgentProcessManager — session race conditions", () => {
});
describe("AgentProcessManager — onAgentActivity (derived activity reporting)", () => {
+ it.each(["turn-before-spawn", "spawn-before-turn"] as const)(
+ "publishes one running edge without a transient idle when admission order is %s",
+ async (order) => {
+ const session = fakeSession(`activity-${order}`);
+ const onAgentActivity = vi.fn();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: (hooks) => bindFactorySession(hooks, session),
+ onAgentActivity,
+ });
+ const dispatch = (event: Record<string, unknown>) => (
+ mgr as unknown as { dispatch(value: unknown): void }
+ ).dispatch(event);
+
+ mgr.register("a1");
+ mgr.deliver("a1", { id: "root", text: "hello" });
+ expect(onAgentActivity.mock.calls.map((call) => call[0])).toEqual([
+ { agentId: "a1", state: "starting" },
+ ]);
+
+ const turnStarted = {
+ type: "turn_started",
+ agentId: "a1",
+ sessionInstanceId: session.sessionInstanceId,
+ turnId: "root-turn",
+ commandIds: ["root"],
+ nowMs: 1,
+ };
+ if (order === "turn-before-spawn") {
+ dispatch(turnStarted);
+ dispatch({ type: "spawned", agentId: "a1", nowMs: 2 });
+ } else {
+ dispatch({ type: "spawned", agentId: "a1", nowMs: 1 });
+ dispatch({
+ type: "admission_settled",
+ agentId: "a1",
+ sessionInstanceId: session.sessionInstanceId,
+ commandId: "root",
+ outcome: "accepted",
+ });
+ dispatch(turnStarted);
+ }
+
+ expect(onAgentActivity.mock.calls.map((call) => call[0])).toEqual([
+ { agentId: "a1", state: "starting" },
+ { agentId: "a1", state: "running" },
+ ]);
+ },
+ );
+
it("fires exactly once per real derived transition — the turn_end→idle transition fires once, not re-fired while the FSM stays running until hibernation", async () => {
vi.useFakeTimers();
try {
@@ -3434,7 +4323,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {
expect(closeFailureReported).toHaveBeenCalledWith("codex", "session_closed_rejected");
});
- it("maps all public session event families into backend-neutral runtime signals", async () => {
+ it("maps only turn activity into runtime signals and excludes telemetry/diagnostics", async () => {
const rows: B1TraceRow[] = [];
const session = b1Session([]);
const { mgr } = b1Manager({ sessions: [session], trace: (row) => rows.push(row) });
@@ -3458,6 +4347,8 @@ describe("B1 red gate — exact-once terminal matrix", () => {
publish({ type: "diagnostic", severity: "warning", source: "codex", message: "warning" });
publish({ type: "token_usage", turnId: "test-turn", source: "codex", usage: {}, details: {} });
publish({ type: "rate_limits", turnId: "test-turn", source: "codex", details: {} });
+ publish({ type: "thinking_delta", turnId: "test-turn", text: "" });
+ publish({ type: "text_delta", turnId: "test-turn", text: "" });
publish({ type: "command_queued", commandId: "queued", reason: "runtime_busy" });
publish({ type: "command_accepted", commandId: "accepted", turnId: "test-turn", delivery: "steer" });
publish({
@@ -3467,7 +4358,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {
});
publish({ type: "turn_started", turnId: "test-turn", commandIds: ["accepted"] });
- expect(rows.filter((row) => row.event === "runtime_signal")).toHaveLength(baselineSignals + 9);
+ expect(rows.filter((row) => row.event === "runtime_signal")).toHaveLength(baselineSignals + 4);
});
it("keeps tail telemetry observational while turn-correlated work can recover a false terminal", async () => {
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index 2c76cf8b5..8b30ad2e7 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -8,6 +8,8 @@ import {
type AgentMsg,
type AgentState,
type AgentStatus,
+ type NativeActivityKind,
+ type RuntimePhase,
isActivelyWorking,
} from "./managerPolicy.js";
import type {
@@ -25,6 +27,8 @@ import { scrubRuntimeErrorDiagnosticText } from "../runtime/errorDiagnostics.js"
import { buildCliSystemPrompt } from "../drivers/systemPrompt.js";
import { createLogger, type Logger } from "../logger.js";
import { nowLocalISO } from "../util/localTime.js";
+import type { ResumeSessionResolution } from "../timeline/timeline.js";
+import type { SystemEntryType } from "../timeline/types.js";
import { randomUUID } from "node:crypto";
const SESSION_STOP_GRACE_MS = 2_000;
export type AgentActivityState = "idle" | "starting" | "running" | "stopping";
@@ -71,6 +75,8 @@ interface ActiveSpawnState {
handshakeTimer: ReturnType<typeof setTimeout> | null;
torndown: boolean;
superseded: boolean;
+ discardEvents: boolean;
+ stalledSessionIdAtLaunch: string | null;
spawnFailureReason: string | null;
terminationSemantics: string | null;
spawnOrdinal: number;
@@ -101,6 +107,13 @@ interface FsmTraceRecord extends TraceRecordBase, Partial<TraceSpanMetadata> {
inbox: number;
lastDeliverAt: number | null;
lastProgressAt: number;
+ lastNativeActivityAt: number;
+ lastNativeActivityKind: NativeActivityKind | null;
+ runtimePhase: RuntimePhase;
+ backendTurnId: string | null;
+ turnSilenceBudgetMs: number;
+ nativeDeadlineAt: number | null;
+ recoveryExtensionsUsed: number;
idleSince: number | null;
resetting: boolean;
resettingSince: number | null;
@@ -116,6 +129,7 @@ interface FsmTraceRecord extends TraceRecordBase, Partial<TraceSpanMetadata> {
resumeOutcome?: "not_requested" | "pending" | "resumed" | "reset_required" | "failed";
terminalOwnerKind?: "transport_request" | "vendor_message" | "prompt_invocation" | "lane_generation";
sinceProgressMs: number;
+ sinceNativeActivityMs: number;
sinceDeliverMs: number | null;
sinceStoppingMs: number | null;
endReason?: "errored";
@@ -232,10 +246,13 @@ export interface ManagerRuntimeOpts {
logger?: Logger;
}
export interface TimelineRecorder {
- setSession(agentId: string, sessionId: string): void;
+ setSession(agentId: string, sessionId: string): boolean;
appendResponseToLatest(agentId: string, text: string): void;
resumeSessionId(agentId: string, provider: string | null): string | null;
- forgetSession(agentId: string, barrierType?: "reset_session" | "nap"): void;
+ resolveResumeSession?(agentId: string, provider: string | null): ResumeSessionResolution;
+ recordSessionStall?(agentId: string, sessionId: string): boolean;
+ clearSessionStall?(agentId: string, sessionId: string): boolean;
+ forgetSession(agentId: string, barrierType?: SystemEntryType, forgottenSessionId?: string): boolean;
}
const THINKING_MAX_BYTES = 4096;
const AUDIT_ERROR_MESSAGE_MAX_LEN = 2000;
@@ -407,6 +424,7 @@ export class AgentProcessManager {
private readonly liveSessions = new Map<string, string>();
private readonly thinkingBuffers = new Map<string, string>();
private readonly activeSpawnState = new Map<string, ActiveSpawnState>();
+ private readonly publishedAgentActivity = new Map<string, AgentActivityState>();
private readonly traceProcessNonce = randomUUID();
private nextSpawnOrdinal = 1;
private nextDaemonTurnOrdinal = 1;
@@ -492,11 +510,21 @@ export class AgentProcessManager {
const effects = this.dispatch({ type: "wake", agentId, message: normalized, nowMs: this.now() });
return effects.length > 0;
}
- forgetSession(agentId: string, barrierType: "reset_session" | "nap" = "reset_session"): void {
+ forgetSession(agentId: string, barrierType: "reset_session" | "nap" = "reset_session"): boolean {
+ if (!this.forgetSessionSources(agentId, barrierType)) return false;
+ this.dispatch({ type: "reset_session", agentId });
+ return true;
+ }
+ private forgetSessionSources(
+ agentId: string,
+ barrierType: SystemEntryType,
+ forgottenSessionId?: string,
+ ): boolean {
+ const persisted = this.opts.timeline?.forgetSession(agentId, barrierType, forgottenSessionId);
+ if (persisted === false) return false;
this.resumeSessions.delete(agentId);
this.liveSessions.delete(agentId);
- this.dispatch({ type: "reset_session", agentId });
- this.opts.timeline?.forgetSession(agentId, barrierType);
+ return true;
}
enqueueRewake(agentId: string, message: AgentMsg): void {
this.dispatch({ type: "rewake_after_reset", agentId, message });
@@ -537,8 +565,12 @@ export class AgentProcessManager {
opName: string;
},
): Promise<void> {
+ if (opts.forgetSession && !this.forgetSession(agentId, opts.barrierType ?? "reset_session")) {
+ this.log.error("resume control transition failed; reset aborted", { agentId, barrierType: opts.barrierType });
+ this.emitErrorAudit(agentId, "reset", "resume_control_update_failed", "Reset aborted because resume control could not be persisted");
+ throw new Error("Reset aborted because resume control could not be persisted");
+ }
this.register(agentId, { runtimeConfig: opts.runtimeConfig, launchId: opts.launchId });
- if (opts.forgetSession) this.forgetSession(agentId, opts.barrierType ?? "reset_session");
this.abortCurrentTurn(agentId, opts.abortCause);
this.markResetting(agentId);
const status = this.state.agents[agentId]?.status;
@@ -753,6 +785,19 @@ export class AgentProcessManager {
inbox: a.inbox.length,
lastDeliverAt: a.lastDeliverAt,
lastProgressAt: a.lastProgressAt,
+ lastNativeActivityAt: a.lastNativeActivityAt,
+ lastNativeActivityKind: a.lastNativeActivityKind,
+ runtimePhase: a.runtimePhase,
+ backendTurnId: a.backendTurnId,
+ turnSilenceBudgetMs: a.turnSilence.normalBudgetMs,
+ nativeDeadlineAt:
+ a.execution.lease.state === "active" || a.execution.lease.state === "suspect_active"
+ ? a.execution.lease.nativeDeadlineAt
+ : null,
+ recoveryExtensionsUsed:
+ a.execution.lease.state === "active" || a.execution.lease.state === "suspect_active"
+ ? a.execution.lease.recoveryExtensionsUsed
+ : 0,
idleSince: a.idleSince,
resetting: a.resetting,
resettingSince: a.resettingSince,
@@ -763,6 +808,7 @@ export class AgentProcessManager {
timeIso: new Date(nowMs).toISOString(),
...(activeSpan ? activeSpan : {}),
sinceProgressMs: nowMs - a.lastProgressAt,
+ sinceNativeActivityMs: nowMs - a.lastNativeActivityAt,
sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,
sinceStoppingMs: a.stoppingSince === null ? null : nowMs - a.stoppingSince,
...(event.type === "turn_completed" && (event as { endReason?: "errored" }).endReason === "errored"
@@ -820,14 +866,20 @@ export class AgentProcessManager {
}
if (
this.opts.onAgentActivity
- && event.type !== "spawned"
&& event.type !== "admission_started"
&& event.type !== "admission_settled"
) {
const after = this.deriveActivitySnapshot(state);
for (const [agentId, activity] of Object.entries(after)) {
- if (agentId in before && before[agentId] !== activity) {
+ if (!(agentId in before)) {
+ this.publishedAgentActivity.set(agentId, activity);
+ continue;
+ }
+ const previouslyPublished = this.publishedAgentActivity.get(agentId) ?? before[agentId];
+ const publishable = event.type !== "spawned" || activity === "running";
+ if (publishable && previouslyPublished !== activity) {
this.opts.onAgentActivity({ agentId, state: activity });
+ this.publishedAgentActivity.set(agentId, activity);
}
}
}
@@ -1022,6 +1074,56 @@ export class AgentProcessManager {
case "terminate_stalled": {
const session = this.sessions.get(effect.agentId);
const spawnState = this.activeSpawnState.get(effect.agentId);
+ const endedSessionId = this.liveSessions.get(effect.agentId) ?? "";
+ if (effect.type === "terminate_stalled" && effect.recordSessionId) {
+ const persisted = this.opts.timeline?.recordSessionStall?.(effect.agentId, effect.recordSessionId);
+ if (persisted === false) {
+ this.dispatch({
+ type: "stall_control_failed",
+ agentId: effect.agentId,
+ sessionId: effect.recordSessionId,
+ transition: "attempt",
+ });
+ this.log.error("stall recovery attempt was not persisted; termination deferred", {
+ agentId: effect.agentId,
+ sessionId: effect.recordSessionId,
+ });
+ this.emitErrorAudit(
+ effect.agentId,
+ "runtime",
+ "resume_control_update_failed",
+ "Stall termination deferred because the recovery attempt could not be persisted",
+ );
+ break;
+ }
+ }
+ if (effect.type === "terminate_stalled" && effect.forgetSessionId) {
+ const persisted = this.forgetSessionSources(effect.agentId, "stall_recovery", effect.forgetSessionId);
+ if (!persisted) {
+ this.dispatch({
+ type: "stall_control_failed",
+ agentId: effect.agentId,
+ sessionId: effect.forgetSessionId,
+ transition: "fence",
+ });
+ this.log.error("repeated-session fence was not persisted; termination deferred", {
+ agentId: effect.agentId,
+ sessionId: effect.forgetSessionId,
+ });
+ this.emitErrorAudit(
+ effect.agentId,
+ "runtime",
+ "resume_control_update_failed",
+ "Stall termination deferred because the exact session fence could not be persisted",
+ );
+ break;
+ }
+ if (spawnState) spawnState.discardEvents = true;
+ this.log.warn("repeatedly stalled backend session fenced", {
+ agentId: effect.agentId,
+ sessionId: effect.forgetSessionId,
+ });
+ }
if (effect.type === "terminate_stalled" && spawnState) {
this.closeTurn(spawnState, spawnState.activeSpan, {
event: "turn_abort",
@@ -1039,10 +1141,36 @@ export class AgentProcessManager {
if (spawnState) {
spawnState.terminationSemantics = effect.type === "terminate_stalled" ? "killed_stalled" : "idle_stop";
}
- this.logSessionEnded(effect.agentId, effect.type === "stop" ? "stopped" : "terminate_stalled");
+ this.logSessionEnded(
+ effect.agentId,
+ effect.type === "stop" ? "stopped" : "terminate_stalled",
+ endedSessionId,
+ );
this.opts.onAgentLocallyStopped?.({ agentId: effect.agentId, reason: effect.type });
break;
}
+ case "clear_stall_recovery": {
+ const persisted = this.opts.timeline?.clearSessionStall?.(effect.agentId, effect.sessionId);
+ if (persisted === false) {
+ this.dispatch({
+ type: "stall_control_failed",
+ agentId: effect.agentId,
+ sessionId: effect.sessionId,
+ transition: "clear",
+ });
+ this.log.error("stall recovery clear was not persisted; allowance remains consumed", {
+ agentId: effect.agentId,
+ sessionId: effect.sessionId,
+ });
+ this.emitErrorAudit(
+ effect.agentId,
+ "runtime",
+ "resume_control_update_failed",
+ "Stall recovery allowance remains consumed because its clear could not be persisted",
+ );
+ }
+ break;
+ }
case "expire_admission": {
const owner = this.activeSpawnState.get(effect.agentId);
if (!owner || owner.sessionInstanceId !== effect.sessionInstanceId) break;
@@ -1095,8 +1223,12 @@ export class AgentProcessManager {
}
}
}
- private logSessionEnded(agentId: string, reason: "turn_end" | "stopped" | "terminate_stalled" | "exit"): void {
- this.log.info("agent session ended", { agentId, sessionId: this.liveSessions.get(agentId) ?? "", reason });
+ private logSessionEnded(
+ agentId: string,
+ reason: "turn_end" | "stopped" | "terminate_stalled" | "exit",
+ sessionId = this.liveSessions.get(agentId) ?? "",
+ ): void {
+ this.log.info("agent session ended", { agentId, sessionId, reason });
}
private doSpawn(agentId: string, messages: AgentMsg[], resumeSessionId: string | null): void {
const [first, ...pending] = messages;
@@ -1120,11 +1252,31 @@ export class AgentProcessManager {
mode: { kind: "default" },
} as RuntimeConfig);
const provider = runtimeConfig.runtime;
- const sessionId =
+ const timelineResolution = this.opts.timeline?.resolveResumeSession?.(agentId, provider);
+ const timelineSessionId = timelineResolution?.kind === "session"
+ ? timelineResolution.sessionId
+ : timelineResolution === undefined
+ ? this.opts.timeline?.resumeSessionId(agentId, provider)
+ : null;
+ const candidateSessionId =
resumeSessionId ??
this.resumeSessions.get(agentId) ??
- this.opts.timeline?.resumeSessionId(agentId, provider) ??
+ timelineSessionId ??
base.config?.sessionId;
+ const blockedByBarrier = timelineResolution?.kind === "barrier"
+ && (
+ timelineResolution.type !== "stall_recovery"
+ || timelineResolution.forgottenSessionId === null
+ || timelineResolution.forgottenSessionId === candidateSessionId
+ );
+ const blockedByExactFence = candidateSessionId !== undefined
+ && timelineResolution?.fencedSessionId === candidateSessionId;
+ const sessionId = blockedByBarrier || blockedByExactFence ? undefined : candidateSessionId;
+ const stalledSessionIdAtLaunch = timelineResolution !== undefined
+ && (timelineResolution.kind === "session" || timelineResolution.kind === "none")
+ && timelineResolution.stalledSessionId === sessionId
+ ? timelineResolution.stalledSessionId
+ : null;
const description = runtimeConfig.instruction ?? base.config?.description ?? runtimeConfig.agentName;
const agentName = runtimeConfig.agentName ?? base.config?.agentName;
const agentHandle = runtimeConfig.agentHandle ?? base.config?.agentHandle;
@@ -1155,6 +1307,8 @@ export class AgentProcessManager {
handshakeTimer: null,
torndown: false,
superseded: false,
+ discardEvents: false,
+ stalledSessionIdAtLaunch,
spawnFailureReason: null,
terminationSemantics: null,
spawnOrdinal: this.nextSpawnOrdinal++,
@@ -1237,10 +1391,23 @@ export class AgentProcessManager {
};
const onEvent = (event: ManagedEvent) => {
if (state.torndown) return;
+ if (state.discardEvents) {
+ this.log.warn("ignored event from discarded backend session owner", { agentId, event: event.type });
+ return;
+ }
if (event.type === "session_failed" && !state.hasEstablished) {
reportSpawnFailure(event.error.code || "failed_to_start", { message: event.error.message });
}
if (event.type === "session_started") {
+ const persisted = this.opts.timeline?.setSession(agentId, event.backendSessionId);
+ if (persisted === false) {
+ state.discardEvents = true;
+ reportSpawnFailure("resume_control_update_failed", {
+ message: "Backend session rejected because resume control could not be persisted",
+ });
+ void state.session?.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });
+ return;
+ }
state.hasEstablished = true;
clearHandshakeTimer();
this.opts.onRuntimeSessionEstablished?.(driver.id);
@@ -1260,6 +1427,7 @@ export class AgentProcessManager {
agentId,
sessionInstanceId: session.sessionInstanceId,
nowMs: this.now(),
+ turnSilence: session.snapshot().diagnostics?.turnSilence,
}, state);
previousOwner?.pendingDeliverySpans.clear();
void (async () => {
@@ -1501,9 +1669,13 @@ export class AgentProcessManager {
}
}
if (event.type === "session_started") {
- this.dispatch({ type: "backend_session", agentId, sessionId: event.backendSessionId }, owner);
+ this.dispatch({
+ type: "backend_session",
+ agentId,
+ sessionId: event.backendSessionId,
+ stalledBefore: owner.stalledSessionIdAtLaunch === event.backendSessionId,
+ }, owner);
this.liveSessions.set(agentId, event.backendSessionId);
- this.opts.timeline?.setSession(agentId, event.backendSessionId);
this.opts.onAgentSession?.({
agentId,
sessionId: event.backendSessionId,
@@ -1560,26 +1732,59 @@ export class AgentProcessManager {
}, owner);
if (!wasActive && this.state.agents[agentId]?.turnActive && !owner.activeSpan) this.openTurn(owner);
}
- const signalKind = (() => {
+ const nativeSignal = (() => {
switch (event.type) {
- case "session_started": return "session_init";
- case "thinking_delta": return "thinking";
- case "text_delta": return "text";
- case "tool_started": return "tool_call";
- case "tool_finished": return "tool_output";
- case "diagnostic": return "runtime_diagnostic";
- case "token_usage":
- case "rate_limits": return "telemetry";
- case "turn_completed": return "turn_end";
- case "session_failed": return "error";
- case "command_queued":
- case "command_accepted":
- case "command_failed":
- case "turn_started": return "internal_progress";
- default: return event.type;
+ case "turn_started":
+ return { kind: "turn_started" as const, phase: "inference" as const, turnId: event.turnId };
+ case "backend_turn_started":
+ return {
+ kind: "backend_turn_started" as const,
+ phase: "inference" as const,
+ turnId: event.turnId,
+ backendTurnId: event.backendTurnId,
+ };
+ case "thinking_delta":
+ return { kind: "thinking" as const, phase: "inference" as const, turnId: event.turnId };
+ case "text_delta":
+ return event.text.length > 0
+ ? { kind: "text" as const, phase: "inference" as const, turnId: event.turnId }
+ : null;
+ case "tool_started":
+ return { kind: "tool_call" as const, phase: "tool" as const, turnId: event.turnId };
+ case "tool_finished":
+ return { kind: "tool_output" as const, phase: "inference" as const, turnId: event.turnId };
+ case "compaction_started":
+ case "compaction_finished":
+ case "review_started":
+ case "review_finished":
+ case "internal_progress":
+ return event.turnId
+ ? { kind: "internal_progress" as const, phase: "inference" as const, turnId: event.turnId }
+ : null;
+ case "recovery":
+ return event.turnId
+ ? {
+ kind: "recovery" as const,
+ phase: event.stage === "retrying" ? "recovery" as const : "inference" as const,
+ recoveryStage: event.stage,
+ turnId: event.turnId,
+ }
+ : null;
+ case "turn_completed":
+ return { kind: "turn_end" as const, phase: "terminal" as const, turnId: event.turnId };
+ default:
+ return null;
}
})();
- this.dispatch({ type: "runtime_signal", agentId, kind: signalKind, nowMs: this.now() }, owner);
+ if (nativeSignal) {
+ this.dispatch({
+ type: "runtime_signal",
+ agentId,
+ sessionInstanceId: event.sessionInstanceId,
+ ...nativeSignal,
+ nowMs: this.now(),
+ }, owner);
+ }
if (event.type === "turn_completed") {
this.logSessionEnded(agentId, "turn_end");
const marker = this.nonCleanEndMarker.get(agentId);
diff --git a/src/daemon/src/manager/managerTimeline.test.ts b/src/daemon/src/manager/managerTimeline.test.ts
index 6e9f712ef..830ad4043 100644
--- a/src/daemon/src/manager/managerTimeline.test.ts
+++ b/src/daemon/src/manager/managerTimeline.test.ts
@@ -150,7 +150,7 @@ describe("manager.resetSession", () => {
mode: { kind: "default" as const },
};
- it("idle path: register → forgetSession → deliver rewake; spawn fires with sessionId=undefined and rewake prompt", async () => {
+ it("idle path: durable forget → register → deliver rewake; spawn fires with sessionId=undefined and rewake prompt", async () => {
const { rec, calls } = fakeRecorder();
const cap: { ctx?: LaunchContext } = {};
const { mgr } = manager(rec, cap);
diff --git a/src/daemon/src/timeline/recorder.test.ts b/src/daemon/src/timeline/recorder.test.ts
index f8fd868df..006fe88e0 100644
--- a/src/daemon/src/timeline/recorder.test.ts
+++ b/src/daemon/src/timeline/recorder.test.ts
@@ -1,5 +1,7 @@
import { describe, it, expect, afterEach, vi } from "vitest";
import * as fs from "fs";
+import nodeFs from "node:fs";
+import { syncBuiltinESMExports } from "node:module";
import * as os from "os";
import * as path from "path";
import { createTimelineRecorder } from "./recorder";
@@ -146,6 +148,276 @@ describe("createTimelineRecorder (append-only, 4-field schema)", () => {
});
describe("forgetSession — inline system row", () => {
+ it("keeps resume resolution at a fixed open-file bound with 1,000 historical day files", () => {
+ const dir = mkDir();
+ const emptyTurn = `${JSON.stringify(createTimelineEntry({ messages: [] }))}\n`;
+ const historicalStart = new Date("2000-01-01T12:00:00Z");
+ for (let index = 0; index < 1_000; index++) {
+ const day = new Date(historicalStart);
+ day.setUTCDate(day.getUTCDate() + index);
+ fs.writeFileSync(path.join(dir, filenameForDate(day)), emptyTurn);
+ }
+ for (let index = 0; index < 7; index++) {
+ const day = new Date(NOW());
+ day.setDate(day.getDate() - index);
+ fs.writeFileSync(path.join(dir, filenameForDate(day)), emptyTurn);
+ }
+ fs.writeFileSync(path.join(dir, ".resume-control.json"), JSON.stringify({
+ version: 1,
+ attemptedSessionId: null,
+ fencedSessionId: null,
+ fullBarrier: null,
+ }) + "\n");
+
+ const originalOpenSync = nodeFs.openSync;
+ const open = vi.fn(originalOpenSync);
+ nodeFs.openSync = open as typeof nodeFs.openSync;
+ syncBuiltinESMExports();
+ try {
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "none",
+ stalledSessionId: null,
+ fencedSessionId: null,
+ });
+ expect(open).toHaveBeenCalledTimes(8);
+ } finally {
+ nodeFs.openSync = originalOpenSync;
+ syncBuiltinESMExports();
+ }
+ });
+
+ it("uses recent history only when control state is missing and fails closed for corrupt control inputs", () => {
+ const makeSessionHistory = (): { dir: string; controlPath: string } => {
+ const dir = mkDir();
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ rec.setSession("agent_1", "sess-existing");
+ rec.appendEntryForAgent("agent_1", [msg("#1", "existing")]);
+ return { dir, controlPath: path.join(dir, ".resume-control.json") };
+ };
+ const resolve = (dir: string) => createTimelineRecorder({
+ timelineDirFor: () => dir,
+ providerFor: () => "codex",
+ now: NOW,
+ }).resolveResumeSession("agent_1", "codex");
+
+ const missing = makeSessionHistory();
+ fs.unlinkSync(missing.controlPath);
+ expect(resolve(missing.dir)).toEqual({
+ kind: "session",
+ sessionId: "sess-existing",
+ stalledSessionId: null,
+ fencedSessionId: null,
+ });
+
+ const corrupt = makeSessionHistory();
+ fs.writeFileSync(corrupt.controlPath, "{not-json}\n");
+ expect(resolve(corrupt.dir)).toEqual({
+ kind: "barrier",
+ type: "reset_session",
+ forgottenSessionId: null,
+ fencedSessionId: null,
+ });
+
+ const oversized = makeSessionHistory();
+ fs.writeFileSync(oversized.controlPath, "x".repeat(4_097));
+ expect(resolve(oversized.dir)).toEqual({
+ kind: "barrier",
+ type: "reset_session",
+ forgottenSessionId: null,
+ fencedSessionId: null,
+ });
+
+ const nonRegular = makeSessionHistory();
+ fs.unlinkSync(nonRegular.controlPath);
+ fs.mkdirSync(nonRegular.controlPath);
+ expect(resolve(nonRegular.dir)).toEqual({
+ kind: "barrier",
+ type: "reset_session",
+ forgottenSessionId: null,
+ fencedSessionId: null,
+ });
+
+ if (process.platform !== "win32") {
+ const linked = makeSessionHistory();
+ const outside = path.join(mkDir(), "outside-control.json");
+ fs.writeFileSync(outside, JSON.stringify({
+ version: 1,
+ attemptedSessionId: null,
+ fencedSessionId: null,
+ fullBarrier: null,
+ }));
+ fs.unlinkSync(linked.controlPath);
+ fs.symlinkSync(outside, linked.controlPath);
+ expect(resolve(linked.dir)).toEqual({
+ kind: "barrier",
+ type: "reset_session",
+ forgottenSessionId: null,
+ fencedSessionId: null,
+ });
+ }
+ });
+
+ it("persists and clears the one-stall recovery allowance", () => {
+ const dir = mkDir();
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ rec.setSession("agent_1", "sess-poison");
+ rec.appendEntryForAgent("agent_1", [msg("#1", "before stall")]);
+
+ rec.recordSessionStall("agent_1", "sess-poison");
+ const controlPath = path.join(dir, ".resume-control.json");
+ const controlStat = fs.statSync(controlPath);
+ expect(controlStat.isFile()).toBe(true);
+ if (process.platform !== "win32") expect(controlStat.mode & 0o777).toBe(0o600);
+ expect(fs.readdirSync(dir).filter((name) => name.includes("resume-control") && name.endsWith(".tmp"))).toEqual([]);
+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+
+ rec.clearSessionStall("agent_1", "sess-poison");
+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: null,
+ fencedSessionId: null,
+ });
+ });
+
+ for (const failureMode of ["lock", "rename"] as const) {
+ for (const transition of ["attempt", "fence", "reset_session", "nap", "clear"] as const) {
+ it(`does not append a ${transition} forensic row when the control ${failureMode} transition fails`, () => {
+ const dir = mkDir();
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ expect(rec.setSession("agent_1", "sess-poison")).toBe(true);
+ rec.appendEntryForAgent("agent_1", [msg("#1", "before transition")]);
+ if (transition === "fence" || transition === "clear") {
+ expect(rec.recordSessionStall("agent_1", "sess-poison")).toBe(true);
+ }
+
+ const controlPath = path.join(dir, ".resume-control.json");
+ const beforeControl = fs.readFileSync(controlPath, "utf8");
+ const beforeSystemRows = readRecentEntries(dir, { now: NOW() })
+ .filter((row) => row.system)
+ .map((row) => row.system);
+ const lockPath = lockPathFor(dir, ".resume-control.json");
+ const originalRenameSync = nodeFs.renameSync;
+ if (failureMode === "lock") expect(acquireLock(lockPath)).toBe(true);
+ else {
+ nodeFs.renameSync = ((oldPath: fs.PathLike, newPath: fs.PathLike) => {
+ if (String(newPath) === controlPath) throw new Error("injected control rename failure");
+ return originalRenameSync(oldPath, newPath);
+ }) as typeof nodeFs.renameSync;
+ syncBuiltinESMExports();
+ }
+
+ try {
+ const result = transition === "attempt"
+ ? rec.recordSessionStall("agent_1", "sess-poison")
+ : transition === "fence"
+ ? rec.forgetSession("agent_1", "stall_recovery", "sess-poison")
+ : transition === "clear"
+ ? rec.clearSessionStall("agent_1", "sess-poison")
+ : rec.forgetSession("agent_1", transition);
+ expect(result).toBe(false);
+ } finally {
+ if (failureMode === "lock") releaseLock(lockPath);
+ else {
+ nodeFs.renameSync = originalRenameSync;
+ syncBuiltinESMExports();
+ }
+ }
+
+ expect(fs.readFileSync(controlPath, "utf8")).toBe(beforeControl);
+ expect(readRecentEntries(dir, { now: NOW() })
+ .filter((row) => row.system)
+ .map((row) => row.system)).toEqual(beforeSystemRows);
+ const rebuilt = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ expect(rebuilt.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: transition === "fence" || transition === "clear" ? "sess-poison" : null,
+ fencedSessionId: null,
+ });
+ });
+ }
+
+ it(`keeps the prior in-memory session when setSession hits a control ${failureMode} failure`, () => {
+ const dir = mkDir();
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ expect(rec.setSession("agent_1", "sess-old")).toBe(true);
+ const controlPath = path.join(dir, ".resume-control.json");
+ const beforeControl = fs.readFileSync(controlPath, "utf8");
+ const lockPath = lockPathFor(dir, ".resume-control.json");
+ const originalRenameSync = nodeFs.renameSync;
+ if (failureMode === "lock") expect(acquireLock(lockPath)).toBe(true);
+ else {
+ nodeFs.renameSync = ((oldPath: fs.PathLike, newPath: fs.PathLike) => {
+ if (String(newPath) === controlPath) throw new Error("injected control rename failure");
+ return originalRenameSync(oldPath, newPath);
+ }) as typeof nodeFs.renameSync;
+ syncBuiltinESMExports();
+ }
+ try {
+ expect(rec.setSession("agent_1", "sess-new")).toBe(false);
+ } finally {
+ if (failureMode === "lock") releaseLock(lockPath);
+ else {
+ nodeFs.renameSync = originalRenameSync;
+ syncBuiltinESMExports();
+ }
+ }
+ expect(fs.readFileSync(controlPath, "utf8")).toBe(beforeControl);
+ rec.appendEntryForAgent("agent_1", [msg("#1", "after failed set")]);
+ expect(readRecentEntries(dir, { now: NOW() }).at(-1)?.session_id).toBe("sess-old");
+ });
+ }
+
+ it("resolves an attempt marker outside the seven-day turn-history window", () => {
+ const dir = mkDir();
+ const oldRecorder = createTimelineRecorder({
+ timelineDirFor: () => dir,
+ providerFor: () => "codex",
+ now: () => new Date("2026-06-01T12:00:00Z"),
+ });
+ oldRecorder.recordSessionStall("agent_1", "sess-poison");
+
+ const rebuiltRecorder = createTimelineRecorder({
+ timelineDirFor: () => dir,
+ providerFor: () => "codex",
+ now: () => new Date("2026-06-25T12:00:00Z"),
+ });
+ expect(rebuiltRecorder.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "none",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+ });
+
+ it("persists the exact repeatedly stalled session in a stall_recovery barrier", () => {
+ const dir = mkDir();
+ const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "codex", now: NOW });
+ rec.setSession("agent_1", "sess-poison");
+
+ rec.forgetSession("agent_1", "stall_recovery", "sess-poison");
+
+ const rows = readRecentEntries(dir, { now: NOW() });
+ expect(rows).toHaveLength(1);
+ expect(rows[0].system).toEqual({
+ type: "stall_recovery",
+ time: NOW().toISOString(),
+ backend_session_id: "sess-poison",
+ });
+ expect(rec.resolveResumeSession("agent_1", "codex")).toEqual({
+ kind: "barrier",
+ type: "stall_recovery",
+ forgottenSessionId: "sess-poison",
+ fencedSessionId: "sess-poison",
+ });
+ });
+
it("appends a bare reset_session system row (no forgot_session_id) and clears the map", () => {
const dir = mkDir();
const rec = createTimelineRecorder({ timelineDirFor: () => dir, providerFor: () => "claude", now: NOW });
diff --git a/src/daemon/src/timeline/recorder.ts b/src/daemon/src/timeline/recorder.ts
index fbefcb58f..464306cd0 100644
--- a/src/daemon/src/timeline/recorder.ts
+++ b/src/daemon/src/timeline/recorder.ts
@@ -24,14 +24,17 @@ import {
appendOrMergeEntry,
updateLatestEntryResult,
readRecentEntries,
+ readResumeControlState,
+ updateResumeControlState,
createTimelineEntry,
createSystemEntry,
- findResumableSession,
+ resolveResumableSession,
appendEntry,
prepareTimelineDirectory,
} from "./timeline.js";
import type { Message } from "../server/contract.js";
import type { SystemEntryType } from "./types.js";
+import type { ResumeSessionResolution } from "./timeline.js";
const MAX_AGENT_RESPONSES = 5;
@@ -44,24 +47,32 @@ function appendAgentResponse(entry: { agent_responses: string[] }, text: string)
/** Manager/daemon-facing recorder interface (structural, avoids a cyclic import). */
export interface TimelineRecorderLike {
- /** CONTROL plane: remember the runtime session id (baked into new entries). */
- setSession(agentId: string, sessionId: string): void;
+ /** CONTROL plane: remember the runtime session id; false means no state may advance. */
+ setSession(agentId: string, sessionId: string): boolean;
/** DATA plane: a successful inbox pull opens a new entry of what the agent saw. */
appendEntryForAgent(agentId: string, messages: Message[]): void;
/** CONTROL plane: append the agent's text output to its latest open entry. */
appendResponseToLatest(agentId: string, text: string): void;
/** Latest session id recorded for this agent (resume target), or null. */
resumeSessionId(agentId: string, provider: string | null): string | null;
+ /** Latest resume target or the durable barrier that supersedes older sources. */
+ resolveResumeSession(agentId: string, provider: string | null): ResumeSessionResolution;
+ /** Persist the consumed allowance; false means the caller must defer termination. */
+ recordSessionStall(agentId: string, sessionId: string): boolean;
+ /** Persist a clean replenishment; false leaves the allowance consumed. */
+ clearSessionStall(agentId: string, sessionId: string): boolean;
/**
* Reset: append a `system` barrier row to the timeline (`reset_session` for
* an owner reset, `nap` for an agent self-reset — default `reset_session`).
- * The resume walker treats either as a barrier — every turn at or before it
+ * The resume walker treats reset/nap as full barriers and stall recovery as
+ * an exact-session barrier — every superseded turn
* becomes invisible to `resumeSessionId` — and the agent's own history read
* (for recall) sees it in-line with turns. Also clears the in-memory session
* cache so a racing `appendEntryForAgent` can't bake the stale id into a
- * fresh row.
+ * fresh row. Returns false unless the authoritative control transition was
+ * committed; the forensic row is appended only after that precondition.
*/
- forgetSession(agentId: string, barrierType?: SystemEntryType): void;
+ forgetSession(agentId: string, barrierType?: SystemEntryType, forgottenSessionId?: string): boolean;
}
export interface TimelineRecorderOptions {
@@ -79,10 +90,59 @@ export function createTimelineRecorder(opts: TimelineRecorderOptions): TimelineR
// session_init (control plane) lands before the agent's first pull (data plane)
// opens an entry, so hold the latest session id here and bake it into new rows.
const sessionByAgent = new Map<string, string>();
+ const resolveForAgent = (agentId: string, provider: string | null): ResumeSessionResolution => {
+ const dir = dirFor(agentId);
+ const recent = resolveResumableSession(
+ readRecentEntries(dir, { now: now() }),
+ provider ?? undefined,
+ );
+ const control = readResumeControlState(dir);
+ if (control.kind === "missing") return recent;
+ if (control.kind === "invalid") {
+ return {
+ kind: "barrier",
+ type: "reset_session",
+ forgottenSessionId: null,
+ fencedSessionId: null,
+ };
+ }
+ const { attemptedSessionId, fencedSessionId, fullBarrier } = control.state;
+ if (fullBarrier !== null) {
+ return {
+ kind: "barrier",
+ type: fullBarrier,
+ forgottenSessionId: null,
+ fencedSessionId,
+ };
+ }
+ if (recent.kind === "session") {
+ return {
+ kind: "session",
+ sessionId: recent.sessionId,
+ stalledSessionId: attemptedSessionId === recent.sessionId ? attemptedSessionId : null,
+ fencedSessionId,
+ };
+ }
+ if (recent.kind === "barrier") {
+ if (recent.type !== "stall_recovery" || recent.forgottenSessionId === fencedSessionId) {
+ return { ...recent, fencedSessionId };
+ }
+ }
+ return { kind: "none", stalledSessionId: attemptedSessionId, fencedSessionId };
+ };
return {
setSession(agentId, sessionId) {
+ const dir = dirFor(agentId);
+ if (!prepareTimelineDirectory(dir)) return false;
+ const persisted = updateResumeControlState(dir, (state) => ({
+ ...state,
+ fullBarrier: null,
+ attemptedSessionId: state.attemptedSessionId === sessionId ? state.attemptedSessionId : null,
+ }));
+ if (!persisted) return false;
sessionByAgent.set(agentId, sessionId);
+ return true;
},
appendEntryForAgent(agentId, messages) {
if (messages.length === 0) return;
@@ -120,20 +180,62 @@ export function createTimelineRecorder(opts: TimelineRecorderOptions): TimelineR
appendEntry(dir, entry, now());
},
resumeSessionId(agentId, provider) {
- const rows = readRecentEntries(dirFor(agentId), { now: now() });
- return findResumableSession(rows, provider ?? undefined);
+ const resolution = resolveForAgent(agentId, provider);
+ return resolution.kind === "session" ? resolution.sessionId : null;
+ },
+ resolveResumeSession(agentId, provider) {
+ return resolveForAgent(agentId, provider);
+ },
+ recordSessionStall(agentId, sessionId) {
+ return appendStallMarker(agentId, "stall_recovery_attempt", sessionId);
+ },
+ clearSessionStall(agentId, sessionId) {
+ return appendStallMarker(agentId, "stall_recovery_clear", sessionId);
},
- forgetSession(agentId, barrierType = "reset_session") {
+ forgetSession(agentId, barrierType = "reset_session", forgottenSessionId) {
const dir = dirFor(agentId);
- // Kill happens BEFORE this call (see `AgentProcessManager.resetSession`),
- // so no race — clear the in-memory session map and append the barrier.
+ // Persist the authoritative transition before clearing the in-memory
+ // session map or appending its best-effort forensic row.
// One `now()` sample so the system-row `time` and the day-file the row
// is written into can't disagree on the boundary between two consecutive
// clock reads.
- sessionByAgent.delete(agentId);
- if (!prepareTimelineDirectory(dir)) return;
+ if (!prepareTimelineDirectory(dir)) return false;
const stamp = now();
- appendEntry(dir, createSystemEntry(barrierType, stamp.toISOString()), stamp);
+ let persisted = false;
+ if (barrierType === "reset_session" || barrierType === "nap") {
+ persisted = updateResumeControlState(dir, (state) => ({
+ ...state,
+ attemptedSessionId: null,
+ fencedSessionId: null,
+ fullBarrier: barrierType,
+ }));
+ } else if (barrierType === "stall_recovery") {
+ persisted = updateResumeControlState(dir, (state) => ({
+ ...state,
+ attemptedSessionId:
+ state.attemptedSessionId === forgottenSessionId ? null : state.attemptedSessionId,
+ fencedSessionId: forgottenSessionId ?? null,
+ fullBarrier: null,
+ }));
+ }
+ if (!persisted) return false;
+ sessionByAgent.delete(agentId);
+ appendEntry(dir, createSystemEntry(barrierType, stamp.toISOString(), forgottenSessionId), stamp);
+ return true;
},
};
+
+ function appendStallMarker(agentId: string, type: SystemEntryType, sessionId: string): boolean {
+ const dir = dirFor(agentId);
+ if (!prepareTimelineDirectory(dir)) return false;
+ const stamp = now();
+ const persisted = updateResumeControlState(dir, (state) => ({
+ ...state,
+ attemptedSessionId: type === "stall_recovery_attempt" ? sessionId : null,
+ fullBarrier: null,
+ }));
+ if (!persisted) return false;
+ appendEntry(dir, createSystemEntry(type, stamp.toISOString(), sessionId), stamp);
+ return true;
+ }
}
diff --git a/src/daemon/src/timeline/timeline.test.ts b/src/daemon/src/timeline/timeline.test.ts
index aa2dcfe1a..afe717c56 100644
--- a/src/daemon/src/timeline/timeline.test.ts
+++ b/src/daemon/src/timeline/timeline.test.ts
@@ -11,6 +11,7 @@ import {
createTimelineEntry,
createSystemEntry,
findResumableSession,
+ resolveResumableSession,
filenameForDate,
localISOString,
sweepTimelineHistory,
@@ -584,6 +585,74 @@ describe("findResumableSession", () => {
expect(findResumableSession(rows)).toBeNull();
});
+ it("returns the exact backend session fenced by a stall_recovery barrier", () => {
+ const rows = [
+ { ...createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }) },
+ createSystemEntry("stall_recovery", "2026-06-25T12:00:00Z", "sess-poison"),
+ ];
+ expect(resolveResumableSession(rows, "codex")).toEqual({
+ kind: "barrier",
+ type: "stall_recovery",
+ forgottenSessionId: "sess-poison",
+ fencedSessionId: "sess-poison",
+ });
+ expect(findResumableSession(rows, "codex")).toBeNull();
+ });
+
+ it("carries an un-cleared first-stall attempt across later rows for the same session", () => {
+ const rows = [
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),
+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),
+ ];
+ expect(resolveResumableSession(rows, "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+ });
+
+ it("retains the consumed attempt even when no timeline session row remains", () => {
+ const rows = [
+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),
+ ];
+ expect(resolveResumableSession(rows, "codex")).toEqual({
+ kind: "none",
+ stalledSessionId: "sess-poison",
+ fencedSessionId: null,
+ });
+ });
+
+ it("a clean-completion marker clears the durable first-stall attempt", () => {
+ const rows = [
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),
+ createSystemEntry("stall_recovery_attempt", "2026-06-25T12:00:00Z", "sess-poison"),
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),
+ createSystemEntry("stall_recovery_clear", "2026-06-25T12:01:00Z", "sess-poison"),
+ ];
+ expect(resolveResumableSession(rows, "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-poison",
+ stalledSessionId: null,
+ fencedSessionId: null,
+ });
+ });
+
+ it("carries an exact fence alongside a newer healthy session", () => {
+ const rows = [
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-poison" }),
+ createSystemEntry("stall_recovery", "2026-06-25T12:00:00Z", "sess-poison"),
+ createTimelineEntry({ messages: [], provider: "codex", sessionId: "sess-healthy" }),
+ ];
+ expect(resolveResumableSession(rows, "codex")).toEqual({
+ kind: "session",
+ sessionId: "sess-healthy",
+ stalledSessionId: null,
+ fencedSessionId: "sess-poison",
+ });
+ });
+
it("returns a session id from a row appended AFTER the barrier", () => {
const rows = [
{ ...createTimelineEntry({ messages: [], provider: "claude", sessionId: "s-old" }) },
diff --git a/src/daemon/src/timeline/timeline.ts b/src/daemon/src/timeline/timeline.ts
index ad42dfbd3..9dc00f763 100644
--- a/src/daemon/src/timeline/timeline.ts
+++ b/src/daemon/src/timeline/timeline.ts
@@ -24,6 +24,27 @@ export { localISOString };
export const TIMELINE_MAX_BYTES = 1_048_576;
export const TIMELINE_READ_CHUNK_BYTES = 65_536;
const DATE_FILENAME_PATTERN = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
+const RESUME_CONTROL_FILENAME = ".resume-control.json";
+const RESUME_CONTROL_MAX_BYTES = 4_096;
+
+export interface ResumeControlState {
+ version: 1;
+ attemptedSessionId: string | null;
+ fencedSessionId: string | null;
+ fullBarrier: "reset_session" | "nap" | null;
+}
+
+export type ResumeControlReadResult =
+ | { kind: "missing" }
+ | { kind: "invalid" }
+ | { kind: "state"; state: ResumeControlState };
+
+const EMPTY_RESUME_CONTROL: ResumeControlState = {
+ version: 1,
+ attemptedSessionId: null,
+ fencedSessionId: null,
+ fullBarrier: null,
+};
interface TimelineLine {
text: string;
@@ -33,7 +54,7 @@ interface TimelineLine {
}
function isBarrier(entry: ContextTimelineEntry): boolean {
- return entry.system?.type === "reset_session" || entry.system?.type === "nap";
+ return entry.system !== undefined;
}
function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {
@@ -41,10 +62,24 @@ function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {
const entry = value as Partial<ContextTimelineEntry>;
if (entry.system) {
if (
- (entry.system.type !== "reset_session" && entry.system.type !== "nap") ||
- typeof entry.system.time !== "string"
+ (
+ entry.system.type !== "reset_session"
+ && entry.system.type !== "nap"
+ && entry.system.type !== "stall_recovery_attempt"
+ && entry.system.type !== "stall_recovery_clear"
+ && entry.system.type !== "stall_recovery"
+ )
+ || typeof entry.system.time !== "string"
+ || (
+ entry.system.backend_session_id !== undefined
+ && typeof entry.system.backend_session_id !== "string"
+ )
) return null;
- return createSystemEntry(entry.system.type, entry.system.time);
+ return createSystemEntry(
+ entry.system.type,
+ entry.system.time,
+ entry.system.backend_session_id,
+ );
}
if (entry.session_id !== null && typeof entry.session_id !== "string") return null;
if (entry.provider !== null && typeof entry.provider !== "string") return null;
@@ -60,7 +95,7 @@ function canonicalTimelineEntry(value: unknown): ContextTimelineEntry | null {
function timelineLine(entry: ContextTimelineEntry): TimelineLine | null {
const boundedEntry = entry.system
- ? createSystemEntry(entry.system.type, entry.system.time)
+ ? createSystemEntry(entry.system.type, entry.system.time, entry.system.backend_session_id)
: { ...entry, agent_responses: entry.agent_responses.slice(-5) };
const text = JSON.stringify(boundedEntry);
const bytes = Buffer.byteLength(text, "utf8") + 1;
@@ -341,6 +376,114 @@ export function readRecentEntries(timelineDir: string, opts: ReadRecentOptions =
return entries;
}
+/** Latest durable resume-control row across the complete retained timeline. */
+export function readResumeControlState(timelineDir: string): ResumeControlReadResult {
+ if (timelineDirectoryState(timelineDir) !== "safe") return { kind: "missing" };
+ const filePath = join(timelineDir, RESUME_CONTROL_FILENAME);
+ let source: fs.Stats;
+ try {
+ source = fs.lstatSync(filePath);
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code === "ENOENT"
+ ? { kind: "missing" }
+ : { kind: "invalid" };
+ }
+ if (!source.isFile() || source.size <= 0 || source.size > RESUME_CONTROL_MAX_BYTES) {
+ return { kind: "invalid" };
+ }
+ let fd: number | null = null;
+ try {
+ fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+ const stat = fs.fstatSync(fd);
+ if (!stat.isFile() || stat.size <= 0 || stat.size > RESUME_CONTROL_MAX_BYTES) {
+ return { kind: "invalid" };
+ }
+ const bounded = Buffer.allocUnsafe(stat.size + 1);
+ let bytesRead = 0;
+ while (bytesRead < bounded.length) {
+ const count = fs.readSync(fd, bounded, bytesRead, bounded.length - bytesRead, bytesRead);
+ if (count <= 0) break;
+ bytesRead += count;
+ }
+ if (bytesRead !== stat.size) return { kind: "invalid" };
+ const raw = bounded.subarray(0, bytesRead).toString("utf8");
+ const value = JSON.parse(raw) as Partial<ResumeControlState>;
+ const validSessionId = (candidate: unknown): candidate is string | null =>
+ candidate === null || (typeof candidate === "string" && candidate.length > 0 && candidate.length <= 512);
+ if (
+ value.version !== 1
+ || !validSessionId(value.attemptedSessionId)
+ || !validSessionId(value.fencedSessionId)
+ || (
+ value.fullBarrier !== null
+ && value.fullBarrier !== "reset_session"
+ && value.fullBarrier !== "nap"
+ )
+ ) return { kind: "invalid" };
+ return {
+ kind: "state",
+ state: {
+ version: 1,
+ attemptedSessionId: value.attemptedSessionId,
+ fencedSessionId: value.fencedSessionId,
+ fullBarrier: value.fullBarrier,
+ },
+ };
+ } catch {
+ return { kind: "invalid" };
+ } finally {
+ if (fd !== null) {
+ try { fs.closeSync(fd); } catch { /* best effort */ }
+ }
+ }
+}
+
+export function updateResumeControlState(
+ timelineDir: string,
+ update: (state: ResumeControlState) => ResumeControlState,
+): boolean {
+ if (timelineDirectoryState(timelineDir) !== "safe") return false;
+ const lockPath = lockPathFor(timelineDir, RESUME_CONTROL_FILENAME);
+ if (!acquireLock(lockPath)) return false;
+ try {
+ const current = readResumeControlState(timelineDir);
+ const base = current.kind === "state" ? current.state : EMPTY_RESUME_CONTROL;
+ const next = update({ ...base });
+ const canonical: ResumeControlState = {
+ version: 1,
+ attemptedSessionId: next.attemptedSessionId,
+ fencedSessionId: next.fencedSessionId,
+ fullBarrier: next.fullBarrier,
+ };
+ const body = JSON.stringify(canonical) + "\n";
+ if (Buffer.byteLength(body, "utf8") > RESUME_CONTROL_MAX_BYTES) return false;
+ const filePath = join(timelineDir, RESUME_CONTROL_FILENAME);
+ const tempPath = join(
+ timelineDir,
+ `.${RESUME_CONTROL_FILENAME}.${process.pid}.${randomBytes(12).toString("hex")}.tmp`,
+ );
+ let fd: number | null = null;
+ try {
+ fd = fs.openSync(tempPath, "wx", 0o600);
+ fs.writeFileSync(fd, body, "utf8");
+ fs.fsyncSync(fd);
+ fs.closeSync(fd);
+ fd = null;
+ fs.renameSync(tempPath, filePath);
+ return true;
+ } finally {
+ if (fd !== null) {
+ try { fs.closeSync(fd); } catch { /* best effort */ }
+ }
+ try { fs.unlinkSync(tempPath); } catch { /* best effort */ }
+ }
+ } catch {
+ return false;
+ } finally {
+ releaseLock(lockPath);
+ }
+}
+
/* ------------------------------------------------------------------ */
/* Write (lock-guarded) */
/* ------------------------------------------------------------------ */
@@ -588,19 +731,23 @@ export function createTimelineEntry(fields: NewEntryFields): ContextTimelineEntr
/**
* Build a system entry. System rows are inlined in the JSONL alongside turns
- * so the resume walker and any agent-facing reader see them in place. The
- * first (and only) type today is `reset_session`.
+ * so the resume walker and any agent-facing reader see them in place.
*/
export function createSystemEntry(
type: SystemEntryType,
time: string,
+ backendSessionId?: string,
): ContextTimelineEntry {
return {
session_id: null,
messages: [],
agent_responses: [],
provider: null,
- system: { type, time },
+ system: {
+ type,
+ time,
+ ...(backendSessionId ? { backend_session_id: backendSessionId } : {}),
+ },
};
}
@@ -617,22 +764,85 @@ export function createSystemEntry(
* a codex launch). One session per agent and each timeline lives in that
* agent's own workdir, so there's no thread keying.
*
- * A `system: { type: "reset_session" }` row is a barrier: the walker returns
- * null the moment it hits one going newest→oldest. Every turn at or before
- * the reset becomes invisible to resume without editing those rows. Since
- * kill happens BEFORE the barrier is written (see
- * `AgentProcessManager.resetSession`), no `forgot_session_id` fallback is
- * needed — no turn row can land after the barrier from the old session.
+ * Reset/nap rows are full barriers. A `stall_recovery` row fences only its
+ * exact backend session, while attempt/clear rows carry the bounded retry
+ * state across daemon restarts without disabling healthy persistence.
*/
-export function findResumableSession(rows: ContextTimelineEntry[], provider?: string): string | null {
+export type ResumeSessionResolution =
+ | {
+ kind: "session";
+ sessionId: string;
+ stalledSessionId: string | null;
+ fencedSessionId: string | null;
+ }
+ | {
+ kind: "barrier";
+ type: SystemEntryType;
+ forgottenSessionId: string | null;
+ fencedSessionId: string | null;
+ }
+ | { kind: "none"; stalledSessionId: string | null; fencedSessionId: string | null };
+
+export function resolveResumableSession(
+ rows: ContextTimelineEntry[],
+ provider?: string,
+): ResumeSessionResolution {
+ let candidateSessionId: string | null = null;
+ let recoveryMarkerSeen = false;
+ let stalledSessionId: string | null = null;
+ let fencedSessionId: string | null = null;
for (let i = rows.length - 1; i >= 0; i--) {
const e = rows[i];
- // Both reset_session (owner) and nap (agent self-reset) are resume
- // barriers — a fresh session was deliberately started at this point.
- if (e.system?.type === "reset_session" || e.system?.type === "nap") return null;
+ if (e.system) {
+ if (e.system.type === "stall_recovery_attempt") {
+ if (!recoveryMarkerSeen) {
+ recoveryMarkerSeen = true;
+ stalledSessionId = e.system.backend_session_id ?? null;
+ }
+ continue;
+ }
+ if (e.system.type === "stall_recovery_clear") {
+ if (!recoveryMarkerSeen) recoveryMarkerSeen = true;
+ continue;
+ }
+ if (e.system.type === "stall_recovery" && fencedSessionId === null) {
+ fencedSessionId = e.system.backend_session_id ?? null;
+ }
+ if (candidateSessionId !== null) {
+ return {
+ kind: "session",
+ sessionId: candidateSessionId,
+ stalledSessionId: stalledSessionId === candidateSessionId ? stalledSessionId : null,
+ fencedSessionId,
+ };
+ }
+ return {
+ kind: "barrier",
+ type: e.system.type,
+ forgottenSessionId: e.system.backend_session_id ?? null,
+ fencedSessionId,
+ };
+ }
if (!e.session_id) continue;
if (provider && e.provider !== provider) continue;
- return e.session_id;
+ if (candidateSessionId === null) {
+ candidateSessionId = e.session_id;
+ continue;
+ }
+ if (candidateSessionId !== e.session_id) break;
+ }
+ if (candidateSessionId !== null) {
+ return {
+ kind: "session",
+ sessionId: candidateSessionId,
+ stalledSessionId: stalledSessionId === candidateSessionId ? stalledSessionId : null,
+ fencedSessionId,
+ };
}
- return null;
+ return { kind: "none", stalledSessionId, fencedSessionId };
+}
+
+export function findResumableSession(rows: ContextTimelineEntry[], provider?: string): string | null {
+ const resolution = resolveResumableSession(rows, provider);
+ return resolution.kind === "session" ? resolution.sessionId : null;
}
diff --git a/src/daemon/src/timeline/types.ts b/src/daemon/src/timeline/types.ts
index 2800e5c1d..e8c0d03c7 100644
--- a/src/daemon/src/timeline/types.ts
+++ b/src/daemon/src/timeline/types.ts
@@ -25,27 +25,33 @@ import type { Message } from "../server/contract.js";
* message carries its own `time`.
* - **System** (`system` set): an out-of-band event the daemon needs to
* record in-line with turns so both the resume walker AND the agent's
- * own history read see it. First (and only) type today is
- * `reset_session`: the owner asked to forget prior conversation. On
- * resume, `findResumableSession` walks newest→oldest and STOPS if it
- * hits a `reset_session` row before finding a session id — so every
- * row at or before the reset is invisible for resume without touching
- * the rows themselves.
+ * own history read see it. `reset_session` and `nap` deliberately forget
+ * all prior conversation; stall-recovery rows persist the one allowed
+ * recovery attempt, its clean reset, and the final exact-session fence. On
+ * resume, the resolver walks newest→oldest: reset/nap rows hide older
+ * candidates, while an exact stall fence remains attached to any newer
+ * healthy candidate so stale external resume sources cannot resurrect it.
*
* On a system row, `session_id`/`provider` are null and `messages`/
* `agent_responses` are empty. The system row is not mergeable with a
* following turn (see `appendOrMergeEntry`).
*/
-// `reset_session` — owner-triggered reset. `nap` — the agent reset its OWN
-// session (see agent:nap). Both are resume barriers with identical walker
-// semantics; the distinct type just records who initiated it.
-export type SystemEntryType = "reset_session" | "nap";
+// Stall recovery uses durable attempt/clear markers plus a final barrier so a
+// daemon restart cannot replenish the same poisoned session's retry allowance.
+export type SystemEntryType =
+ | "reset_session"
+ | "nap"
+ | "stall_recovery_attempt"
+ | "stall_recovery_clear"
+ | "stall_recovery";
export interface SystemEntry {
/** Event kind. Extend by adding to `SystemEntryType`. */
type: SystemEntryType;
/** ISO timestamp when the event landed — the row is otherwise untimed. */
time: string;
+ /** Exact backend session associated with a stall-recovery row. */
+ backend_session_id?: string;
}
export interface ContextTimelineEntry {