Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

feat(daemon): reset sessions after extended idle (#548)

Gustavo Yebfff12d2f7b1
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.ts
index 2842809b5..09c7a66ea 100644
--- a/src/daemon/src/manager/managerPolicy.test.ts
+++ b/src/daemon/src/manager/managerPolicy.test.ts
@@ -2,6 +2,8 @@ import { describe, it, expect } from "vitest";
import {
reduceManager,
createInitialManagerState,
+ DEFAULT_IDLE_RESET_TIMEOUT_MS,
+ DEFAULT_IDLE_TIMEOUT_MS,
isActivelyWorking,
type ManagerState,
type AgentState,
@@ -118,6 +120,16 @@ function startAdmission(
}
describe("reduceManager — single-flight spawn", () => {
+ it("defaults to 30-minute process hibernation and 6-hour session reset", () => {
+ const state = createInitialManagerState();
+ expect(DEFAULT_IDLE_TIMEOUT_MS).toBe(30 * 60 * 1_000);
+ expect(DEFAULT_IDLE_RESET_TIMEOUT_MS).toBe(6 * 60 * 60 * 1_000);
+ expect(state).toMatchObject({
+ idleTimeoutMs: DEFAULT_IDLE_TIMEOUT_MS,
+ idleResetTimeoutMs: DEFAULT_IDLE_RESET_TIMEOUT_MS,
+ });
+ });
+
it("ignores a stale session close", () => {
let state = register(createInitialManagerState(), "a", PERSISTENT_GATED);
state = spawnRoot(state, 1);
@@ -620,6 +632,104 @@ describe("reduceManager — tick: stall + idle hibernation", () => {
expect(r.state.agents.a.sessionId).toBe("sess-1"); // preserved for resume
});
+ it("keeps the per-agent idle clock across hibernation and resets the resumable session at its deadline", () => {
+ let s = createInitialManagerState(100_000, 100, 100_000, 100_000, 1_000);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0);
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-1" }).state;
+ s = completeRoot(s, "turn-a", 0).state;
+
+ const hibernating = reduceManager(s, { type: "tick", nowMs: 100 });
+ expect(hibernating.effects).toEqual([{ type: "stop", agentId: "a", reason: "idle_timeout" }]);
+ expect(hibernating.state.agents.a).toMatchObject({ status: "stopping", idleSince: 0 });
+
+ s = reduceManager(hibernating.state, {
+ type: "session_closed",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ }).state;
+ expect(s.agents.a.idleSince).toBe(0);
+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;
+ expect(s.agents.a).toMatchObject({ status: "idle", idleSince: 0, sessionId: "sess-1" });
+ expect(reduceManager(s, { type: "tick", nowMs: 999 }).effects).toEqual([]);
+
+ const due = reduceManager(s, { type: "tick", nowMs: 1_000 });
+ expect(due.effects).toEqual([
+ { type: "reset_idle_session", agentId: "a", sessionId: "sess-1" },
+ ]);
+ const committed = reduceManager(due.state, {
+ type: "idle_reset_committed",
+ agentId: "a",
+ nowMs: 1_000,
+ });
+ expect(committed.effects).toEqual([]);
+ expect(committed.state.agents.a).toMatchObject({
+ status: "idle",
+ sessionId: null,
+ idleSince: null,
+ });
+ expect(reduceManager(committed.state, { type: "tick", nowMs: 12_000 }).effects).toEqual([]);
+ });
+
+ it("cancels the idle reset clock when a new message wakes a hibernated agent", () => {
+ let s = createInitialManagerState(100_000, 100, 100_000, 100_000, 1_000);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0);
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-1" }).state;
+ s = completeRoot(s, "turn-a", 0).state;
+ s = reduceManager(s, { type: "tick", nowMs: 100 }).state;
+ s = reduceManager(s, {
+ type: "session_closed",
+ agentId: "a",
+ sessionInstanceId: SESSION_INSTANCE,
+ }).state;
+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;
+
+ const wake = reduceManager(s, {
+ type: "wake",
+ agentId: "a",
+ message: { text: "new work" },
+ nowMs: 500,
+ });
+ expect(wake.effects).toEqual([{
+ type: "spawn",
+ agentId: "a",
+ messages: [{ text: "new work" }],
+ resumeSessionId: "sess-1",
+ }]);
+ expect(wake.state.agents.a.idleSince).toBeNull();
+ expect(reduceManager(wake.state, { type: "tick", nowMs: 2_000 }).effects).toEqual([]);
+ });
+
+ it("commits a due idle reset for a still-running quiescent agent before stopping it", () => {
+ let s = createInitialManagerState(100_000, 0, 100_000, 100_000, 100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = spawnRoot(s, 0);
+ s = reduceManager(s, { type: "backend_session", agentId: "a", sessionId: "sess-1" }).state;
+ s = completeRoot(s, "turn-a", 0).state;
+
+ expect(reduceManager(s, { type: "tick", nowMs: 100 }).effects).toEqual([
+ { type: "reset_idle_session", agentId: "a", sessionId: "sess-1" },
+ ]);
+ const committed = reduceManager(s, {
+ type: "idle_reset_committed",
+ agentId: "a",
+ nowMs: 100,
+ });
+ expect(committed.effects).toEqual([
+ { type: "stop", agentId: "a", reason: "idle_session_reset" },
+ ]);
+ expect(committed.state.agents.a).toMatchObject({
+ status: "stopping",
+ sessionId: null,
+ idleSince: null,
+ stoppingSince: 100,
+ });
+ });
+
it("root progress after a stale turn_end restores active work and cancels idle hibernation", () => {
let s = createInitialManagerState(10_000, 100);
s = register(s, "a", PERSISTENT_GATED);
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index c4f69cfe0..5a04f915c 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -106,6 +106,7 @@ export interface ManagerState {
agents: Record<string, AgentState>;
staleThresholdMs: number;
idleTimeoutMs: number;
+ idleResetTimeoutMs: number;
resetStuckThresholdMs: number;
stoppingStuckThresholdMs: number;
}
@@ -178,6 +179,7 @@ export type ManagerEvent =
}
| { type: "tick"; nowMs: number }
| { type: "reset_session"; agentId: string }
+ | { type: "idle_reset_committed"; agentId: string; nowMs: number }
| { type: "begin_reset"; agentId: string; nowMs: number }
| { type: "rewake_after_reset"; agentId: string; message: AgentMsg }
| {
@@ -212,6 +214,7 @@ export type ManagerEffect =
| { type: "clear_stall_recovery"; agentId: string; sessionId: string }
| { type: "expire_admission"; agentId: string; sessionInstanceId: string; commandIds: string[] }
| { type: "requeue_delivery"; agentId: string; message: AgentMsg; mode: "busy" | "idle" }
+ | { type: "reset_idle_session"; agentId: string; sessionId: string }
| { type: "force_exit"; agentId: string; reason: string };
export const DEFAULT_STALE_THRESHOLD_MS = 120_000;
@@ -222,7 +225,8 @@ export const DEFAULT_TURN_SILENCE_POLICY: TurnSilencePolicy = {
maxRecoveryExtensions: 1,
normalBudgetMs: 360_000,
};
-export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;
+export const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1_000;
+export const DEFAULT_IDLE_RESET_TIMEOUT_MS = 6 * 60 * 60 * 1_000;
export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;
export const DEFAULT_STOPPING_STUCK_THRESHOLD_MS = 30_000;
@@ -236,8 +240,16 @@ export function createInitialManagerState(
idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,
resetStuckThresholdMs = DEFAULT_RESET_STUCK_THRESHOLD_MS,
stoppingStuckThresholdMs = DEFAULT_STOPPING_STUCK_THRESHOLD_MS,
+ idleResetTimeoutMs = DEFAULT_IDLE_RESET_TIMEOUT_MS,
): ManagerState {
- return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs, stoppingStuckThresholdMs };
+ return {
+ agents: {},
+ staleThresholdMs,
+ idleTimeoutMs,
+ idleResetTimeoutMs,
+ resetStuckThresholdMs,
+ stoppingStuckThresholdMs,
+ };
}
export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceResult {
@@ -347,8 +359,24 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
return mutate(state, event.agentId, (a) => {
a.sessionId = null;
a.stalledSessionId = null;
+ a.idleSince = null;
});
+ case "idle_reset_committed": {
+ const existing = state.agents[event.agentId];
+ if (!existing) return { state, effects: [] };
+ const agent = clone(existing);
+ agent.sessionId = null;
+ agent.stalledSessionId = null;
+ agent.idleSince = null;
+ if (agent.status !== "running") return commit(state, agent, []);
+ agent.status = "stopping";
+ agent.stoppingSince = event.nowMs;
+ return commit(state, agent, [
+ { type: "stop", agentId: event.agentId, reason: "idle_session_reset" },
+ ]);
+ }
+
case "begin_reset":
if (!state.agents[event.agentId]) return { state, effects: [] };
return mutate(state, event.agentId, (a) => {
@@ -467,7 +495,6 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
const closing = agent.pendingAdmissions.filter((entry) => entry.sessionInstanceId === event.sessionInstanceId);
agent.pendingAdmissions = agent.pendingAdmissions.filter((entry) => entry.sessionInstanceId !== event.sessionInstanceId);
agent.execution = { sessionInstanceId: null, lease: { state: "detached" } };
- agent.idleSince = null;
syncExecutionProjection(agent);
return commit(state, agent, recoveryEffects(agent, closing));
}
@@ -776,6 +803,23 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
continue;
}
+ const idleResetEligible =
+ a.idleSince !== null &&
+ a.sessionId !== null &&
+ a.inbox.length === 0 &&
+ a.pendingAdmissions.length === 0 &&
+ !a.resetting &&
+ state.idleResetTimeoutMs > 0 &&
+ Number.isFinite(state.idleResetTimeoutMs) &&
+ (
+ (a.status === "idle" && lease.state === "detached") ||
+ (a.status === "running" && lease.state === "none" && lease.lastTerminal !== null)
+ );
+ if (idleResetEligible && nowMs - a.idleSince! >= state.idleResetTimeoutMs) {
+ effects.push({ type: "reset_idle_session", agentId: id, sessionId: a.sessionId! });
+ continue;
+ }
+
const idleEligible =
a.status === "running" &&
lease.state === "none" &&
@@ -785,7 +829,9 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
state.idleTimeoutMs > 0 &&
Number.isFinite(state.idleTimeoutMs);
if (idleEligible && a.idleSince !== null && nowMs - a.idleSince >= state.idleTimeoutMs) {
- agents[id] = { ...a, status: "stopping", idleSince: null, stoppingSince: nowMs };
+ // Preserve idleSince across process hibernation: the per-agent inactivity
+ // clock continues until work arrives or its resumable session is reset.
+ agents[id] = { ...a, status: "stopping", stoppingSince: nowMs };
effects.push({ type: "stop", agentId: id, reason: "idle_timeout" });
}
}
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index cae9ae9e4..739bff8cf 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -279,7 +279,7 @@ function bindFactorySession(
return session;
}
-function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; staleThresholdMs?: number; resetStuckThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {
+function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; idleResetTimeoutMs?: number; staleThresholdMs?: number; resetStuckThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {
const session = fakeSession();
const factory = sessionFactoryFor(session);
const onRuntimeSpawnFailed = vi.fn();
@@ -313,6 +313,113 @@ function exactTimelineLifecycleStub() {
};
}
+describe("AgentProcessManager — idle session reset", () => {
+ it("persists an exact reset barrier and clears the local session without waking an idle agent", () => {
+ const forgetSession = vi.fn(() => true);
+ const sessionFactory = vi.fn();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory,
+ timeline: { ...exactTimelineLifecycleStub(), forgetSession } as never,
+ now: () => 123,
+ });
+ mgr.register("a1");
+ const internal = mgr as unknown as { applyEffect(effect: object): void };
+
+ internal.applyEffect({ type: "reset_idle_session", agentId: "a1", sessionId: "sess-old" });
+
+ expect(forgetSession).toHaveBeenCalledWith("a1", "reset_session", "sess-old");
+ expect(sessionFactory).not.toHaveBeenCalled();
+ expect(mgr.snapshot().agents.a1).toMatchObject({ status: "idle", sessionId: null, idleSince: null });
+ });
+
+ it("defers the reset and leaves state retryable when the barrier cannot be persisted", () => {
+ const logger = stubLogger();
+ const forgetSession = vi.fn(() => false);
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ timeline: { ...exactTimelineLifecycleStub(), forgetSession } as never,
+ logger,
+ });
+ mgr.register("a1");
+ const before = mgr.snapshot();
+ const internal = mgr as unknown as { applyEffect(effect: object): void };
+
+ internal.applyEffect({ type: "reset_idle_session", agentId: "a1", sessionId: "sess-old" });
+ internal.applyEffect({ type: "reset_idle_session", agentId: "a1", sessionId: "sess-old" });
+
+ expect(forgetSession).toHaveBeenCalledTimes(2);
+ expect(mgr.snapshot()).toBe(before);
+ expect(logger.calls.error.map(([message]) => message)).toEqual([
+ "idle session reset barrier was not persisted; reset deferred",
+ "idle session reset barrier was not persisted; reset deferred",
+ ]);
+ });
+
+ it("fences the old spawn owner before stop so a late session_started cannot restore the reset session", async () => {
+ const session = fakeSession("idle-reset-instance");
+ session.stop = vi.fn(session.stop.bind(session));
+ const setSession = vi.fn(() => true);
+ const forgetSession = vi.fn(() => true);
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory: sessionFactoryFor(session),
+ timeline: {
+ ...exactTimelineLifecycleStub(),
+ setSession,
+ forgetSession,
+ resumeSessionId: () => null,
+ } as never,
+ now: () => 123,
+ });
+ mgr.register("a1");
+ mgr.deliver("a1", { id: "first", text: "hello" });
+ session.startResolver?.();
+ await Promise.resolve();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-old" });
+ await session.fire("runtime_event", { kind: "turn_end", sessionId: "sess-old" });
+ expect(mgr.snapshot().agents.a1).toMatchObject({ sessionId: "sess-old", idleSince: 123 });
+
+ const internal = mgr as unknown as {
+ applyEffect(effect: object): void;
+ activeSpawnState: Map<string, { discardEvents: boolean }>;
+ };
+ internal.applyEffect({ type: "reset_idle_session", agentId: "a1", sessionId: "sess-old" });
+
+ expect(forgetSession).toHaveBeenCalledWith("a1", "reset_session", "sess-old");
+ expect(internal.activeSpawnState.get("a1")?.discardEvents).toBe(true);
+ expect(session.stop).toHaveBeenCalledWith({ reason: "idle_timeout", forceAfterMs: 2_000 });
+ expect(mgr.snapshot().agents.a1.sessionId).toBeNull();
+ expect(setSession).toHaveBeenCalledTimes(1);
+
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "sess-old" });
+
+ expect(setSession).toHaveBeenCalledTimes(1);
+ expect(mgr.snapshot().agents.a1.sessionId).toBeNull();
+ });
+});
+
function recordObservedTurn(
recorder: ReturnType<typeof createTimelineRecorder>,
agentId: string,
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index 325bb490e..07b4df220 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -1,6 +1,8 @@
import {
reduceManager,
createInitialManagerState,
+ DEFAULT_IDLE_RESET_TIMEOUT_MS,
+ DEFAULT_IDLE_TIMEOUT_MS,
DEFAULT_STOPPING_STUCK_THRESHOLD_MS,
type ManagerState,
type ManagerEvent,
@@ -214,6 +216,7 @@ export interface ManagerRuntimeOpts {
credentialProxy?: HostLaunchContext["credentialProxy"];
staleThresholdMs?: number;
idleTimeoutMs?: number;
+ idleResetTimeoutMs?: number;
resetStuckThresholdMs?: number;
stoppingStuckThresholdMs?: number;
handshakeTimeoutMs?: number;
@@ -481,7 +484,8 @@ export class AgentProcessManager {
this.opts = {
tickIntervalMs: 5_000,
staleThresholdMs: 120_000,
- idleTimeoutMs: 300_000,
+ idleTimeoutMs: DEFAULT_IDLE_TIMEOUT_MS,
+ idleResetTimeoutMs: DEFAULT_IDLE_RESET_TIMEOUT_MS,
resetStuckThresholdMs: 120_000,
stoppingStuckThresholdMs: DEFAULT_STOPPING_STUCK_THRESHOLD_MS,
handshakeTimeoutMs: 60_000,
@@ -495,6 +499,7 @@ export class AgentProcessManager {
this.opts.idleTimeoutMs,
this.opts.resetStuckThresholdMs,
this.opts.stoppingStuckThresholdMs,
+ this.opts.idleResetTimeoutMs,
);
}
register(agentId: string, launch?: { runtimeConfig?: RuntimeConfig; sessionId?: string; launchId?: string }): void {
@@ -515,8 +520,12 @@ export class AgentProcessManager {
const effects = this.dispatch({ type: "wake", agentId, message: normalized, nowMs: this.now() });
return effects.length > 0;
}
- forgetSession(agentId: string, barrierType: "reset_session" | "nap" = "reset_session"): boolean {
- if (!this.forgetSessionSources(agentId, barrierType)) return false;
+ forgetSession(
+ agentId: string,
+ barrierType: "reset_session" | "nap" = "reset_session",
+ forgottenSessionId?: string,
+ ): boolean {
+ if (!this.forgetSessionSources(agentId, barrierType, forgottenSessionId)) return false;
this.dispatch({ type: "reset_session", agentId });
return true;
}
@@ -1205,6 +1214,34 @@ export class AgentProcessManager {
mode: effect.mode,
});
break;
+ case "reset_idle_session": {
+ const spawnState = this.activeSpawnState.get(effect.agentId);
+ const persisted = this.forgetSession(
+ effect.agentId,
+ "reset_session",
+ effect.sessionId,
+ );
+ if (!persisted) {
+ this.log.error("idle session reset barrier was not persisted; reset deferred", {
+ agentId: effect.agentId,
+ sessionId: effect.sessionId,
+ });
+ this.emitErrorAudit(
+ effect.agentId,
+ "reset",
+ "resume_control_update_failed",
+ "Idle session reset deferred because resume control could not be persisted",
+ );
+ break;
+ }
+ if (spawnState) spawnState.discardEvents = true;
+ this.dispatch({ type: "idle_reset_committed", agentId: effect.agentId, nowMs: this.now() });
+ this.log.info("idle agent session reset", {
+ agentId: effect.agentId,
+ sessionId: effect.sessionId,
+ });
+ break;
+ }
case "force_exit": {
const session = this.sessions.get(effect.agentId);
const state = this.activeSpawnState.get(effect.agentId);