A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, bug] fix(daemon): answer heartbeats_list from live state without queueing behind serialized work (#2378)
Seth Karten3dfb0276b413
diff --git a/packages/coding-agent/.changes/heartbeats-list-response-latency.md b/packages/coding-agent/.changes/heartbeats-list-response-latency.mdnew file mode 100644index 0000000000..2717be099c--- /dev/null+++ b/packages/coding-agent/.changes/heartbeats-list-response-latency.md@@ -0,0 +1 @@+- Fixed `heartbeats_list` timing out on busy daemons: the supervisor now answers from a shared in-memory scheduled-jobs snapshot (one in-flight disk scan serves every concurrent catalog request, heartbeat mutations drop the snapshot, aged snapshots refresh in the background), and sibling name scans no longer hold the serialized ledger queue that scheduled-catalog topology reads wait on.diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.tsindex e1a76ea24e..2b8b34fc04 100644--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts@@ -208,6 +208,10 @@ export const HEARTBEAT_LIST_LAUNCH_WAIT_MS = 15_000;// worker request default (24h) would turn a stuck worker into a client transport// timeout instead of a daemon-side failure.export const HEARTBEAT_LIST_FORWARD_TIMEOUT_MS = 25_000;+// A served scheduled-jobs snapshot older than this refreshes in the background+// (stale-while-revalidate) so external artifact writes converge without ever+// blocking a catalog response behind a fresh saved-session disk scan.+const PASSIVE_SCHEDULED_JOBS_REFRESH_MS = 5_000;const INPUT_PAUSE_CLEANUP_TIMEOUT_MS = 5_000;const UPDATE_RESTART_MUTATION_DRAIN_TIMEOUT_MS = 80_000;const UPDATE_RESTART_WORKER_REQUEST_TIMEOUT_MS = 90_000;@@ -427,6 +431,18 @@ interface WorkerMatch {summary: SessionSummary;}+/** A durable scheduled job whose owning session has no resident worker. */+interface PassiveScheduledJob {+ rootSessionFile: string;+ job: AgentCronJob;+ info: SessionInfo;+}++/** Catalog rows keep the same status filter a fresh scan applies. */+function activeScheduledJobs(rows: PassiveScheduledJob[]): PassiveScheduledJob[] {+ return rows.filter((row) => row.job.status === "active" || row.job.status === "paused");+}+interface WorkerAttachData {result: DaemonAttachResult;worker: ResidentWorker;@@ -774,6 +790,16 @@ export class DaemonSupervisor {private scheduledWakeRecompute?: Promise<void>;private scheduledWakeRecomputeQueued = false;private readonly scheduledWakeFailures = new Map<string, number>();+ /** Shared passive scheduled-jobs snapshot; undefined before the first scan and after each invalidation. */+ private passiveScheduledJobs?: { rows: PassiveScheduledJob[]; scannedAt: number };+ /** One in-flight scan shared by every concurrent catalog read. */+ private passiveScheduledJobsScan?: Promise<PassiveScheduledJob[]>;+ /**+ * Monotonic epoch claimed at each scan start and at each invalidation: only+ * the newest claim may publish, so an older scan cannot overwrite a newer+ * scan's rows and a scan that raced a mutation cannot republish stale rows.+ */+ private passiveScheduledJobsEpoch = 0;constructor(private readonly socketPath: string,@@ -952,10 +978,109 @@ export class DaemonSupervisor {});}- /** Durable truth: the ledger family (fork headers stripped) plus each session's scheduled-jobs artifact. */- private async collectPassiveScheduledJobs(- includeInactive = false,- ): Promise<Array<{ rootSessionFile: string; job: AgentCronJob; info: SessionInfo }>> {+ /**+ * Durable truth: the ledger family (fork headers stripped) plus each session's+ * scheduled-jobs artifact. Always scans fresh per caller: management+ * decisions, wake recomputes, and the ephemeral-cancel retries they drive+ * must see the latest disk state. Every completed scan opportunistically+ * refreshes the snapshot that heartbeats_list serves, but only while it is+ * still the newest scan: an older scan loses the publish race.+ */+ private async collectPassiveScheduledJobs(includeInactive = false): Promise<PassiveScheduledJob[]> {+ const epoch = this.claimPassiveScheduledJobsEpoch();+ const rows = await this.scanPassiveScheduledJobs();+ this.storePassiveScheduledJobs(epoch, rows);+ return includeInactive ? rows : activeScheduledJobs(rows);+ }++ /**+ * Snapshot-served rows for heartbeats_list responses, where per-request+ * freshness matters less than not queueing: with hundreds of saved sessions,+ * N concurrent catalog requests used to enqueue N full scans and push the+ * last response past the client's 30s transport timeout. Daemon-owned+ * mutations drop the snapshot and detach the in-flight shared scan+ * (broadcastHeartbeatsChanged and the saved-session delete/rename paths);+ * the next list or the wake recompute armed by the same broadcast rescans,+ * and concurrent cold lists share one in-flight scan. A served snapshot+ * older than PASSIVE_SCHEDULED_JOBS_REFRESH_MS refreshes in the background+ * without being dropped, so lists that arrive during the refresh keep+ * serving bounded-stale rows and external artifact writes converge without+ * blocking a response.+ */+ private async catalogPassiveScheduledJobs(): Promise<PassiveScheduledJob[]> {+ const cached = this.passiveScheduledJobs;+ if (cached) {+ if (Date.now() - cached.scannedAt >= PASSIVE_SCHEDULED_JOBS_REFRESH_MS && !this.shuttingDown) {+ // Stale-while-revalidate: serve the snapshot now and refresh in the+ // background without dropping it, so requests that arrive during the+ // refresh keep answering from bounded-stale rows instead of queueing+ // behind the scan. A failure only logs; the next read retries.+ void this.sharedPassiveScheduledJobsScan().catch((error: unknown) =>+ this.log(`Passive scheduled-jobs refresh failed: ${String(error)}`),+ );+ }+ return activeScheduledJobs(cached.rows);+ }+ return activeScheduledJobs(await this.sharedPassiveScheduledJobsScan());+ }++ private sharedPassiveScheduledJobsScan(): Promise<PassiveScheduledJob[]> {+ this.passiveScheduledJobsScan ??= this.startSharedPassiveScheduledJobsScan();+ return this.passiveScheduledJobsScan;+ }++ /**+ * Claims the publish epoch when the scan actually starts, so callers that+ * join an existing in-flight scan keep its rows, while a caller that starts+ * after an invalidation never adopts the detached pre-mutation scan.+ */+ private startSharedPassiveScheduledJobsScan(): Promise<PassiveScheduledJob[]> {+ const epoch = this.claimPassiveScheduledJobsEpoch();+ const scan = this.scanPassiveScheduledJobs()+ .then((rows) => {+ this.storePassiveScheduledJobs(epoch, rows);+ return rows;+ })+ .finally(() => {+ // An invalidation may have detached this promise and let a newer+ // scan take the slot; clear it only while it is still ours.+ if (this.passiveScheduledJobsScan === scan) {+ this.passiveScheduledJobsScan = undefined;+ }+ });+ return scan;+ }++ /**+ * Compare-and-swap publish: a scan may store only while it still owns the+ * newest epoch, so an older scan cannot overwrite a newer snapshot and a+ * scan that raced an invalidation cannot republish pre-mutation rows.+ */+ private storePassiveScheduledJobs(epoch: number, rows: PassiveScheduledJob[]): void {+ if (epoch === this.passiveScheduledJobsEpoch) {+ this.passiveScheduledJobs = { rows, scannedAt: Date.now() };+ }+ }++ private claimPassiveScheduledJobsEpoch(): number {+ this.passiveScheduledJobsEpoch += 1;+ return this.passiveScheduledJobsEpoch;+ }++ /**+ * Daemon-owned mutations invalidate: claim the epoch so in-flight scans can+ * no longer publish, drop the cached snapshot, and detach the shared+ * in-flight promise so a list that starts after the mutation rescans+ * instead of awaiting pre-mutation rows.+ */+ private invalidatePassiveScheduledJobs(): void {+ this.claimPassiveScheduledJobsEpoch();+ this.passiveScheduledJobs = undefined;+ this.passiveScheduledJobsScan = undefined;+ }++ /** The uncached disk scan; all statuses load so one row set serves every caller's filter. */+ private async scanPassiveScheduledJobs(): Promise<PassiveScheduledJob[]> {const pendingCancelRoots = new Set<string>();for (const intent of this.collectEphemeralCancelIntents()) {const context = this.workerSessionArtifactContext(intent);@@ -999,7 +1124,7 @@ export class DaemonSupervisor {}return current.path;};- const results: Array<{ rootSessionFile: string; job: AgentCronJob; info: SessionInfo }> = [];+ const results: PassiveScheduledJob[] = [];for (const [artifactSessionId, store] of storeBySessionId) {let jobs: AgentCronJob[];try {@@ -1009,7 +1134,6 @@ export class DaemonSupervisor {continue;}for (const job of jobs) {- if (!includeInactive && job.status !== "active" && job.status !== "paused") continue;const info = infoBySessionId.get(job.sessionId);if (!info) continue;const rootSessionFile = uncoveredRootFor(info);@@ -2399,7 +2523,7 @@ export class DaemonSupervisor {}}// Passivated sessions keep their armed heartbeats; no worker can list them.- for (const { job, info } of await this.collectPassiveScheduledJobs()) {+ for (const { job, info } of await this.catalogPassiveScheduledJobs()) {if (!isHeartbeatCronJob(job) || heartbeats.has(job.id)) continue;heartbeats.set(job.id, {job,@@ -2540,6 +2664,9 @@ export class DaemonSupervisor {if (entry) {this.writeRosterEntry({ ...entry, summary: { ...entry.summary, sessionName: target.name } });}+ // Heartbeat rows carry the owning session's display name;+ // drop the snapshot so the next list shows the new one.+ this.invalidatePassiveScheduledJobs();return success(command.id, command.type);}const match = await this.findWorkerForClient(client, command.activeSessionId);@@ -2574,6 +2701,9 @@ export class DaemonSupervisor {if (result.ok && entry && this.roster().get(entry.agentId) === entry) {this.roster().delete(entry.agentId);}+ // The deleted session's armed heartbeats must leave the catalog+ // snapshot too; their rows would otherwise linger until the TTL.+ if (result.ok) this.invalidatePassiveScheduledJobs();return success(command.id, command.type, result);}break;@@ -6971,6 +7101,10 @@ export class DaemonSupervisor {}private broadcastHeartbeatsChanged(): void {+ // Every daemon-owned scheduled-job mutation and worker residency change+ // lands here: drop the shared snapshot so the next catalog read (or the+ // recompute armed below) rescans instead of serving pre-mutation rows.+ this.invalidatePassiveScheduledJobs();this.scheduleScheduledSessionWakeRecompute();for (const client of this.clients) {this.write(client, { type: "heartbeats_changed" });diff --git a/packages/coding-agent/src/modes/daemon/rlm-ledger.ts b/packages/coding-agent/src/modes/daemon/rlm-ledger.tsindex 635d92125c..56a3c44a34 100644--- a/packages/coding-agent/src/modes/daemon/rlm-ledger.ts+++ b/packages/coding-agent/src/modes/daemon/rlm-ledger.ts@@ -430,45 +430,58 @@ export class RlmSpawnLedger {/** Same-parent rows for a child session path, including the child itself. */siblings(sessionPath: string): Promise<SessionInfo[]> {- return this.enqueue(async () => {- const target = cachedCanonicalSessionPath(sessionPath);- const family = await this.familyUnlocked(await this.liveEdgesUnlocked());- const edges = [...this.replaySyncCached().values()].filter((edge) => !edge.deleted);- const parentByChild = new Map(- edges.map((edge) => [cachedCanonicalSessionPath(edge.child), cachedCanonicalSessionPath(edge.parent)]),- );- const parent = parentByChild.get(target);- if (parent !== undefined) {- const rows = family.filter((row) => parentByChild.get(cachedCanonicalSessionPath(row.path)) === parent);- // The target's edge can be reconciliation-dropped (parent file- // gone) while its own file still exists: fall back to presenting- // the survivor alone rather than an empty set the callers would- // read as "session not found".- if (!rows.some((row) => cachedCanonicalSessionPath(row.path) === target)) {- try {- if ((await stat(target)).isFile()) {- return [await this.sessionRow(target, 0, undefined, undefined)];- }- } catch {- // fall through to the (possibly empty) sibling rows+ // Capture ordered topology first; transcript metadata must not block+ // spawn appends or scheduled-catalog topology reads behind a full+ // saved-session scan (same split as family()).+ return this.enqueue(async () => ({+ alive: await this.liveEdgesUnlocked(),+ edges: [...this.replaySyncCached().values()].filter((edge) => !edge.deleted),+ })).then((topology) => this.siblingsUnlocked(sessionPath, topology));+ }++ private async siblingsUnlocked(+ sessionPath: string,+ topology: { alive: RlmLedgerEdge[]; edges: RlmLedgerEdge[] },+ ): Promise<SessionInfo[]> {+ const target = cachedCanonicalSessionPath(sessionPath);+ const family = await this.familyUnlocked(topology.alive);+ const parentByChild = new Map(+ topology.edges.map((edge) => [+ cachedCanonicalSessionPath(edge.child),+ cachedCanonicalSessionPath(edge.parent),+ ]),+ );+ const parent = parentByChild.get(target);+ if (parent !== undefined) {+ const rows = family.filter((row) => parentByChild.get(cachedCanonicalSessionPath(row.path)) === parent);+ // The target's edge can be reconciliation-dropped (parent file+ // gone) while its own file still exists: fall back to presenting+ // the survivor alone rather than an empty set the callers would+ // read as "session not found".+ if (!rows.some((row) => cachedCanonicalSessionPath(row.path) === target)) {+ try {+ if ((await stat(target)).isFile()) {+ return [await this.sessionRow(target, 0, undefined, undefined)];}+ } catch {+ // fall through to the (possibly empty) sibling rows}- return rows;- }- // Roots are siblings of the other roots. A session outside both the- // ledger and the sessions dir is presented alone (matching the- // registry-walking reader's behavior for parentless sessions).- const roots = family.filter((row) => row.rlmDepth === 0);- if (roots.some((row) => cachedCanonicalSessionPath(row.path) === target)) {- return roots;- }- try {- if (!(await stat(target)).isFile()) return [];- } catch {- return [];}- return [await this.sessionRow(target, 0, undefined, undefined)];- });+ return rows;+ }+ // Roots are siblings of the other roots. A session outside both the+ // ledger and the sessions dir is presented alone (matching the+ // registry-walking reader's behavior for parentless sessions).+ const roots = family.filter((row) => row.rlmDepth === 0);+ if (roots.some((row) => cachedCanonicalSessionPath(row.path) === target)) {+ return roots;+ }+ try {+ if (!(await stat(target)).isFile()) return [];+ } catch {+ return [];+ }+ return [await this.sessionRow(target, 0, undefined, undefined)];}private enqueue<T>(fn: () => Promise<T> | T): Promise<T> {diff --git a/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts b/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.tsnew file mode 100644index 0000000000..702dc39a3c--- /dev/null+++ b/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts@@ -0,0 +1,83 @@+import { realpathSync } from "node:fs";+import { join } from "node:path";+import { afterEach, describe, expect, it, vi } from "vitest";+import { AgentCronJobStore } from "../../../src/core/cron-jobs.js";+import * as sessionManager from "../../../src/core/session-manager.js";+import { DaemonSupervisor } from "../../../src/modes/daemon/daemon-supervisor.js";+import { createHarness } from "../harness.js";+import { createDeferred } from "../scheduling.js";++const harnesses: Array<{ cleanup: () => void }> = [];+afterEach(() => {+ vi.restoreAllMocks();+ for (const harness of harnesses.splice(0)) harness.cleanup();+});+async function createSupervisorHarness() {+ const harness = await createHarness();+ harnesses.push(harness);+ const directory = harness.tempDir;+ return new DaemonSupervisor(join(directory, "daemon.sock"), {+ defaultSessionConfig: { agentDir: directory, cwd: directory },+ descriptorDir: join(directory, "workers"),+ }) as unknown as {+ defaultSessionConfig: { agentDir: string };+ rlmSpawnLedger: DaemonSupervisor["rlmSpawnLedger"];+ handleCommand: DaemonSupervisor["handleCommand"];+ broadcastHeartbeatsChanged: DaemonSupervisor["broadcastHeartbeatsChanged"];+ passiveScheduledJobs?: { rows: unknown[]; scannedAt: number };+ };+}+type Supervisor = Awaited<ReturnType<typeof createSupervisorHarness>>;+function listHeartbeats(supervisor: Supervisor, id: string) {+ return supervisor.handleCommand({} as never, { id, type: "heartbeats_list" });+}+function heartbeatIds(response: Awaited<ReturnType<Supervisor["handleCommand"]>>): string[] {+ const rows = (response as { data?: { heartbeats?: Array<{ job: { id: string } }> } })?.data?.heartbeats;+ return (rows ?? []).map((h) => h.job.id);+}+describe("heartbeats_list response latency", () => {+ it("serves heartbeats_list from one shared scan without queueing behind metadata reads", async () => {+ const supervisor = await createSupervisorHarness();+ const directory = realpathSync(supervisor.defaultSessionConfig.agentDir);+ const manager = sessionManager.SessionManager.create(directory, join(directory, "sessions"));+ manager.newSession();+ manager.flushNow();+ const store = AgentCronJobStore.forSessionArtifacts();+ store.registerSessionArtifact(manager.getSessionId(), manager.getSessionArtifactDir()!);+ const job = store.createHeartbeat({+ activeSessionId: manager.getSessionId(),+ sessionId: manager.getSessionId(),+ sessionFile: manager.getSessionFile()!,+ cwd: directory,+ scheduleText: "every 1h",+ prompt: "continue",+ }).id;+ const family = vi.spyOn(supervisor.rlmSpawnLedger(), "family");+ const responses = await Promise.all(["1", "2", "3", "4", "5"].map((id) => listHeartbeats(supervisor, id)));+ expect(responses.map(heartbeatIds)).toEqual([[job], [job], [job], [job], [job]]); // one scan served five lists+ await listHeartbeats(supervisor, "6");+ expect(family).toHaveBeenCalledTimes(1); // all six lists shared one scan, then served from the snapshot+ const readSessionInfo = sessionManager.readSessionInfo;+ const [scanStarted, releaseScan] = [createDeferred(), createDeferred()];+ vi.spyOn(sessionManager, "readSessionInfo").mockImplementationOnce(async (...args) => {+ scanStarted.resolve();+ await releaseScan.promise;+ return readSessionInfo(...args);+ });+ supervisor.passiveScheduledJobs = undefined; // force the next list to scan cold+ const siblings = supervisor.rlmSpawnLedger().siblings(manager.getSessionFile()!);+ await scanStarted.promise;+ try {+ expect(heartbeatIds(await listHeartbeats(supervisor, "cold"))).toEqual([job]);+ } finally {+ releaseScan.resolve();+ await siblings;+ }+ store.manageHeartbeat(manager.getSessionId(), job, "pause");+ supervisor.broadcastHeartbeatsChanged();+ expect(heartbeatIds(await listHeartbeats(supervisor, "7"))).toEqual([job]); // broadcast dropped the snapshot+ store.manageHeartbeat(manager.getSessionId(), job, "stop");+ supervisor.broadcastHeartbeatsChanged();+ expect(heartbeatIds(await listHeartbeats(supervisor, "8"))).toEqual([]);+ });+});