Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[RSI, performance] stop the heartbeat/scheduler fan-out to every worker, client, and artifact path (#2487)

Sebastian Müllerb1d82f0ce5ed
diff --git a/packages/coding-agent/.changes/scheduler-fanout.md b/packages/coding-agent/.changes/scheduler-fanout.md
new file mode 100644
index 0000000000..605a63fab4
--- /dev/null
+++ b/packages/coding-agent/.changes/scheduler-fanout.md
@@ -0,0 +1 @@
+- Stopped fanning supervisor heartbeat lists, scheduled-job wake recomputes, and cron-store mutations out to every worker, client, and artifact path: worker heartbeat snapshots refresh per reporting worker, the wake timer arms from a cached aggregate (a changed stale-while-revalidate refresh re-arms it), heartbeats_changed pushes reach only clients that track heartbeats (by scheduled-job command or the heartbeat_catalog attach capability; daemon-side lookups now also resolve paused and cancelled passive jobs), and cron-store mutations lock just the paths they write with one fsync and non-blocking retries.
diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts
index 8d7a60660d..806570fd61 100644
--- a/packages/coding-agent/src/core/agent-session.ts
+++ b/packages/coding-agent/src/core/agent-session.ts
@@ -1794,7 +1794,8 @@ export class AgentSession {
this._pendingNextTurnMessages.push(createGoalContextMessage(this._goalState, "continuation"));
}
this._restoreLateIpythonSentAgentMessages();
- this._restoreQuotaPark();
+ // Cannot await in a constructor; a live wake job restores within a microtask, only a rebuilt job waits on the store.
+ void this._restoreQuotaPark();
if (this._goalState.status === "active") {
this._goalAccountingStartedAt = Date.now();
}
@@ -4053,7 +4054,10 @@ export class AgentSession {
* These heartbeats are internal to this active session and never read or
* mutate the user-level /heartbeat.
*/
- handleRlmHeartbeatHostRequest(type: string, payload: Record<string, unknown> = {}): Record<string, unknown> {
+ async handleRlmHeartbeatHostRequest(
+ type: string,
+ payload: Record<string, unknown> = {},
+ ): Promise<Record<string, unknown>> {
const controller = this._rlmHeartbeatController;
if (!controller) {
throw new Error("RLM heartbeat skill is not available in this session");
@@ -4080,7 +4084,7 @@ export class AgentSession {
const deliveryMode = normalizeHeartbeatDeliveryMode(payload.delivery_mode ?? payload.deliveryMode);
return {
heartbeat: rlmHeartbeatHostResponse(
- controller.createRlmHeartbeat({
+ await controller.createRlmHeartbeat({
instruction: payload.instruction,
interval: payload.interval,
label: payload.label,
@@ -4116,7 +4120,7 @@ export class AgentSession {
) {
throw new Error("rlm_heartbeat.update requires at least one field to update");
}
- const heartbeat = controller.updateRlmHeartbeat({
+ const heartbeat = await controller.updateRlmHeartbeat({
id: payload.id,
instruction: payload.instruction,
interval: payload.interval,
@@ -4132,7 +4136,7 @@ export class AgentSession {
if (typeof payload.id !== "string") {
throw new Error("rlm_heartbeat.delete id must be a string");
}
- const heartbeat = controller.deleteRlmHeartbeat(payload.id);
+ const heartbeat = await controller.deleteRlmHeartbeat(payload.id);
return {
heartbeat: heartbeat ? rlmHeartbeatHostResponse(heartbeat) : null,
};
@@ -4622,11 +4626,11 @@ export class AgentSession {
this._retryAuthFailureSources = [];
}
if (assistantMsg.stopReason === "aborted") {
- this._handleAbortedQuotaPark();
+ await this._handleAbortedQuotaPark();
} else if (assistantMsg.stopReason !== "error" && this._quotaPark) {
// A parked session that completes a model call has its quota back:
// clear the park (cancelling the pending wake) and resume the task.
- this._completeQuotaParkResume();
+ await this._completeQuotaParkResume();
}
if (this._accountGoalUsageForAssistantMessage(assistantMsg)) {
const message = createGoalContextMessage(this._goalState, "budget_limit");
@@ -4674,7 +4678,7 @@ export class AgentSession {
this._finishActiveRetryWithFailure(msg);
this._resolveRetry();
if (!compactionWillRetry) {
- this._handleErroredQuotaParkProbe(msg);
+ await this._handleErroredQuotaParkProbe(msg);
this._finishGoalForTerminalAssistantMessage(msg);
// In serialized mode, agent-callable refine.run is serviced
// at the shouldStopAfterTurn boundary, not here at agent_end.
@@ -13272,7 +13276,7 @@ export class AgentSession {
// Future-scheduled parks survive; only stale post-wake parks clear.
const stalePark = this._quotaPark;
if (reason === "usage" && stalePark !== undefined && stalePark.resumeAtMs <= Date.now()) {
- this._cancelQuotaParkWake(stalePark);
+ await this._cancelQuotaParkWake(stalePark);
this._quotaPark = undefined;
}
this._markProviderAuthStaleForRetryFailure(message, options);
@@ -13313,7 +13317,7 @@ export class AgentSession {
* reset time. While parked the session makes no model calls; the wake
* delivers the resume marker, whose first model call probes the quota.
*/
- private _parkForQuotaReset(
+ private async _parkForQuotaReset(
message: AssistantMessage,
options:
| {
@@ -13324,7 +13328,7 @@ export class AgentSession {
pauseMs: number,
abortMessage: string,
parkedAttempt: number,
- ): boolean {
+ ): Promise<boolean> {
const existing = this._quotaPark;
if (existing !== undefined && existing.resumeAtMs > Date.now()) {
// Already parked for this window (e.g. a heartbeat turn failed while
@@ -13337,10 +13341,10 @@ export class AgentSession {
);
return false;
}
- this._cancelQuotaParkWake(existing);
+ await this._cancelQuotaParkWake(existing);
const parkCount = (existing?.parkCount ?? 0) + 1;
const resumeAtMs = Date.now() + pauseMs;
- const jobId = this._createQuotaResumeJob(resumeAtMs);
+ const jobId = await this._createQuotaResumeJob(resumeAtMs);
const timer = this._scheduleQuotaResumeTimer(resumeAtMs);
this._quotaPark = {
parkCount,
@@ -13389,14 +13393,14 @@ export class AgentSession {
* resume marker at the reset time. Best-effort: the in-process timer covers
* live sessions when this cannot be persisted (e.g. in-memory sessions).
*/
- private _createQuotaResumeJob(resumeAtMs: number): string | undefined {
+ private async _createQuotaResumeJob(resumeAtMs: number): Promise<string | undefined> {
const sessionFile = this.sessionFile;
const store = this._quotaResumeStore();
if (!sessionFile || !store) {
return undefined;
}
try {
- const job = store.create({
+ const job = await store.create({
activeSessionId: this.sessionId,
sessionId: this.sessionId,
sessionFile,
@@ -13444,7 +13448,7 @@ export class AgentSession {
* that has not run, and report a user cancellation as such. Anything else is
* gone, leaving the in-process timer as the wake.
*/
- private _resolveQuotaResumeJob(jobId: string): "delivered" | "user-cancelled" | "cancelled" | "gone" {
+ private async _resolveQuotaResumeJob(jobId: string): Promise<"delivered" | "user-cancelled" | "cancelled" | "gone"> {
const job = this._findQuotaResumeJob(jobId);
if (job?.status === "completed") {
return "delivered";
@@ -13457,23 +13461,23 @@ export class AgentSession {
return "gone";
}
try {
- return store.cancel(jobId) === undefined ? "gone" : "cancelled";
+ return (await store.cancel(jobId)) === undefined ? "gone" : "cancelled";
} catch {
return "gone";
}
}
/** Cancel a park's pending wake: the in-process timer and the durable job. */
- private _cancelQuotaParkWake(
+ private async _cancelQuotaParkWake(
park: { resumeAtMs: number; jobId?: string; timer?: ReturnType<typeof setTimeout> } | undefined,
- ): void {
+ ): Promise<void> {
if (!park) return;
if (park.timer) {
clearTimeout(park.timer);
park.timer = undefined;
}
if (park.jobId !== undefined) {
- this._resolveQuotaResumeJob(park.jobId);
+ await this._resolveQuotaResumeJob(park.jobId);
}
park.jobId = undefined;
}
@@ -13491,7 +13495,7 @@ export class AgentSession {
return;
}
if (park.jobId !== undefined) {
- const resolved = this._resolveQuotaResumeJob(park.jobId);
+ const resolved = await this._resolveQuotaResumeJob(park.jobId);
if (resolved === "delivered") {
// The daemon dispatched the durable wake; its prompt drives the resume.
park.waking = true;
@@ -13515,7 +13519,7 @@ export class AgentSession {
// A refused admission must not leave a park whose wake is gone: re-arm
// it (bounded) so the session still resumes, or drop the park.
park.waking = false;
- this._recoverQuotaParkWake("wake-failed");
+ await this._recoverQuotaParkWake("wake-failed");
}
}
@@ -13524,13 +13528,13 @@ export class AgentSession {
* marker is still queued owns the resume, so an abort of some other turn must
* not re-arm the wake under it; a wake this turn consumed re-arms instead.
*/
- private _handleAbortedQuotaPark(): void {
+ private async _handleAbortedQuotaPark(): Promise<void> {
const park = this._quotaPark;
if (!park || (park.waking === true && this._hasQueuedQuotaResumeMarker())) {
return;
}
park.waking = false;
- this._recoverQuotaParkWake("wake-aborted");
+ await this._recoverQuotaParkWake("wake-aborted");
}
/** Whether the park wake's resume marker is still waiting in the session input queue. */
@@ -13551,13 +13555,13 @@ export class AgentSession {
* or drop the park once the retries are spent. A marker still queued owns
* the resume, so an error from another turn must not re-arm under it.
*/
- private _handleErroredQuotaParkProbe(message: AssistantMessage): void {
+ private async _handleErroredQuotaParkProbe(message: AssistantMessage): Promise<void> {
const park = this._quotaPark;
if (message.stopReason !== "error" || park?.waking !== true || this._hasQueuedQuotaResumeMarker()) {
return;
}
park.waking = false;
- this._recoverQuotaParkWake("wake-error");
+ await this._recoverQuotaParkWake("wake-error");
}
/**
@@ -13566,21 +13570,21 @@ export class AgentSession {
* never wake ends instead of staying parked with no wake and no way to
* resume.
*/
- private _recoverQuotaParkWake(outcome: "wake-failed" | "wake-aborted" | "wake-error"): void {
+ private async _recoverQuotaParkWake(outcome: "wake-failed" | "wake-aborted" | "wake-error"): Promise<void> {
const park = this._quotaPark;
if (!park || park.waking || park.resumeAtMs > Date.now()) {
return;
}
const retries = (park.wakeRetries ?? 0) + 1;
if (retries > QUOTA_WAKE_MAX_RETRIES) {
- this._cancelQuotaParkWake(park);
+ await this._cancelQuotaParkWake(park);
this._quotaPark = undefined;
this.sessionManager.appendCustomEntry(QUOTA_RESUME_CUSTOM_ENTRY_TYPE, { outcome });
return;
}
park.wakeRetries = retries;
park.resumeAtMs = Date.now() + QUOTA_WAKE_RETRY_DELAY_MS;
- park.jobId = this._createQuotaResumeJob(park.resumeAtMs);
+ park.jobId = await this._createQuotaResumeJob(park.resumeAtMs);
park.timer = this._scheduleQuotaResumeTimer(park.resumeAtMs);
// Record the replacement wake, or a restart reads the spent park entry,
// drops the park, and leaves this retry job armed with no owner to cancel.
@@ -13608,7 +13612,10 @@ export class AgentSession {
* recreate one that was cancelled (a navigation cancels the left-behind
* leaf's wake) or removed, so a restored park never waits on a dead job.
*/
- private _restoreQuotaWakeJob(jobId: string | undefined, resumeAtMs: number): string | undefined | "user-cancelled" {
+ private async _restoreQuotaWakeJob(
+ jobId: string | undefined,
+ resumeAtMs: number,
+ ): Promise<string | undefined | "user-cancelled"> {
if (jobId === undefined) {
return this._createQuotaResumeJob(resumeAtMs);
}
@@ -13630,7 +13637,7 @@ export class AgentSession {
* wake is cancelled with it, so a parked leaf that was left behind cannot
* resume its task on the selected branch.
*/
- private _reloadQuotaParkFromBranch(): void {
+ private async _reloadQuotaParkFromBranch(): Promise<void> {
const previous = this._quotaPark;
this._quotaPark = undefined;
if (previous?.timer) {
@@ -13640,11 +13647,11 @@ export class AgentSession {
if (previous?.jobId !== undefined) {
// Only a wake this navigation actually cancels may be rebuilt on the way
// back; a wake the user cancelled in /cron stays cancelled.
- if (this._resolveQuotaResumeJob(previous.jobId) === "cancelled") {
+ if ((await this._resolveQuotaResumeJob(previous.jobId)) === "cancelled") {
this._navigationCancelledWakeJobs.add(previous.jobId);
}
}
- this._restoreQuotaPark();
+ await this._restoreQuotaPark();
}
/**
@@ -13655,7 +13662,7 @@ export class AgentSession {
* a timer when its durable job still owns the wake; with no job the
* in-process timer wakes it at once.
*/
- private _restoreQuotaPark(): void {
+ private async _restoreQuotaPark(): Promise<void> {
const branch = this.sessionManager.getBranch();
for (let index = branch.length - 1; index >= 0; index -= 1) {
const entry = branch[index];
@@ -13672,7 +13679,7 @@ export class AgentSession {
if (!Number.isFinite(resumeAtMs)) {
return;
}
- const jobId = this._restoreQuotaWakeJob(entry.data.jobId, resumeAtMs);
+ const jobId = await this._restoreQuotaWakeJob(entry.data.jobId, resumeAtMs);
if (jobId === "user-cancelled") {
return;
}
@@ -13709,14 +13716,14 @@ export class AgentSession {
* transition, and — unless this success WAS the wake probe — deliver the
* resume marker so the interrupted task continues right away.
*/
- private _completeQuotaParkResume(): void {
+ private async _completeQuotaParkResume(): Promise<void> {
const park = this._quotaPark;
if (!park) return;
// A wake the daemon already delivered owns the resume even when the timer
// never observed it: the job's marker prompt is the continuation, so this
// success must not queue a second one.
- const delivered = park.jobId !== undefined && this._resolveQuotaResumeJob(park.jobId) === "delivered";
- this._cancelQuotaParkWake(park);
+ const delivered = park.jobId !== undefined && (await this._resolveQuotaResumeJob(park.jobId)) === "delivered";
+ await this._cancelQuotaParkWake(park);
const wasWaking = park.waking === true || delivered;
const restoredModel = this._restorePrimaryModelAfterBackup();
this._quotaPark = undefined;
@@ -14357,7 +14364,7 @@ export class AgentSession {
this._ensureHarnessDigestContext();
this._reloadGoalStateFromBranch({ monotonicTokens: Boolean(summaryText) });
this._reloadRlmMaxDepthFromBranch();
- this._reloadQuotaParkFromBranch();
+ await this._reloadQuotaParkFromBranch();
this._invalidateQueuedPromptPreparation();
await this._extensionRunner.emit({
diff --git a/packages/coding-agent/src/core/cron-jobs.ts b/packages/coding-agent/src/core/cron-jobs.ts
index 6ceca4df86..edf1979412 100644
--- a/packages/coding-agent/src/core/cron-jobs.ts
+++ b/packages/coding-agent/src/core/cron-jobs.ts
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, statSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
-import { lockSync } from "proper-lockfile";
+import { lock } from "proper-lockfile";
import { writeFileAtomicSync } from "../utils/atomic-file.js";
import { getSessionArtifactPathForFile } from "./session-manager.js";
@@ -114,6 +114,12 @@ interface CronJobsStateSnapshot {
state: CronJobsState;
}
+/** Mutators must return all outcomes here: mutateStates probes them once and discards that pass's results. */
+interface CronJobsStateMutation<T> {
+ dispatches: AgentCronDispatch[];
+ results: T[];
+}
+
export const SESSION_SCHEDULED_JOBS_FILENAME = "scheduled-jobs.json";
const MAX_TIMEOUT_MS = 2_147_483_647;
@@ -136,7 +142,7 @@ export interface AgentRlmHeartbeatController {
interval?: string;
label?: string;
deliveryMode?: AgentHeartbeatDeliveryMode;
- }): AgentCronJob;
+ }): Promise<AgentCronJob>;
updateRlmHeartbeat(input: {
id: string;
instruction?: string;
@@ -144,8 +150,8 @@ export interface AgentRlmHeartbeatController {
label?: string;
status?: AgentRlmHeartbeatStatusUpdate;
deliveryMode?: AgentHeartbeatDeliveryMode;
- }): AgentCronJob | undefined;
- deleteRlmHeartbeat(id: string): AgentCronJob | undefined;
+ }): Promise<AgentCronJob | undefined>;
+ deleteRlmHeartbeat(id: string): Promise<AgentCronJob | undefined>;
}
function heartbeatCatalogSignature(jobs: readonly AgentCronJob[]): string {
@@ -171,6 +177,20 @@ function heartbeatCatalogSignature(jobs: readonly AgentCronJob[]): string {
);
}
+function isLiveHeartbeatFor(job: AgentCronJob, activeSessionId: string): boolean {
+ return (
+ job.activeSessionId === activeSessionId &&
+ job.source === "heartbeat" &&
+ (job.status === "active" || job.status === "paused")
+ );
+}
+
+function latestHeartbeat(jobs: readonly AgentCronJob[], activeSessionId: string): AgentCronJob | undefined {
+ return jobs
+ .filter((job) => isLiveHeartbeatFor(job, activeSessionId))
+ .sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))[0];
+}
+
export class AgentCronJobStore {
private readonly sessionArtifactFiles = new Map<string, string>();
private readonly heartbeatChangeListeners = new Set<() => void>();
@@ -207,7 +227,7 @@ export class AgentCronJobStore {
return true;
}
- recoverSessionArtifact(sessionId: string, now = new Date()): AgentCronJob[] {
+ async recoverSessionArtifact(sessionId: string, now = new Date()): Promise<AgentCronJob[]> {
const path = this.sessionArtifactFiles.get(sessionId);
if (!path) {
return [];
@@ -227,7 +247,7 @@ export class AgentCronJobStore {
return this.readJobs().sort((a, b) => compareOptionalIso(a.nextRunAt, b.nextRunAt));
}
- create(input: CreateAgentCronJobInput): AgentCronJob {
+ async create(input: CreateAgentCronJobInput): Promise<AgentCronJob> {
const now = input.now ?? new Date();
const prompt = input.prompt.trim();
if (!prompt) {
@@ -252,7 +272,7 @@ export class AgentCronJobStore {
nextRunAt: parsed.nextRunAt.toISOString(),
runCount: 0,
};
- this.writeJobs([...this.readJobs(), job]);
+ await this.writeJobs((jobs) => ({ jobs: [...jobs, job], result: undefined }));
return job;
}
@@ -263,52 +283,43 @@ export class AgentCronJobStore {
* its stable active session id to the new file so future restores target the
* current session instead of the previous one.
*/
- rebindSessionJobs(input: {
+ async rebindSessionJobs(input: {
activeSessionId: string;
sessionId: string;
sessionFile: string;
cwd: string;
- }): AgentCronJob[] {
+ }): Promise<AgentCronJob[]> {
const targetSessionFile = resolve(input.sessionFile);
- const reboundJobs: AgentCronJob[] = [];
- const jobs = this.readJobs().map((job) => {
- if (job.activeSessionId !== input.activeSessionId && resolve(job.sessionFile) !== targetSessionFile) {
- return job;
- }
- if (
- job.activeSessionId === input.activeSessionId &&
- job.sessionId === input.sessionId &&
- resolve(job.sessionFile) === targetSessionFile &&
- job.cwd === input.cwd
- ) {
- return job;
- }
- const rebound = {
- ...job,
- activeSessionId: input.activeSessionId,
- sessionId: input.sessionId,
- sessionFile: input.sessionFile,
- cwd: input.cwd,
- };
- reboundJobs.push(rebound);
- return rebound;
+ return this.writeJobs((jobs) => {
+ const reboundJobs: AgentCronJob[] = [];
+ const next = jobs.map((job) => {
+ if (job.activeSessionId !== input.activeSessionId && resolve(job.sessionFile) !== targetSessionFile) {
+ return job;
+ }
+ if (
+ job.activeSessionId === input.activeSessionId &&
+ job.sessionId === input.sessionId &&
+ resolve(job.sessionFile) === targetSessionFile &&
+ job.cwd === input.cwd
+ ) {
+ return job;
+ }
+ const rebound = {
+ ...job,
+ activeSessionId: input.activeSessionId,
+ sessionId: input.sessionId,
+ sessionFile: input.sessionFile,
+ cwd: input.cwd,
+ };
+ reboundJobs.push(rebound);
+ return rebound;
+ });
+ return { jobs: next, result: reboundJobs };
});
- if (reboundJobs.length > 0) {
- this.writeJobs(jobs);
- }
- return reboundJobs;
}
getHeartbeat(activeSessionId: string): AgentCronJob | undefined {
- return this.readJobs()
- .filter((job) => {
- return (
- job.activeSessionId === activeSessionId &&
- job.source === "heartbeat" &&
- (job.status === "active" || job.status === "paused")
- );
- })
- .sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))[0];
+ return latestHeartbeat(this.readJobs(), activeSessionId);
}
getLatestHeartbeat(activeSessionId: string): AgentCronJob | undefined {
@@ -317,22 +328,12 @@ export class AgentCronJobStore {
.sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))[0];
}
- createHeartbeat(input: CreateAgentCronJobInput): AgentCronJob {
+ async createHeartbeat(input: CreateAgentCronJobInput): Promise<AgentCronJob> {
const now = input.now ?? new Date();
const parsed = parseAgentCronSchedule(input.scheduleText, now);
if (parsed.schedule.kind === "once") {
throw new Error("Heartbeat schedule must be recurring");
}
- const existing = this.readJobs().map((job) => {
- if (
- job.activeSessionId === input.activeSessionId &&
- job.source === "heartbeat" &&
- (job.status === "active" || job.status === "paused")
- ) {
- return { ...job, status: "cancelled" as const, nextRunAt: undefined, updatedAt: now.toISOString() };
- }
- return job;
- });
const prompt = input.prompt.trim();
if (!prompt) {
throw new Error("Heartbeat instruction cannot be empty");
@@ -356,7 +357,17 @@ export class AgentCronJobStore {
nextRunAt: parsed.nextRunAt.toISOString(),
runCount: 0,
};
- this.writeJobs([...existing, job]);
+ await this.writeJobs((jobs) => ({
+ jobs: [
+ ...jobs.map((job) =>
+ isLiveHeartbeatFor(job, input.activeSessionId)
+ ? { ...job, status: "cancelled" as const, nextRunAt: undefined, updatedAt: nowIso }
+ : job,
+ ),
+ job,
+ ],
+ result: undefined,
+ }));
return job;
}
@@ -374,7 +385,7 @@ export class AgentCronJobStore {
.sort((a, b) => compareOptionalIso(a.nextRunAt, b.nextRunAt));
}
- createRlmHeartbeat(input: CreateAgentCronJobInput): AgentCronJob {
+ async createRlmHeartbeat(input: CreateAgentCronJobInput): Promise<AgentCronJob> {
const now = input.now ?? new Date();
const parsed = parseAgentCronSchedule(input.scheduleText, now);
if (parsed.schedule.kind === "once") {
@@ -403,11 +414,11 @@ export class AgentCronJobStore {
nextRunAt: parsed.nextRunAt.toISOString(),
runCount: 0,
};
- this.writeJobs([...this.readJobs(), job]);
+ await this.writeJobs((jobs) => ({ jobs: [...jobs, job], result: undefined }));
return job;
}
- updateRlmHeartbeat(
+ async updateRlmHeartbeat(
activeSessionId: string,
id: string,
update: {
@@ -418,287 +429,288 @@ export class AgentCronJobStore {
deliveryMode?: AgentHeartbeatDeliveryMode;
now?: Date;
},
- ): AgentCronJob | undefined {
+ ): Promise<AgentCronJob | undefined> {
const now = update.now ?? new Date();
- let updated: AgentCronJob | undefined;
- let matchedRlmHeartbeat = false;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id || job.activeSessionId !== activeSessionId || job.source !== "rlm_heartbeat") {
- return job;
- }
- matchedRlmHeartbeat = true;
- if (job.status === "cancelled" || job.status === "completed") {
- return job;
- }
- let nextJob: AgentCronJob = { ...job };
- if (update.label !== undefined) {
- nextJob = { ...nextJob, label: normalizeOptionalLabel(update.label) };
- }
- if (update.deliveryMode !== undefined) {
- nextJob = { ...nextJob, deliveryMode: update.deliveryMode };
- }
- if (update.prompt !== undefined) {
- const prompt = update.prompt.trim();
- if (!prompt) {
- throw new Error("RLM heartbeat instruction cannot be empty");
+ const updated = await this.writeJobs((jobs) => {
+ let updated: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id || job.activeSessionId !== activeSessionId || job.source !== "rlm_heartbeat") {
+ return job;
}
- nextJob = { ...nextJob, prompt };
- }
- if (update.scheduleText !== undefined) {
- const parsed = parseAgentCronSchedule(update.scheduleText, now);
- if (parsed.schedule.kind === "once") {
- throw new Error("RLM heartbeat schedule must be recurring");
+ if (job.status === "cancelled" || job.status === "completed") {
+ return job;
}
- nextJob =
- nextJob.status === "paused"
- ? withoutNextRunAt({ ...nextJob, schedule: parsed.schedule })
- : { ...nextJob, schedule: parsed.schedule, nextRunAt: parsed.nextRunAt.toISOString() };
- }
- if (update.status === "pause") {
- nextJob = withoutNextRunAt({ ...nextJob, status: "paused" });
- } else if (update.status === "resume") {
- const nextRunAt = nextRunAtForSchedule(nextJob.schedule, now);
- if (!nextRunAt) {
- throw new Error("RLM heartbeat schedule must be recurring");
+ let nextJob: AgentCronJob = { ...job };
+ if (update.label !== undefined) {
+ nextJob = { ...nextJob, label: normalizeOptionalLabel(update.label) };
}
- nextJob = { ...nextJob, status: "active", nextRunAt: nextRunAt.toISOString() };
- }
- updated = { ...nextJob, updatedAt: now.toISOString() };
- return updated;
+ if (update.deliveryMode !== undefined) {
+ nextJob = { ...nextJob, deliveryMode: update.deliveryMode };
+ }
+ if (update.prompt !== undefined) {
+ const prompt = update.prompt.trim();
+ if (!prompt) {
+ throw new Error("RLM heartbeat instruction cannot be empty");
+ }
+ nextJob = { ...nextJob, prompt };
+ }
+ if (update.scheduleText !== undefined) {
+ const parsed = parseAgentCronSchedule(update.scheduleText, now);
+ if (parsed.schedule.kind === "once") {
+ throw new Error("RLM heartbeat schedule must be recurring");
+ }
+ nextJob =
+ nextJob.status === "paused"
+ ? withoutNextRunAt({ ...nextJob, schedule: parsed.schedule })
+ : { ...nextJob, schedule: parsed.schedule, nextRunAt: parsed.nextRunAt.toISOString() };
+ }
+ if (update.status === "pause") {
+ nextJob = withoutNextRunAt({ ...nextJob, status: "paused" });
+ } else if (update.status === "resume") {
+ const nextRunAt = nextRunAtForSchedule(nextJob.schedule, now);
+ if (!nextRunAt) {
+ throw new Error("RLM heartbeat schedule must be recurring");
+ }
+ nextJob = { ...nextJob, status: "active", nextRunAt: nextRunAt.toISOString() };
+ }
+ updated = { ...nextJob, updatedAt: now.toISOString() };
+ return updated;
+ });
+ return { jobs: next, result: updated };
});
- if (matchedRlmHeartbeat && updated) {
- this.writeJobs(jobs);
- }
return updated;
}
- deleteRlmHeartbeat(activeSessionId: string, id: string, now = new Date()): AgentCronJob | undefined {
- let deleted: AgentCronJob | undefined;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id || job.activeSessionId !== activeSessionId || job.source !== "rlm_heartbeat") {
- return job;
- }
- deleted = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
- return deleted;
+ async deleteRlmHeartbeat(activeSessionId: string, id: string, now = new Date()): Promise<AgentCronJob | undefined> {
+ const deleted = await this.writeJobs((jobs) => {
+ let deleted: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id || job.activeSessionId !== activeSessionId || job.source !== "rlm_heartbeat") {
+ return job;
+ }
+ deleted = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
+ return deleted;
+ });
+ return { jobs: next, result: deleted };
});
- if (deleted) {
- this.writeJobs(jobs);
- }
return deleted;
}
- cancelRlmHeartbeatsForSession(activeSessionId: string, now = new Date()): AgentCronJob[] {
- const cancelled: AgentCronJob[] = [];
- const jobs = this.readJobs().map((job) => {
- if (
- job.activeSessionId !== activeSessionId ||
- job.source !== "rlm_heartbeat" ||
- (job.status !== "active" && job.status !== "paused")
- ) {
- return job;
- }
- const cancelledJob = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
- cancelled.push(cancelledJob);
- return cancelledJob;
+ async cancelRlmHeartbeatsForSession(activeSessionId: string, now = new Date()): Promise<AgentCronJob[]> {
+ return this.writeJobs((jobs) => {
+ const cancelled: AgentCronJob[] = [];
+ const next = jobs.map((job) => {
+ if (
+ job.activeSessionId !== activeSessionId ||
+ job.source !== "rlm_heartbeat" ||
+ (job.status !== "active" && job.status !== "paused")
+ ) {
+ return job;
+ }
+ const cancelledJob = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
+ cancelled.push(cancelledJob);
+ return cancelledJob;
+ });
+ return { jobs: next, result: cancelled };
});
- if (cancelled.length > 0) {
- this.writeJobs(jobs);
- }
- return cancelled;
}
- cancelJobsForSession(
+ async cancelJobsForSession(
input: { activeSessionId?: string; sessionId?: string; sessionFile?: string },
now = new Date(),
- ): AgentCronJob[] {
+ /** Re-evaluated with the state locked; false leaves the catalog unchanged and returns no result. */
+ stillWanted?: () => boolean,
+ ): Promise<AgentCronJob[]> {
const targetSessionFile = input.sessionFile ? resolve(input.sessionFile) : undefined;
- const cancelled: AgentCronJob[] = [];
- const jobs = this.readJobs().map((job) => {
- const matches =
- (input.activeSessionId !== undefined && job.activeSessionId === input.activeSessionId) ||
- (input.sessionId !== undefined && job.sessionId === input.sessionId) ||
- (targetSessionFile !== undefined && resolve(job.sessionFile) === targetSessionFile);
- if (!matches || (job.status !== "active" && job.status !== "paused")) {
- return job;
+ return this.writeJobs((jobs) => {
+ if (stillWanted && !stillWanted()) {
+ return { jobs, result: [] };
}
- const cancelledJob = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
- cancelled.push(cancelledJob);
- return cancelledJob;
+ const cancelled: AgentCronJob[] = [];
+ const next = jobs.map((job) => {
+ const matches =
+ (input.activeSessionId !== undefined && job.activeSessionId === input.activeSessionId) ||
+ (input.sessionId !== undefined && job.sessionId === input.sessionId) ||
+ (targetSessionFile !== undefined && resolve(job.sessionFile) === targetSessionFile);
+ if (!matches || (job.status !== "active" && job.status !== "paused")) {
+ return job;
+ }
+ const cancelledJob = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
+ cancelled.push(cancelledJob);
+ return cancelledJob;
+ });
+ return { jobs: next, result: cancelled };
});
- if (cancelled.length > 0) {
- this.writeJobs(jobs);
- }
- return cancelled;
}
- pauseHeartbeat(activeSessionId: string, now = new Date()): AgentCronJob | undefined {
- let paused: AgentCronJob | undefined;
- const current = this.getHeartbeat(activeSessionId);
- if (!current) {
- return undefined;
- }
- const jobs = this.readJobs().map((job) => {
- if (job.id !== current.id) {
- return job;
+ async pauseHeartbeat(activeSessionId: string, now = new Date()): Promise<AgentCronJob | undefined> {
+ const paused = await this.writeJobs((jobs) => {
+ const current = latestHeartbeat(jobs, activeSessionId);
+ if (!current) {
+ return { jobs, result: undefined };
}
- paused = { ...job, status: "paused", nextRunAt: undefined, updatedAt: now.toISOString() };
- return paused;
+ const paused = { ...current, status: "paused" as const, nextRunAt: undefined, updatedAt: now.toISOString() };
+ return { jobs: jobs.map((job) => (job.id === current.id ? paused : job)), result: paused };
});
- this.writeJobs(jobs);
return paused;
}
- resumeHeartbeat(activeSessionId: string, now = new Date()): AgentCronJob | undefined {
- let resumed: AgentCronJob | undefined;
- const current = this.getHeartbeat(activeSessionId);
- if (!current) {
- return undefined;
- }
- const nextRunAt = nextRunAtForSchedule(current.schedule, now);
- if (!nextRunAt) {
- throw new Error("Heartbeat schedule must be recurring");
- }
- const jobs = this.readJobs().map((job) => {
- if (job.id !== current.id) {
- return job;
+ async resumeHeartbeat(activeSessionId: string, now = new Date()): Promise<AgentCronJob | undefined> {
+ const resumed = await this.writeJobs((jobs) => {
+ const current = latestHeartbeat(jobs, activeSessionId);
+ if (!current) {
+ return { jobs, result: undefined };
+ }
+ const nextRunAt = nextRunAtForSchedule(current.schedule, now);
+ if (!nextRunAt) {
+ throw new Error("Heartbeat schedule must be recurring");
}
- resumed = { ...job, status: "active", nextRunAt: nextRunAt.toISOString(), updatedAt: now.toISOString() };
- return resumed;
+ const resumed = {
+ ...current,
+ status: "active" as const,
+ nextRunAt: nextRunAt.toISOString(),
+ updatedAt: now.toISOString(),
+ };
+ return { jobs: jobs.map((job) => (job.id === current.id ? resumed : job)), result: resumed };
});
- this.writeJobs(jobs);
return resumed;
}
- clearHeartbeat(activeSessionId: string, now = new Date()): AgentCronJob | undefined {
- let cleared: AgentCronJob | undefined;
- const current = this.getHeartbeat(activeSessionId);
- if (!current) {
- return undefined;
- }
- const jobs = this.readJobs().map((job) => {
- if (job.id !== current.id) {
- return job;
+ async clearHeartbeat(activeSessionId: string, now = new Date()): Promise<AgentCronJob | undefined> {
+ const cleared = await this.writeJobs((jobs) => {
+ const current = latestHeartbeat(jobs, activeSessionId);
+ if (!current) {
+ return { jobs, result: undefined };
}
- cleared = { ...job, status: "cancelled", nextRunAt: undefined, updatedAt: now.toISOString() };
- return cleared;
+ const cleared = {
+ ...current,
+ status: "cancelled" as const,
+ nextRunAt: undefined,
+ updatedAt: now.toISOString(),
+ };
+ return { jobs: jobs.map((job) => (job.id === current.id ? cleared : job)), result: cleared };
});
- this.writeJobs(jobs);
return cleared;
}
- manageHeartbeat(
+ async manageHeartbeat(
activeSessionId: string,
id: string,
action: AgentHeartbeatManagementAction,
now = new Date(),
- ): AgentCronJob | undefined {
- let updated: AgentCronJob | undefined;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id || job.activeSessionId !== activeSessionId || !isHeartbeatCronJob(job)) {
- return job;
- }
- if (job.status === "cancelled" || job.status === "completed") {
- return job;
+ /** Re-evaluated with the state locked; false leaves the catalog unchanged and returns no result. */
+ stillWanted?: () => boolean,
+ ): Promise<AgentCronJob | undefined> {
+ const updated = await this.writeJobs((jobs) => {
+ if (stillWanted && !stillWanted()) {
+ return { jobs, result: undefined };
}
- if (action === "pause") {
- updated = withoutNextRunAt({ ...job, status: "paused", updatedAt: now.toISOString() });
- return updated;
- }
- if (action === "stop") {
- updated = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
+ let updated: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id || job.activeSessionId !== activeSessionId || !isHeartbeatCronJob(job)) {
+ return job;
+ }
+ if (job.status === "cancelled" || job.status === "completed") {
+ return job;
+ }
+ if (action === "pause") {
+ updated = withoutNextRunAt({ ...job, status: "paused", updatedAt: now.toISOString() });
+ return updated;
+ }
+ if (action === "stop") {
+ updated = withoutNextRunAt({ ...job, status: "cancelled", updatedAt: now.toISOString() });
+ return updated;
+ }
+ const nextRunAt = nextRunAtForSchedule(job.schedule, now);
+ if (!nextRunAt) {
+ throw new Error("Heartbeat schedule must be recurring");
+ }
+ updated = {
+ ...job,
+ status: "active",
+ nextRunAt: nextRunAt.toISOString(),
+ updatedAt: now.toISOString(),
+ };
return updated;
- }
- const nextRunAt = nextRunAtForSchedule(job.schedule, now);
- if (!nextRunAt) {
- throw new Error("Heartbeat schedule must be recurring");
- }
- updated = {
- ...job,
- status: "active",
- nextRunAt: nextRunAt.toISOString(),
- updatedAt: now.toISOString(),
- };
- return updated;
+ });
+ return { jobs: next, result: updated };
});
- if (updated) {
- this.writeJobs(jobs);
- }
return updated;
}
- cancel(id: string, now = new Date()): AgentCronJob | undefined {
- let cancelled: AgentCronJob | undefined;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id || job.status === "cancelled") {
- return job;
- }
- cancelled = { ...job, status: "cancelled", nextRunAt: undefined, updatedAt: now.toISOString() };
- return cancelled;
+ async cancel(id: string, now = new Date()): Promise<AgentCronJob | undefined> {
+ const cancelled = await this.writeJobs((jobs) => {
+ let cancelled: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id || job.status === "cancelled") {
+ return job;
+ }
+ cancelled = { ...job, status: "cancelled", nextRunAt: undefined, updatedAt: now.toISOString() };
+ return cancelled;
+ });
+ return { jobs: next, result: cancelled };
});
- if (cancelled) {
- this.writeJobs(jobs);
- }
return cancelled;
}
- recordRunResult(id: string, result: { now?: Date; error?: unknown }): AgentCronJob | undefined {
+ async recordRunResult(id: string, result: { now?: Date; error?: unknown }): Promise<AgentCronJob | undefined> {
const now = result.now ?? new Date();
- let updated: AgentCronJob | undefined;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id) {
- return job;
- }
- if (job.status !== "active") {
- updated = job;
- return job;
- }
- const lastError = result.error === undefined ? undefined : errorMessage(result.error);
- const nextRunAt =
- job.schedule.kind === "cron"
- ? nextRunAtForSchedule(job.schedule, new Date(now.getTime() + 1))
- : job.schedule.kind === "interval"
- ? nextRunAtForSchedule(job.schedule, now)
- : undefined;
- updated = {
- ...job,
- status: job.schedule.kind === "once" ? "completed" : "active",
- nextRunAt: nextRunAt?.toISOString(),
- lastRunAt: now.toISOString(),
- lastError,
- runCount: job.runCount + 1,
- updatedAt: now.toISOString(),
- };
- return updated;
+ const updated = await this.writeJobs((jobs) => {
+ let updated: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id) {
+ return job;
+ }
+ if (job.status !== "active") {
+ updated = job;
+ return job;
+ }
+ const lastError = result.error === undefined ? undefined : errorMessage(result.error);
+ const nextRunAt =
+ job.schedule.kind === "cron"
+ ? nextRunAtForSchedule(job.schedule, new Date(now.getTime() + 1))
+ : job.schedule.kind === "interval"
+ ? nextRunAtForSchedule(job.schedule, now)
+ : undefined;
+ updated = {
+ ...job,
+ status: job.schedule.kind === "once" ? "completed" : "active",
+ nextRunAt: nextRunAt?.toISOString(),
+ lastRunAt: now.toISOString(),
+ lastError,
+ runCount: job.runCount + 1,
+ updatedAt: now.toISOString(),
+ };
+ return updated;
+ });
+ return { jobs: next, result: updated };
});
- if (updated) {
- this.writeJobs(jobs);
- }
return updated;
}
- recordSkipResult(id: string, result: { now?: Date }): AgentCronJob | undefined {
+ async recordSkipResult(id: string, result: { now?: Date }): Promise<AgentCronJob | undefined> {
const now = result.now ?? new Date();
- let updated: AgentCronJob | undefined;
- const jobs = this.readJobs().map((job) => {
- if (job.id !== id) {
- return job;
- }
- if (job.status !== "active") {
- updated = job;
- return job;
- }
- const nextRunAt = nextRunAtForSchedule(job.schedule, now);
- updated = {
- ...job,
- nextRunAt: nextRunAt?.toISOString(),
- lastSkippedAt: now.toISOString(),
- updatedAt: now.toISOString(),
- };
- return updated;
+ const updated = await this.writeJobs((jobs) => {
+ let updated: AgentCronJob | undefined;
+ const next = jobs.map((job) => {
+ if (job.id !== id) {
+ return job;
+ }
+ if (job.status !== "active") {
+ updated = job;
+ return job;
+ }
+ const nextRunAt = nextRunAtForSchedule(job.schedule, now);
+ updated = {
+ ...job,
+ nextRunAt: nextRunAt?.toISOString(),
+ lastSkippedAt: now.toISOString(),
+ updatedAt: now.toISOString(),
+ };
+ return updated;
+ });
+ return { jobs: next, result: updated };
});
- if (updated) {
- this.writeJobs(jobs);
- }
return updated;
}
@@ -706,8 +718,12 @@ export class AgentCronJobStore {
return this.readJobs().filter((job) => isDueJob(job, now));
}
- claimDue(dueAt = new Date(), claimedAt = dueAt): AgentCronDispatch[] {
- return this.mutateStates((state) => claimDueInState(state, dueAt, claimedAt));
+ async claimDue(dueAt = new Date(), claimedAt = dueAt): Promise<AgentCronDispatch[]> {
+ const { dispatches } = await this.mutateStates((state) => ({
+ dispatches: claimDueInState(state, dueAt, claimedAt),
+ results: [] as AgentCronDispatch[],
+ }));
+ return dispatches;
}
getClaimedJob(id: string): AgentCronJob | undefined {
@@ -720,65 +736,34 @@ export class AgentCronJobStore {
return undefined;
}
- recordDispatchResult(
+ async recordDispatchResult(
dispatchId: string,
result: { now?: Date; outcome: AgentCronJobRunResult; error?: unknown },
- ): AgentCronJob | undefined {
- let updated: AgentCronJob | undefined;
- this.mutateStates((state) => {
- const dispatch = state.dispatches.find((candidate) => candidate.id === dispatchId);
- if (!dispatch) {
- return [];
- }
- const now = result.now ?? new Date();
- state.dispatches = state.dispatches.filter((candidate) => candidate.id !== dispatchId);
- state.jobs = state.jobs.map((job) => {
- if (job.id !== dispatch.jobId || job.status !== "active") {
- return job;
- }
- if (result.outcome === "skipped" && result.error === undefined) {
- const nextRunAt = nextRunAtForSchedule(job.schedule, now);
- updated = {
- ...job,
- status: job.schedule.kind === "once" ? "completed" : job.status,
- nextRunAt: nextRunAt?.toISOString(),
- lastSkippedAt: now.toISOString(),
- updatedAt: now.toISOString(),
- };
- return updated;
- }
- updated = {
- ...job,
- status: job.schedule.kind === "once" ? "completed" : job.status,
- lastRunAt: now.toISOString(),
- lastError: result.error === undefined ? undefined : errorMessage(result.error),
- runCount: job.runCount + 1,
- updatedAt: now.toISOString(),
- };
- return updated;
- });
- return [];
- });
- return updated;
+ ): Promise<AgentCronJob | undefined> {
+ const { results } = await this.mutateStates((state) => ({
+ dispatches: [] as AgentCronDispatch[],
+ results: recordDispatchResultInState(state, dispatchId, result),
+ }));
+ return results[0];
}
- recoverInterruptedDispatches(now = new Date()): AgentCronJob[] {
- const recovered: AgentCronJob[] = [];
- this.mutateStates((state) => {
+ async recoverInterruptedDispatches(now = new Date()): Promise<AgentCronJob[]> {
+ const { results } = await this.mutateStates((state) => {
+ const recovered: AgentCronJob[] = [];
recoverInterruptedInState(state, now, recovered);
- return [];
+ return { dispatches: [] as AgentCronDispatch[], results: recovered };
});
- return recovered;
+ return results;
}
- recoverInterruptedDispatchesById(dispatchIds: readonly string[], now = new Date()): AgentCronJob[] {
- const recovered: AgentCronJob[] = [];
+ async recoverInterruptedDispatchesById(dispatchIds: readonly string[], now = new Date()): Promise<AgentCronJob[]> {
const interruptedDispatchIds = new Set(dispatchIds);
- this.mutateStates((state) => {
+ const { results } = await this.mutateStates((state) => {
+ const recovered: AgentCronJob[] = [];
recoverInterruptedInState(state, now, recovered, interruptedDispatchIds);
- return [];
+ return { dispatches: [] as AgentCronDispatch[], results: recovered };
});
- return recovered;
+ return results;
}
getDueJob(id: string, now = new Date()): AgentCronJob | undefined {
@@ -859,92 +844,110 @@ export class AgentCronJobStore {
return [this.readState(this.requireFilePath())];
}
- private mutateStates(mutator: (state: CronJobsState) => AgentCronDispatch[]): AgentCronDispatch[] {
+ /**
+ * Applies the mutator twice (lock-free probe to find changeable paths, then for
+ * real under those paths' locks); the mutator must be pure apart from its return value.
+ */
+ private async mutateStates<T>(
+ mutator: (state: CronJobsState) => CronJobsStateMutation<T>,
+ ): Promise<CronJobsStateMutation<T>> {
const paths = this.sessionArtifactMode ? [...this.sessionArtifactFiles.values()] : [this.requireFilePath()];
const previousHeartbeats = heartbeatCatalogSignature(this.readJobs());
+ const changedPaths: string[] = [];
+ for (const path of paths) {
+ const probe = roundTripJobsState(this.readState(path));
+ const before = JSON.stringify(probe);
+ mutator(probe);
+ if (JSON.stringify(probe) !== before) {
+ changedPaths.push(path);
+ }
+ }
+ // An empty probe locks every path instead: a change landing in the
+ // probe->lock gap is applied under the locks, not missed until restart.
+ const pathsToLock = changedPaths.length > 0 ? changedPaths : paths;
let changed = false;
- const dispatches = withCronJobsStateLocks(paths, () => {
- const dispatches: AgentCronDispatch[] = [];
- for (const path of paths) {
+ const outcome = await withCronJobsStateLocks(pathsToLock, () => {
+ const mutation: CronJobsStateMutation<T> = { dispatches: [], results: [] };
+ for (const path of pathsToLock) {
// Mutators work on a private clone, so a mid-edit throw or a cached read
// never leaks into the published snapshot; the frozen view stays intact.
const state = roundTripJobsState(this.readState(path));
const before = JSON.stringify(state);
- dispatches.push(...mutator(state));
+ const mutated = mutator(state);
+ mutation.dispatches.push(...mutated.dispatches);
+ mutation.results.push(...mutated.results);
if (JSON.stringify(state) !== before) {
this.writeState(path, state);
changed = true;
}
}
- return dispatches;
+ return mutation;
});
+ // Only this mutation's own writes notify; an external racing write must not fire a spurious event.
if (changed && heartbeatCatalogSignature(this.readJobs()) !== previousHeartbeats) {
this.notifyHeartbeatChange();
}
- return dispatches;
+ return outcome;
}
- private writeJobs(jobs: readonly AgentCronJob[]): void {
+ /**
+ * Recomputes the mutation from the state read under the changed paths' locks (all
+ * paths when the probe changes nothing). The mutator must be pure apart from its
+ * return value; only the locked pass's result is returned.
+ */
+ private async writeJobs<T>(
+ mutate: (current: readonly AgentCronJob[]) => { jobs: readonly AgentCronJob[]; result: T },
+ ): Promise<T> {
const previousHeartbeats = heartbeatCatalogSignature(this.readJobs());
+ let result: T;
if (this.sessionArtifactMode) {
- const registeredSessionIds = new Set(this.sessionArtifactFiles.keys());
- const unregistered = jobs.find((job) => !registeredSessionIds.has(job.sessionId));
- if (unregistered) {
- throw new Error(`Cron job ${unregistered.id} targets an unregistered session artifact`);
- }
- const paths = [...this.sessionArtifactFiles.values()];
- withCronJobsStateLocks(paths, () => {
- const currentBySessionId = new Map(
- [...this.sessionArtifactFiles].map(([sessionId, path]) => [sessionId, this.readState(path)]),
- );
- const incomingById = new Map(jobs.map((job) => [job.id, job]));
- const mergedJobsBySessionId = new Map<string, AgentCronJob[]>();
- for (const [sessionId, current] of currentBySessionId) {
- const retained = current.jobs.filter((job) => {
- const incoming = incomingById.get(job.id);
- return incoming === undefined || incoming.sessionId === sessionId;
- });
- mergedJobsBySessionId.set(
- sessionId,
- mergeFreshJobs(
- retained,
- jobs.filter((job) => job.sessionId === sessionId),
- ),
- );
+ const apply = (currentBySessionId: ReadonlyMap<string, CronJobsState>) => {
+ const mutation = mutate([...currentBySessionId.values()].flatMap((state) => state.jobs));
+ const unregistered = mutation.jobs.find((job) => !this.sessionArtifactFiles.has(job.sessionId));
+ if (unregistered) {
+ throw new Error(`Cron job ${unregistered.id} targets an unregistered session artifact`);
}
- const sessionIdByJobId = new Map(
- [...mergedJobsBySessionId].flatMap(([sessionId, sessionJobs]) =>
- sessionJobs.map((job) => [job.id, sessionId] as const),
- ),
+ const next = partitionJobsStates(currentBySessionId, mutation.jobs);
+ const changedSessionIds = [...this.sessionArtifactFiles.keys()].filter(
+ (sessionId) => JSON.stringify(currentBySessionId.get(sessionId)) !== JSON.stringify(next.get(sessionId)),
);
- const dispatches = [...currentBySessionId.values()].flatMap((state) => state.dispatches);
- for (const [sessionId, path] of this.sessionArtifactFiles) {
- const current = currentBySessionId.get(sessionId) ?? { jobs: [], dispatches: [] };
- const nextState = {
- jobs: mergedJobsBySessionId.get(sessionId) ?? [],
- dispatches: dispatches.filter((dispatch) => sessionIdByJobId.get(dispatch.jobId) === sessionId),
- };
- if (JSON.stringify(current) !== JSON.stringify(nextState)) {
- this.writeState(path, nextState);
+ return { mutation, next, changedSessionIds };
+ };
+ const readAll = () =>
+ new Map([...this.sessionArtifactFiles].map(([sessionId, path]) => [sessionId, this.readState(path)]));
+ const probe = apply(readAll());
+ // Empty probe: lock every path, as mutateStates does.
+ const lockedSessionIds =
+ probe.changedSessionIds.length > 0 ? probe.changedSessionIds : [...this.sessionArtifactFiles.keys()];
+ const pathsToLock = lockedSessionIds.map((sessionId) => this.sessionArtifactFiles.get(sessionId)!);
+ const lockedSessionIdSet = new Set(lockedSessionIds);
+ result = await withCronJobsStateLocks(pathsToLock, () => {
+ const { mutation, next, changedSessionIds } = apply(readAll());
+ for (const sessionId of changedSessionIds) {
+ // A concurrent writer must not make an unlocked path changeable: fail closed.
+ if (!lockedSessionIdSet.has(sessionId)) {
+ throw new Error(`Cron jobs state changed concurrently for session ${sessionId}`);
}
+ this.writeState(this.sessionArtifactFiles.get(sessionId)!, next.get(sessionId)!);
}
+ return mutation.result;
});
- if (heartbeatCatalogSignature(this.readJobs()) !== previousHeartbeats) {
- this.notifyHeartbeatChange();
- }
- return;
- }
- const path = this.requireFilePath();
- withCronJobsStateLocks([path], () => {
- const current = this.readState(path);
- this.writeState(path, {
- jobs: mergeFreshJobs(current.jobs, jobs),
- dispatches: current.dispatches,
+ } else {
+ const path = this.requireFilePath();
+ result = await withCronJobsStateLocks([path], () => {
+ const current = this.readState(path);
+ const mutation = mutate(current.jobs);
+ const next: CronJobsState = { jobs: [...mutation.jobs], dispatches: current.dispatches };
+ if (JSON.stringify(next) !== JSON.stringify(current)) {
+ this.writeState(path, next);
+ }
+ return mutation.result;
});
- });
+ }
if (heartbeatCatalogSignature(this.readJobs()) !== previousHeartbeats) {
this.notifyHeartbeatChange();
}
+ return result;
}
private notifyHeartbeatChange(): void {
@@ -1004,6 +1007,8 @@ export class AgentCronScheduler {
private stopped = true;
private hasStarted = false;
private readonly dispatchLanes = new Map<string, Promise<void>>();
+ /** Set once startup recovery landed; a failed recovery is retried by the next claim instead of being skipped. */
+ private recovered = false;
constructor(
private readonly store: AgentCronJobStore,
@@ -1012,11 +1017,9 @@ export class AgentCronScheduler {
start(): void {
this.stopped = false;
- if (!this.hasStarted) {
- this.hasStarted = true;
- this.store.recoverInterruptedDispatches(this.now());
- }
- this.scheduleNext();
+ this.hasStarted = true;
+ // A stale dispatch record would make claimDue skip a due job, so the first claim recovers first.
+ this.scheduleNext(this.recovered ? undefined : 0);
}
stop(): void {
@@ -1042,14 +1045,19 @@ export class AgentCronScheduler {
const dispatches: Array<{ dispatch: AgentCronDispatch; endDispatch?: () => void }> = [];
let claimedDispatches: AgentCronDispatch[] | undefined;
try {
- claimedDispatches = this.store.claimDue(now, this.now());
+ if (this.hasStarted && !this.recovered) {
+ await this.store.recoverInterruptedDispatches(this.now());
+ this.recovered = true;
+ if (this.stopped) return 0;
+ }
+ claimedDispatches = await this.store.claimDue(now, this.now());
for (const dispatch of claimedDispatches) {
dispatches.push({ dispatch, endDispatch: this.hooks.beginDispatch?.(dispatch) });
}
} catch (error) {
for (const claimed of dispatches) claimed.endDispatch?.();
if (claimedDispatches) {
- this.store.recoverInterruptedDispatchesById(
+ await this.store.recoverInterruptedDispatchesById(
claimedDispatches.map((dispatch) => dispatch.id),
this.now(),
);
@@ -1079,7 +1087,7 @@ export class AgentCronScheduler {
try {
const job = this.store.getClaimedJob(dispatch.job.id);
if (!job) {
- this.store.recordDispatchResult(dispatch.id, { now: this.now(), outcome: "skipped" });
+ await this.store.recordDispatchResult(dispatch.id, { now: this.now(), outcome: "skipped" });
return "skipped";
}
let runResult: AgentCronJobRunResult | undefined;
@@ -1090,7 +1098,7 @@ export class AgentCronScheduler {
error = runError;
this.hooks.onError?.(job, runError);
}
- this.store.recordDispatchResult(dispatch.id, {
+ await this.store.recordDispatchResult(dispatch.id, {
now: this.now(),
outcome: runResult === "skipped" && error === undefined ? "skipped" : "ran",
error,
@@ -1562,47 +1570,55 @@ function isDueJob(job: AgentCronJob, now: Date): boolean {
return job.status === "active" && job.nextRunAt !== undefined && Date.parse(job.nextRunAt) <= now.getTime();
}
-function withCronJobsStateLocks<T>(paths: readonly string[], action: () => T): T {
- const releases: Array<() => void> = [];
- try {
- for (const path of [...new Set(paths)].sort()) {
- mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
- let release: (() => void) | undefined;
- let lockCompromised = false;
- for (let attempt = 0; attempt < 100; attempt++) {
- try {
- release = lockSync(path, {
- realpath: false,
- lockfilePath: `${path}.lock`,
- stale: 30_000,
- onCompromised: () => {
- lockCompromised = true;
- },
- });
- if (lockCompromised) {
- release();
- throw new Error(`Cron jobs lock compromised: ${path}`);
- }
- break;
- } catch (error) {
- if (lockCompromised) {
- throw new Error(`Cron jobs lock compromised: ${path}`);
- }
- if ((error as NodeJS.ErrnoException).code !== "ELOCKED" || attempt === 99) {
- throw error;
- }
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
- }
+// A contended cross-process lock retries asynchronously so it never blocks the event loop.
+const CRON_JOBS_LOCK_ATTEMPTS = 100;
+const CRON_JOBS_LOCK_RETRY_MS = 10;
+
+async function acquireCronJobsLock(path: string): Promise<() => Promise<void>> {
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+ for (let attempt = 0; attempt < CRON_JOBS_LOCK_ATTEMPTS; attempt++) {
+ let lockCompromised = false;
+ try {
+ const release = await lock(path, {
+ realpath: false,
+ lockfilePath: `${path}.lock`,
+ stale: 30_000,
+ onCompromised: () => {
+ lockCompromised = true;
+ },
+ });
+ if (lockCompromised) {
+ await release().catch(() => undefined);
+ throw new Error(`Cron jobs lock compromised: ${path}`);
}
- if (!release) {
- throw new Error(`Could not coordinate scheduled jobs: ${path}`);
+ return release;
+ } catch (error) {
+ if (lockCompromised) {
+ throw new Error(`Cron jobs lock compromised: ${path}`);
+ }
+ if ((error as NodeJS.ErrnoException).code !== "ELOCKED" || attempt === CRON_JOBS_LOCK_ATTEMPTS - 1) {
+ throw error;
}
- releases.push(release);
+ await new Promise((resolveDelay) => setTimeout(resolveDelay, CRON_JOBS_LOCK_RETRY_MS));
+ }
+ }
+ throw new Error(`Could not coordinate scheduled jobs: ${path}`);
+}
+
+/**
+ * Serialize a read-modify-write against other processes: acquisition retries
+ * asynchronously, and the critical section runs synchronously once held.
+ */
+async function withCronJobsStateLocks<T>(paths: readonly string[], action: () => T): Promise<T> {
+ const releases: Array<() => Promise<void>> = [];
+ try {
+ for (const path of [...new Set(paths)].sort()) {
+ releases.push(await acquireCronJobsLock(path));
}
return action();
} finally {
for (const release of releases.reverse()) {
- release();
+ await release().catch(() => undefined);
}
}
}
@@ -1676,10 +1692,11 @@ function writeJobsFile(path: string, jobs: readonly AgentCronJob[], mergeCurrent
function writeJobsState(path: string, state: CronJobsState): CronJobsFileIdentity | undefined {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
let written: CronJobsFileIdentity | undefined;
+ // One fsync per write: losing the atomic rename after a power failure rolls
+ // back to the previous valid file, which recovery already tolerates.
writeFileAtomicSync(path, `${JSON.stringify(state, null, 2)}\n`, {
mode: 0o600,
fsync: true,
- fsyncDir: true,
beforeRename: (tempPath) => {
written = statJobsFileIdentity(tempPath);
},
@@ -1687,6 +1704,48 @@ function writeJobsState(path: string, state: CronJobsState): CronJobsFileIdentit
return written;
}
+function recordDispatchResultInState(
+ state: CronJobsState,
+ dispatchId: string,
+ result: { now?: Date; outcome: AgentCronJobRunResult; error?: unknown },
+): AgentCronJob[] {
+ const dispatch = state.dispatches.find((candidate) => candidate.id === dispatchId);
+ if (!dispatch) {
+ return [];
+ }
+ const now = result.now ?? new Date();
+ state.dispatches = state.dispatches.filter((candidate) => candidate.id !== dispatchId);
+ const updated: AgentCronJob[] = [];
+ state.jobs = state.jobs.map((job) => {
+ if (job.id !== dispatch.jobId || job.status !== "active") {
+ return job;
+ }
+ if (result.outcome === "skipped" && result.error === undefined) {
+ const nextRunAt = nextRunAtForSchedule(job.schedule, now);
+ const skipped = {
+ ...job,
+ status: job.schedule.kind === "once" ? ("completed" as const) : job.status,
+ nextRunAt: nextRunAt?.toISOString(),
+ lastSkippedAt: now.toISOString(),
+ updatedAt: now.toISOString(),
+ };
+ updated.push(skipped);
+ return skipped;
+ }
+ const ran = {
+ ...job,
+ status: job.schedule.kind === "once" ? ("completed" as const) : job.status,
+ lastRunAt: now.toISOString(),
+ lastError: result.error === undefined ? undefined : errorMessage(result.error),
+ runCount: job.runCount + 1,
+ updatedAt: now.toISOString(),
+ };
+ updated.push(ran);
+ return ran;
+ });
+ return updated;
+}
+
function claimDueInState(state: CronJobsState, dueAt: Date, claimedAt: Date): AgentCronDispatch[] {
const dispatches: AgentCronDispatch[] = [];
const claimedJobIds = new Set(state.dispatches.map((dispatch) => dispatch.jobId));
@@ -1744,6 +1803,23 @@ function recoverInterruptedInState(
});
}
+/** Partitions the next catalog by session id; each dispatch record follows its job's artifact. */
+function partitionJobsStates(
+ currentBySessionId: ReadonlyMap<string, CronJobsState>,
+ jobs: readonly AgentCronJob[],
+): Map<string, CronJobsState> {
+ const sessionIdByJobId = new Map(jobs.map((job) => [job.id, job.sessionId] as const));
+ const dispatches = [...currentBySessionId.values()].flatMap((state) => state.dispatches);
+ const next = new Map<string, CronJobsState>();
+ for (const sessionId of currentBySessionId.keys()) {
+ next.set(sessionId, {
+ jobs: jobs.filter((job) => job.sessionId === sessionId),
+ dispatches: dispatches.filter((dispatch) => sessionIdByJobId.get(dispatch.jobId) === sessionId),
+ });
+ }
+ return next;
+}
+
function mergeFreshJobs(currentJobs: readonly AgentCronJob[], nextJobs: readonly AgentCronJob[]): AgentCronJob[] {
const merged = new Map<string, AgentCronJob>();
for (const job of currentJobs) {
diff --git a/packages/coding-agent/src/core/session-file-actions.ts b/packages/coding-agent/src/core/session-file-actions.ts
index 7818c4fd21..900b2b58b4 100644
--- a/packages/coding-agent/src/core/session-file-actions.ts
+++ b/packages/coding-agent/src/core/session-file-actions.ts
@@ -7,7 +7,7 @@ import { getSessionArtifactPathForFile } from "./session-manager.js";
export type DeleteSessionFileResult = { ok: true; method: "trash" | "unlink" } | { ok: false; error: string };
export interface DeleteSessionFileOptions {
- afterFileRemoved?: () => void;
+ afterFileRemoved?: () => void | Promise<void>;
}
/**
@@ -67,7 +67,7 @@ export async function deleteSessionFile(
): Promise<DeleteSessionFileResult> {
const result = await removeSessionFile(sessionPath);
if (result.ok) {
- options.afterFileRemoved?.();
+ await options.afterFileRemoved?.();
await deleteSessionArtifacts(sessionPath);
}
return result;
diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts
index 61a683f421..74962631b7 100644
--- a/packages/coding-agent/src/main.ts
+++ b/packages/coding-agent/src/main.ts
@@ -1050,6 +1050,7 @@ async function createDaemonClientConnection(options: {
clientOwned?: boolean;
noSession?: boolean;
supportsExtensionUi?: boolean;
+ tracksHeartbeats?: boolean;
}): Promise<{ connection: DaemonAgentConnection; summary: SessionSummary }> {
// Caller must have awaited ensureInteractiveDaemonRunning for this socket.
const client = new DaemonClient(options.socketPath);
@@ -1064,6 +1065,7 @@ async function createDaemonClientConnection(options: {
ownedSession: options.clientOwned,
ownedSessionRecoveryConfig: options.clientOwned ? options.config : undefined,
supportsExtensionUi: options.supportsExtensionUi,
+ tracksHeartbeats: options.tracksHeartbeats,
recoverDaemon: () => ensureInteractiveDaemonRunning(options.socketPath),
telemetryDisabled: options.config.telemetryDisabled,
});
@@ -1603,6 +1605,8 @@ export async function main(args: string[], options?: MainOptions) {
clientOwned: isClientOwnedDaemonSession(appMode, parsed.noSession),
noSession: parsed.noSession,
supportsExtensionUi: appMode === "rpc",
+ // ACP never issues a scheduled-job command, so attach opts it into heartbeats_changed pushes.
+ tracksHeartbeats: appMode === "acp",
}));
} catch (error) {
if (error instanceof SessionAlreadyActiveError || error instanceof DaemonSessionCreateError) {
diff --git a/packages/coding-agent/src/modes/agent-connection/daemon-agent-connection.ts b/packages/coding-agent/src/modes/agent-connection/daemon-agent-connection.ts
index a65ac61870..cb12325f9c 100644
--- a/packages/coding-agent/src/modes/agent-connection/daemon-agent-connection.ts
+++ b/packages/coding-agent/src/modes/agent-connection/daemon-agent-connection.ts
@@ -214,6 +214,8 @@ export interface DaemonAgentConnectionOptions {
sendClientEnv?: boolean;
/** Advertise support for interactive extension dialogs. */
supportsExtensionUi?: boolean;
+ /** Attaching opts the client into heartbeats_changed pushes without a scheduled-job command (ACP). */
+ tracksHeartbeats?: boolean;
/** Dispose the connection by stopping its hidden worker instead of detaching. */
ownedSession?: boolean;
/** Fresh runtime context used only if the owned worker must be relaunched. */
@@ -457,6 +459,7 @@ export class DaemonAgentConnection implements AgentConnection {
"slim_attach",
"chunked_snapshot",
...(this.options.ownedSession ? (["client_owned_sessions"] as const) : []),
+ ...(this.options.tracksHeartbeats ? (["heartbeat_catalog"] as const) : []),
],
env: this.options.sendClientEnv ? collectDaemonClientEnv() : undefined,
launchEnv: this.options.ownedSession ? collectDaemonLaunchEnv() : undefined,
@@ -1613,6 +1616,7 @@ export class DaemonAgentConnection implements AgentConnection {
"slim_attach",
"chunked_snapshot",
...(this.options.ownedSession ? (["client_owned_sessions"] as const) : []),
+ ...(this.options.tracksHeartbeats ? (["heartbeat_catalog"] as const) : []),
],
env: this.options.sendClientEnv ? collectDaemonClientEnv() : undefined,
launchEnv: this.options.ownedSession ? collectDaemonLaunchEnv() : undefined,
@@ -1759,6 +1763,7 @@ export class DaemonAgentConnection implements AgentConnection {
connection = await DaemonAgentConnection.attach(watchClient, activeSessionId, {
closeClientOnDispose: false,
directTransport: false,
+ tracksHeartbeats: this.options.tracksHeartbeats,
});
} catch {
return undefined;
diff --git a/packages/coding-agent/src/modes/daemon/active-session-state.ts b/packages/coding-agent/src/modes/daemon/active-session-state.ts
index a1403ce272..498873daf2 100644
--- a/packages/coding-agent/src/modes/daemon/active-session-state.ts
+++ b/packages/coding-agent/src/modes/daemon/active-session-state.ts
@@ -24,6 +24,8 @@ export interface DaemonSocketClient {
catchupRetryTimer?: NodeJS.Timeout;
backpressured?: boolean;
rosterSubscribed?: boolean;
+ /** Client has used a scheduled-job command and receives heartbeats_changed pushes. */
+ tracksHeartbeats?: boolean;
/** A push hit backpressure; one full-roster resync goes out on drain. */
rosterResyncPending?: boolean;
authenticated?: boolean;
diff --git a/packages/coding-agent/src/modes/daemon/daemon-extension-binding.ts b/packages/coding-agent/src/modes/daemon/daemon-extension-binding.ts
index cecd046186..1e10b0609e 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-extension-binding.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-extension-binding.ts
@@ -21,7 +21,8 @@ import {
export interface ActiveSessionBindingCallbacks {
broadcast: (state: ActiveSessionState, message: DaemonOutbound) => void;
createConnectionState?: (state: ActiveSessionState) => AgentConnectionState;
- sessionReplaced?: (state: ActiveSessionState) => void;
+ /** May be async (the daemon's rebind awaits cron-store locks); awaited before session_replaced. */
+ sessionReplaced?: (state: ActiveSessionState) => void | Promise<void>;
shutdown: () => void;
subagentRuntimeHost?: SubagentRuntimeHost;
}
@@ -69,7 +70,9 @@ export async function bindActiveSessionState(
state.runtime.setRebindSession(async () => {
await bindActiveSessionState(state, callbacks);
- callbacks.sessionReplaced?.(state);
+ // A floating promise here would let clients observe pre-rebind job state
+ // and turn a cron-store failure into an unhandled rejection.
+ await callbacks.sessionReplaced?.(state);
callbacks.broadcast(state, {
type: "session_replaced",
activeSessionId: state.activeSessionId,
diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
index 5f559585a0..b5cf89c52b 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
@@ -1788,8 +1788,8 @@ export class AgentDaemon {
this.bindingCompletions.delete(state.activeSessionId);
completeBinding();
}
- this.registerCronStoreForState(state);
- this.rebindCronJobsToState(state);
+ await this.registerCronStoreForState(state);
+ await this.rebindCronJobsToState(state);
if (runtime.metadata.kind !== "subagent") {
// Mark the session as daemon-resident so a restarted daemon can
// restore it. Closes for kill/completed/replaced flip this back to
@@ -1806,7 +1806,7 @@ export class AgentDaemon {
return state;
}
- private refreshReplacedSessionState(state: ActiveSessionState): void {
+ private async refreshReplacedSessionState(state: ActiveSessionState): Promise<void> {
this.acpMcpOwners?.delete(state.activeSessionId);
for (const client of state.clients) {
this.abortSideQuestionsFor(client, state.activeSessionId);
@@ -1819,11 +1819,11 @@ export class AgentDaemon {
const summaryState = state.summaryState as ActiveSessionState["summaryState"];
state.runtime.session.setCurrentRecap(summaryState?.summary);
}
- this.registerCronStoreForState(state);
- this.rebindCronJobsToState(state);
+ await this.registerCronStoreForState(state);
+ await this.rebindCronJobsToState(state);
}
- private registerCronStoreForState(state: ActiveSessionState): void {
+ private async registerCronStoreForState(state: ActiveSessionState): Promise<void> {
if (!this.options.worker) {
return;
}
@@ -1833,7 +1833,7 @@ export class AgentDaemon {
return;
}
if (this.cronStore.registerSessionArtifact(session.sessionId, artifactDir)) {
- this.cronStore.recoverSessionArtifact(session.sessionId);
+ await this.cronStore.recoverSessionArtifact(session.sessionId);
this.cronScheduler.wake();
}
// A fresh worker only knows resident sessions' jobs; passive descendants' schedules must fire without hydration.
@@ -1851,7 +1851,7 @@ export class AgentDaemon {
const artifactDir = getSessionArtifactPathForFile(resolve(passive.entry.sessionFile), passive.info.id);
if (this.cronStore.registerSessionArtifact(passive.info.id, artifactDir)) {
registered = true;
- this.cronStore.recoverSessionArtifact(passive.info.id);
+ await this.cronStore.recoverSessionArtifact(passive.info.id);
}
} catch (error) {
this.log(
@@ -1911,7 +1911,7 @@ export class AgentDaemon {
await this.setStateSessionName(state, command.name);
}
this.adoptClientEnv(state, clientEnv);
- this.rebindCronJobsToState(state);
+ await this.rebindCronJobsToState(state);
return state;
}
@@ -1989,7 +1989,7 @@ export class AgentDaemon {
await this.setStateSessionName(existing, command.name);
}
this.adoptClientEnv(existing, clientEnv);
- this.rebindCronJobsToState(existing);
+ await this.rebindCronJobsToState(existing);
return existing;
}
@@ -2028,7 +2028,7 @@ export class AgentDaemon {
await this.setStateSessionName(existing, command.name);
}
this.adoptClientEnv(existing, clientEnv);
- this.rebindCronJobsToState(existing);
+ await this.rebindCronJobsToState(existing);
return existing;
}
let stateRef: ActiveSessionState | undefined;
@@ -2192,13 +2192,17 @@ export class AgentDaemon {
return this.cronStore.getClaimedJob(jobId) ?? this.cronStore.getDueJob(jobId);
}
- private createCronJobForState(state: ActiveSessionState, schedule: string, prompt: string): AgentCronJob {
+ private async createCronJobForState(
+ state: ActiveSessionState,
+ schedule: string,
+ prompt: string,
+ ): Promise<AgentCronJob> {
const session = state.runtime.session;
const sessionFile = session.sessionFile;
if (!sessionFile) {
throw new Error("Cron jobs require a persisted session file");
}
- const job = this.cronStore.create({
+ const job = await this.cronStore.create({
activeSessionId: state.activeSessionId,
sessionId: session.sessionId,
sessionFile,
@@ -2211,19 +2215,19 @@ export class AgentDaemon {
return job;
}
- private createHeartbeatForState(
+ private async createHeartbeatForState(
state: ActiveSessionState,
schedule: string,
instruction: string,
deliveryMode?: AgentHeartbeatDeliveryMode,
- ): AgentCronJob {
+ ): Promise<AgentCronJob> {
const session = state.runtime.session;
const sessionFile = session.sessionFile;
if (!sessionFile) {
throw new Error("Heartbeats require a persisted session file");
}
const previousHeartbeat = this.cronStore.getHeartbeat(state.activeSessionId);
- const job = this.cronStore.createHeartbeat({
+ const job = await this.cronStore.createHeartbeat({
activeSessionId: state.activeSessionId,
sessionId: session.sessionId,
sessionFile,
@@ -2234,30 +2238,30 @@ export class AgentDaemon {
deliveryMode: deliveryMode ?? previousHeartbeat?.deliveryMode,
});
if (previousHeartbeat) {
- this.removeQueuedHeartbeatFollowUp(state, previousHeartbeat);
+ await this.removeQueuedHeartbeatFollowUp(state, previousHeartbeat);
}
this.cronScheduler.wake();
return job;
}
- private updateHeartbeatForState(
+ private async updateHeartbeatForState(
state: ActiveSessionState,
action: AgentHeartbeatUpdateAction,
- ): AgentCronJob | undefined {
+ ): Promise<AgentCronJob | undefined> {
const job =
action === "pause"
- ? this.cronStore.pauseHeartbeat(state.activeSessionId)
+ ? await this.cronStore.pauseHeartbeat(state.activeSessionId)
: action === "resume"
- ? this.cronStore.resumeHeartbeat(state.activeSessionId)
- : this.cronStore.clearHeartbeat(state.activeSessionId);
+ ? await this.cronStore.resumeHeartbeat(state.activeSessionId)
+ : await this.cronStore.clearHeartbeat(state.activeSessionId);
if (job && action !== "resume") {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
this.cronScheduler.wake();
return job;
}
- private createRlmHeartbeatForState(
+ private async createRlmHeartbeatForState(
state: ActiveSessionState,
input: {
instruction: string;
@@ -2265,13 +2269,13 @@ export class AgentDaemon {
label?: string;
deliveryMode?: AgentHeartbeatDeliveryMode;
},
- ): AgentCronJob {
+ ): Promise<AgentCronJob> {
const session = state.runtime.session;
const sessionFile = session.sessionFile;
if (!sessionFile) {
throw new Error("RLM heartbeats require a persisted session file");
}
- const job = this.cronStore.createRlmHeartbeat({
+ const job = await this.cronStore.createRlmHeartbeat({
activeSessionId: state.activeSessionId,
sessionId: session.sessionId,
sessionFile,
@@ -2286,7 +2290,7 @@ export class AgentDaemon {
return job;
}
- private updateRlmHeartbeatForState(
+ private async updateRlmHeartbeatForState(
state: ActiveSessionState,
input: {
id: string;
@@ -2296,8 +2300,8 @@ export class AgentDaemon {
status?: "pause" | "resume";
deliveryMode?: AgentHeartbeatDeliveryMode;
},
- ): AgentCronJob | undefined {
- const job = this.cronStore.updateRlmHeartbeat(state.activeSessionId, input.id, {
+ ): Promise<AgentCronJob | undefined> {
+ const job = await this.cronStore.updateRlmHeartbeat(state.activeSessionId, input.id, {
label: input.label,
prompt: input.instruction,
scheduleText: input.interval ? normalizeHeartbeatSchedule(input.interval) : undefined,
@@ -2311,17 +2315,17 @@ export class AgentDaemon {
input.status === "pause" ||
input.deliveryMode !== undefined
) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
this.cronScheduler.wake();
}
return job;
}
- private deleteRlmHeartbeatForState(state: ActiveSessionState, id: string): AgentCronJob | undefined {
- const job = this.cronStore.deleteRlmHeartbeat(state.activeSessionId, id);
+ private async deleteRlmHeartbeatForState(state: ActiveSessionState, id: string): Promise<AgentCronJob | undefined> {
+ const job = await this.cronStore.deleteRlmHeartbeat(state.activeSessionId, id);
if (job) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
this.cronScheduler.wake();
}
return job;
@@ -2344,15 +2348,15 @@ export class AgentDaemon {
});
}
- private manageHeartbeat(
+ private async manageHeartbeat(
activeSessionId: string,
jobId: string,
action: AgentHeartbeatManagementAction,
- ): AgentCronJob | undefined {
- const job = this.cronStore.manageHeartbeat(activeSessionId, jobId, action);
+ ): Promise<AgentCronJob | undefined> {
+ const job = await this.cronStore.manageHeartbeat(activeSessionId, jobId, action);
const state = this.sessions.get(activeSessionId);
if (job && action !== "resume" && state) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
if (job) {
this.cronScheduler.wake();
@@ -2360,12 +2364,12 @@ export class AgentDaemon {
return job;
}
- private rebindCronJobsToState(state: ActiveSessionState): void {
+ private async rebindCronJobsToState(state: ActiveSessionState): Promise<void> {
const sessionFile = state.runtime.session.sessionFile;
if (!sessionFile) {
return;
}
- const reboundJobs = this.cronStore.rebindSessionJobs({
+ const reboundJobs = await this.cronStore.rebindSessionJobs({
activeSessionId: state.activeSessionId,
sessionId: state.runtime.session.sessionId,
sessionFile,
@@ -2376,20 +2380,20 @@ export class AgentDaemon {
}
}
- private cancelSubagentRlmHeartbeats(state: ActiveSessionState): void {
+ private async cancelSubagentRlmHeartbeats(state: ActiveSessionState): Promise<void> {
if (state.runtime.metadata.kind !== "subagent") {
return;
}
- const cancelled = this.cronStore.cancelRlmHeartbeatsForSession(state.activeSessionId);
+ const cancelled = await this.cronStore.cancelRlmHeartbeatsForSession(state.activeSessionId);
for (const job of cancelled) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
if (cancelled.length > 0) {
this.cronScheduler.wake();
}
}
- private cancelScheduledJobsForSession(state: ActiveSessionState): void {
+ private async cancelScheduledJobsForSession(state: ActiveSessionState): Promise<void> {
const session = state.runtime.session;
const target: {
activeSessionId: string;
@@ -2404,17 +2408,17 @@ export class AgentDaemon {
if (session?.sessionFile) {
target.sessionFile = session.sessionFile;
}
- const cancelled = this.cronStore.cancelJobsForSession(target);
+ const cancelled = await this.cronStore.cancelJobsForSession(target);
for (const job of cancelled) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
if (cancelled.length > 0) {
this.cronScheduler.wake();
}
}
- private cancelScheduledJobsForSessionFile(sessionFile: string): void {
- const cancelled = this.cronStore.cancelJobsForSession({ sessionFile });
+ private async cancelScheduledJobsForSessionFile(sessionFile: string): Promise<void> {
+ const cancelled = await this.cronStore.cancelJobsForSession({ sessionFile });
if (cancelled.length > 0) {
this.cronScheduler.wake();
}
@@ -2427,7 +2431,7 @@ export class AgentDaemon {
return deleteSessionFile(sessionPath, options);
}
- private removeQueuedHeartbeatFollowUp(state: ActiveSessionState, job: AgentCronJob): void {
+ private async removeQueuedHeartbeatFollowUp(state: ActiveSessionState, job: AgentCronJob): Promise<void> {
if (!isHeartbeatCronJob(job)) {
return;
}
@@ -2455,7 +2459,7 @@ export class AgentDaemon {
// A half-bound match falls through to createRuntime, which awaits the
// pending create for the same session file instead of prompting mid-bind.
if (current && !this.bindingSessions.has(current.activeSessionId) && !requiresRlmSubagentRestore) {
- this.rebindCronJobsToState(current);
+ await this.rebindCronJobsToState(current);
const reboundJob = requirePersistedJob ? this.getRunnableCronJob(job.id) : dueJob;
return reboundJob && this.isCronJobRunnableForState(reboundJob, current, requirePersistedJob)
? current
@@ -2496,7 +2500,7 @@ export class AgentDaemon {
!parentInfo ||
parentInfo.state?.status !== "active"
) {
- this.cancelRlmHeartbeat(job.id);
+ await this.cancelRlmHeartbeat(job.id);
return undefined;
}
@@ -2520,10 +2524,10 @@ export class AgentDaemon {
throw new RuntimeOpenCancelledError();
}
if (!childState || childState.runtime.metadata.kind !== "subagent") {
- this.cancelRlmHeartbeat(job.id);
+ await this.cancelRlmHeartbeat(job.id);
return undefined;
}
- this.rebindCronJobsToState(childState);
+ await this.rebindCronJobsToState(childState);
const reboundJob = this.getRunnableCronJob(job.id);
return reboundJob && this.isCronJobRunnableForState(reboundJob, childState, true) ? childState : undefined;
} catch (error) {
@@ -2534,8 +2538,8 @@ export class AgentDaemon {
}
}
- private cancelRlmHeartbeat(jobId: string): void {
- if (this.cronStore.cancel(jobId)) {
+ private async cancelRlmHeartbeat(jobId: string): Promise<void> {
+ if (await this.cronStore.cancel(jobId)) {
this.cronScheduler.wake();
}
}
@@ -2556,7 +2560,7 @@ export class AgentDaemon {
continue;
}
if (!sessionInfo || sessionInfo.id !== current.sessionId || sessionInfo.state?.status !== "active") {
- this.cancelScheduledJobsForSessionFile(current.sessionFile);
+ await this.cancelScheduledJobsForSessionFile(current.sessionFile);
return false;
}
return true;
@@ -2795,7 +2799,7 @@ export class AgentDaemon {
// would mask the teardown error and skip the sweep.
if (childSessionFile) {
try {
- this.cancelScheduledJobsForSessionFile(childSessionFile);
+ await this.cancelScheduledJobsForSessionFile(childSessionFile);
} catch (error) {
this.log(
`failed to cancel scheduled jobs for deleted RLM subagent ${childId}: ${error instanceof Error ? error.message : String(error)}`,
@@ -4541,8 +4545,8 @@ export class AgentDaemon {
composedEntry?.summary,
);
const result = await this.deleteSavedSessionFile(command.sessionPath, {
- afterFileRemoved: () => {
- this.cancelScheduledJobsForSessionFile(command.sessionPath);
+ afterFileRemoved: async () => {
+ await this.cancelScheduledJobsForSessionFile(command.sessionPath);
},
});
if (result.ok && this.options.worker) {
@@ -5176,7 +5180,7 @@ export class AgentDaemon {
});
case "heartbeat_manage": {
- const heartbeat = this.manageHeartbeat(command.activeSessionId, command.jobId, command.action);
+ const heartbeat = await this.manageHeartbeat(command.activeSessionId, command.jobId, command.action);
if (!heartbeat) {
throw new Error(`No active heartbeat found: ${command.jobId}`);
}
@@ -5185,19 +5189,19 @@ export class AgentDaemon {
case "cron_add": {
const state = this.getSessionState(command.activeSessionId);
- const job = this.createCronJobForState(state, command.schedule, command.prompt);
+ const job = await this.createCronJobForState(state, command.schedule, command.prompt);
this.scheduleRosterFlush();
return success(command.id, "cron_add", { job });
}
case "cron_cancel": {
- const job = this.cronStore.cancel(command.jobId);
+ const job = await this.cronStore.cancel(command.jobId);
if (!job) {
throw new Error(`No cron job found: ${command.jobId}`);
}
const state = this.sessions.get(job.activeSessionId);
if (state) {
- this.removeQueuedHeartbeatFollowUp(state, job);
+ await this.removeQueuedHeartbeatFollowUp(state, job);
}
this.cronScheduler.wake();
this.scheduleRosterFlush();
@@ -5215,13 +5219,13 @@ export class AgentDaemon {
case "heartbeat_set": {
const state = this.getSessionState(command.activeSessionId);
const deliveryMode = normalizeHeartbeatDeliveryMode(command.deliveryMode);
- const heartbeat = this.createHeartbeatForState(state, command.schedule, command.prompt, deliveryMode);
+ const heartbeat = await this.createHeartbeatForState(state, command.schedule, command.prompt, deliveryMode);
return success(command.id, "heartbeat_set", { heartbeat });
}
case "heartbeat_update": {
const state = this.getSessionState(command.activeSessionId);
- const heartbeat = this.updateHeartbeatForState(state, command.action);
+ const heartbeat = await this.updateHeartbeatForState(state, command.action);
return success(command.id, "heartbeat_update", {
heartbeat: heartbeat ?? null,
});
@@ -5361,7 +5365,7 @@ export class AgentDaemon {
const state = this.getSessionState(command.activeSessionId);
const options = command.parentSession ? { parentSession: command.parentSession } : undefined;
const result = await state.runtime.newSession(options);
- this.rebindCronJobsToState(state);
+ await this.rebindCronJobsToState(state);
return success(command.id, "new_session", result);
}
@@ -5370,7 +5374,7 @@ export class AgentDaemon {
const result = await state.runtime.switchSession(command.sessionPath, {
cwdOverride: command.cwdOverride,
});
- this.rebindCronJobsToState(state);
+ await this.rebindCronJobsToState(state);
return success(command.id, "switch_session", result);
}
@@ -5379,7 +5383,7 @@ export class AgentDaemon {
const result = await state.runtime.fork(command.entryId, {
position: command.position,
});
- this.rebindCronJobsToState(state);
+ await this.rebindCronJobsToState(state);
return success(command.id, "fork", result);
}
@@ -6880,10 +6884,10 @@ export class AgentDaemon {
closeError = error;
closeFailed = true;
}
- const reasonUpgrade = (existingClose.reasonUpgrade ?? Promise.resolve()).then(() => {
+ const reasonUpgrade = (existingClose.reasonUpgrade ?? Promise.resolve()).then(async () => {
if (!this.isStrongerCloseReason(requestedReason, existingClose.reason)) return;
try {
- this.applyReasonUpgrade(state, existingClose.descendants, existingClose.reason, requestedReason);
+ await this.applyReasonUpgrade(state, existingClose.descendants, existingClose.reason, requestedReason);
} finally {
existingClose.reason = requestedReason;
}
@@ -6925,17 +6929,17 @@ export class AgentDaemon {
return this.closeReasonStrength(candidate) > this.closeReasonStrength(current);
}
- private applyReasonUpgrade(
+ private async applyReasonUpgrade(
state: ActiveSessionState,
descendants: ReadonlySet<ActiveSessionState>,
from: DaemonSessionClosedReason,
to: DaemonSessionClosedReason,
- ): void {
+ ): Promise<void> {
let persistError: unknown;
let persistenceFailed = false;
for (const target of [state, ...descendants]) {
try {
- if (to === "killed") this.cancelScheduledJobsForSession(target);
+ if (to === "killed") await this.cancelScheduledJobsForSession(target);
} catch (error) {
if (!persistenceFailed) persistError = error;
persistenceFailed = true;
@@ -6979,9 +6983,9 @@ export class AgentDaemon {
return;
}
if (reason === "killed") {
- this.cancelScheduledJobsForSession(state);
+ await this.cancelScheduledJobsForSession(state);
} else if (reason !== "shutdown" && reason !== "update") {
- this.cancelSubagentRlmHeartbeats(state);
+ await this.cancelSubagentRlmHeartbeats(state);
}
// Abort in-flight status work before any await/dispose so it can't write
// agent_status to a session being torn down.
diff --git a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
index fa246eed90..72c44f0c43 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
@@ -96,7 +96,9 @@ export type DaemonClientCapability =
| "extension_ui"
| "slim_attach"
| "chunked_snapshot"
- | "client_owned_sessions";
+ | "client_owned_sessions"
+ // Client declaration, not a command gate: attach with it opts into heartbeats_changed pushes.
+ | "heartbeat_catalog";
export type DaemonPromptAdmissionCancellationStatus = "cancelled" | "owned" | "unknown";
export interface DaemonPromptAdmissionCancellationResult {
status: DaemonPromptAdmissionCancellationStatus;
@@ -152,12 +154,12 @@ export const DAEMON_SUPPORTED_CLIENT_CAPABILITIES: readonly DaemonClientCapabili
"slim_attach",
"chunked_snapshot",
"client_owned_sessions",
+ "heartbeat_catalog",
];
export const DAEMON_DEFAULT_SERVER_CAPABILITIES: readonly DaemonServerCapability[] = [
...DAEMON_SUPPORTED_CLIENT_CAPABILITIES,
"delete_rlm_subagent",
- "heartbeat_catalog",
"heartbeat_management",
"model_catalog",
"side_question_transcript",
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
index c1bb40fd21..f291402560 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
@@ -213,6 +213,11 @@ export const HEARTBEAT_LIST_FORWARD_TIMEOUT_MS = 25_000;
// (stale-while-revalidate) so external artifact writes converge without ever
// blocking a catalog response behind a fresh saved-session disk scan.
const PASSIVE_SCHEDULED_JOBS_REFRESH_MS = 5_000;
+// Backstop only: heartbeats_changed frames are the primary freshness signal, so
+// a missed one cannot pin stale data forever.
+export const WORKER_HEARTBEAT_SNAPSHOT_MAX_AGE_MS = 60_000;
+// Mutation and frame bursts collapse into one trailing heartbeats_changed write per client.
+export const HEARTBEATS_CHANGED_COALESCE_MS = 100;
const INPUT_PAUSE_CLEANUP_TIMEOUT_MS = 5_000;
const UPDATE_RESTART_MUTATION_DRAIN_TIMEOUT_MS = 80_000;
const UPDATE_RESTART_WORKER_REQUEST_TIMEOUT_MS = 90_000;
@@ -361,6 +366,17 @@ const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set([
"restart",
"shutdown",
]);
+// Commands proving the client tracks the catalog; only such clients receive heartbeats_changed pushes.
+const HEARTBEAT_TRACKING_COMMANDS: ReadonlySet<string> = new Set([
+ "cron_list",
+ "cron_add",
+ "cron_cancel",
+ "heartbeats_list",
+ "heartbeat_get",
+ "heartbeat_set",
+ "heartbeat_update",
+ "heartbeat_manage",
+]);
interface ResidentWorker {
descriptor: DaemonWorkerDescriptor;
@@ -368,6 +384,9 @@ interface ResidentWorker {
client?: DaemonWorkerClient;
heartbeatSnapshot?: AgentConnectionHeartbeat[];
heartbeatSnapshotStale?: boolean;
+ heartbeatSnapshotAt?: number;
+ heartbeatSnapshotRefresh?: Promise<void>;
+ heartbeatSnapshotRefreshQueued?: boolean;
summaries: Map<string, SessionSummary>;
snapshotCache: Map<string, DaemonAttachResult>;
transcriptCaches: Map<string, SnapshotTranscriptCache>;
@@ -777,12 +796,18 @@ export class DaemonSupervisor {
private idleEvictionTimer?: ReturnType<typeof setTimeout>;
private idleEvictionSweep?: Promise<void>;
private idleEvictionFence?: Promise<void>;
+ private heartbeatsChangedBroadcastTimer?: ReturnType<typeof setTimeout>;
private scheduledWakeTimer?: ReturnType<typeof setTimeout>;
private scheduledWakeRecompute?: Promise<void>;
private scheduledWakeRecomputeQueued = false;
private readonly scheduledWakeFailures = new Map<string, number>();
/** Shared passive scheduled-jobs snapshot; undefined before the first scan and after each invalidation. */
private passiveScheduledJobs?: { rows: PassiveScheduledJob[]; scannedAt: number };
+ /**
+ * True while an ephemeral scheduled-job cancel never settled: those retries
+ * only run inside scans, so wake recomputes enumerate fresh until it clears.
+ */
+ private ephemeralCancelIntentsPending = false;
/** One in-flight scan shared by every concurrent catalog read. */
private passiveScheduledJobsScan?: Promise<PassiveScheduledJob[]>;
/**
@@ -970,11 +995,10 @@ export class DaemonSupervisor {
/**
* Durable truth: the ledger family (fork headers stripped) plus each session's
- * scheduled-jobs artifact. Always scans fresh per caller: management
- * decisions, wake recomputes, and the ephemeral-cancel retries they drive
- * must see the latest disk state. Every completed scan opportunistically
- * refreshes the snapshot that heartbeats_list serves, but only while it is
- * still the newest scan: an older scan loses the publish race.
+ * scheduled-jobs artifact. Scans fresh per caller: the fallback for snapshot
+ * misses and the recompute behind an invalidating event. Every completed scan
+ * opportunistically refreshes the snapshot that heartbeats_list serves, but
+ * only while it is still the newest scan: an older scan loses the publish race.
*/
private async collectPassiveScheduledJobs(includeInactive = false): Promise<PassiveScheduledJob[]> {
const epoch = this.claimPassiveScheduledJobsEpoch();
@@ -984,20 +1008,20 @@ export class DaemonSupervisor {
}
/**
- * Snapshot-served rows for heartbeats_list responses, where per-request
- * freshness matters less than not queueing: with hundreds of saved sessions,
- * N concurrent catalog requests used to enqueue N full scans and push the
- * last response past the client's 30s transport timeout. Daemon-owned
- * mutations drop the snapshot and detach the in-flight shared scan
- * (broadcastHeartbeatsChanged and the saved-session delete/rename paths);
- * the next list or the wake recompute armed by the same broadcast rescans,
- * and concurrent cold lists share one in-flight scan. A served snapshot
- * older than PASSIVE_SCHEDULED_JOBS_REFRESH_MS refreshes in the background
- * without being dropped, so lists that arrive during the refresh keep
- * serving bounded-stale rows and external artifact writes converge without
- * blocking a response.
+ * Snapshot-served rows for catalog responses (heartbeats_list, cron_list),
+ * where per-request freshness matters less than not queueing: with
+ * hundreds of saved sessions, N concurrent catalog requests used to enqueue
+ * N full scans and push the last response past the client's 30s transport
+ * timeout. Daemon-owned mutations patch their row in place, residency gains
+ * drop covered rows, and worker stops or saved-session delete/rename drop the
+ * snapshot and detach the in-flight scan, so the next list or the armed
+ * recompute rescans; concurrent cold lists share one in-flight scan. A served
+ * snapshot older than PASSIVE_SCHEDULED_JOBS_REFRESH_MS refreshes in the
+ * background without being dropped, so lists that arrive during the
+ * refresh keep serving bounded-stale rows and external artifact writes
+ * converge without blocking a response.
*/
- private async catalogPassiveScheduledJobs(): Promise<PassiveScheduledJob[]> {
+ private async catalogPassiveScheduledJobs(includeInactive = false): Promise<PassiveScheduledJob[]> {
const cached = this.passiveScheduledJobs;
if (cached) {
if (Date.now() - cached.scannedAt >= PASSIVE_SCHEDULED_JOBS_REFRESH_MS && !this.shuttingDown) {
@@ -1005,13 +1029,19 @@ export class DaemonSupervisor {
// background without dropping it, so requests that arrive during the
// refresh keep answering from bounded-stale rows instead of queueing
// behind the scan. A failure only logs; the next read retries.
- void this.sharedPassiveScheduledJobsScan().catch((error: unknown) =>
- this.log(`Passive scheduled-jobs refresh failed: ${String(error)}`),
- );
+ void this.sharedPassiveScheduledJobsScan()
+ .then((rows) => {
+ // External writer convergence: re-arm the wake so an externally created job still wakes.
+ if (!this.shuttingDown && JSON.stringify(rows) !== JSON.stringify(cached.rows)) {
+ this.scheduleScheduledSessionWakeRecompute();
+ }
+ })
+ .catch((error: unknown) => this.log(`Passive scheduled-jobs refresh failed: ${String(error)}`));
}
- return activeScheduledJobs(cached.rows);
+ return includeInactive ? cached.rows : activeScheduledJobs(cached.rows);
}
- return activeScheduledJobs(await this.sharedPassiveScheduledJobsScan());
+ const rows = await this.sharedPassiveScheduledJobsScan();
+ return includeInactive ? rows : activeScheduledJobs(rows);
}
private sharedPassiveScheduledJobsScan(): Promise<PassiveScheduledJob[]> {
@@ -1069,6 +1099,85 @@ export class DaemonSupervisor {
this.passiveScheduledJobsScan = undefined;
}
+ /**
+ * Serves each worker from its cached snapshot so repeated lists don't fan out; a
+ * worker that cannot answer fails the list rather than serving a partial catalog.
+ */
+ private async workerHeartbeatsForList(
+ worker: ResidentWorker,
+ command: Extract<DaemonCommand, { type: "heartbeats_list" }>,
+ ): Promise<{ heartbeats?: AgentConnectionHeartbeat[]; response?: DaemonResponse }> {
+ if (worker.client && worker.descriptor.lifecycle === "ready") {
+ if (worker.heartbeatSnapshot !== undefined && worker.heartbeatSnapshotStale !== true) {
+ this.maybeRefreshAgedWorkerHeartbeatSnapshot(worker);
+ return { heartbeats: worker.heartbeatSnapshot };
+ }
+ try {
+ await this.refreshWorkerHeartbeatSnapshot(worker);
+ } catch (error: unknown) {
+ return { response: failure(command.id, command.type, error, serializeDaemonError(error)) };
+ }
+ if (worker.heartbeatSnapshot !== undefined && worker.heartbeatSnapshotStale !== true) {
+ return { heartbeats: worker.heartbeatSnapshot };
+ }
+ }
+ if (worker.heartbeatSnapshot !== undefined && worker.heartbeatSnapshotStale !== true) {
+ return { heartbeats: worker.heartbeatSnapshot };
+ }
+ const state = worker.descriptor.lifecycle === "ready" ? "disconnected" : worker.descriptor.lifecycle;
+ const error = new Error(`Cannot list heartbeats while session worker is ${state}`);
+ return { response: failure(command.id, command.type, error, serializeDaemonError(error)) };
+ }
+
+ /**
+ * Targeted single-flight refresh of one worker: frames and lists join the
+ * in-flight pass.
+ */
+ private refreshWorkerHeartbeatSnapshot(worker: ResidentWorker): Promise<void> {
+ const inFlight = worker.heartbeatSnapshotRefresh;
+ if (inFlight) {
+ return inFlight;
+ }
+ const refresh = this.runWorkerHeartbeatSnapshotRefresh(worker);
+ const wrapper = refresh.finally(() => {
+ if (worker.heartbeatSnapshotRefresh === wrapper) {
+ worker.heartbeatSnapshotRefresh = undefined;
+ }
+ });
+ worker.heartbeatSnapshotRefresh = wrapper;
+ return wrapper;
+ }
+
+ private async runWorkerHeartbeatSnapshotRefresh(worker: ResidentWorker): Promise<void> {
+ while (true) {
+ worker.heartbeatSnapshotRefreshQueued = false;
+ if (worker.client && worker.descriptor.lifecycle === "ready") {
+ const response = await this.forwardToWorker(worker, { type: "heartbeats_list" }, 5000).catch(
+ (error: unknown) => failure(undefined, "heartbeats_list", error, serializeDaemonError(error)),
+ );
+ if (response.success) {
+ worker.heartbeatSnapshot = heartbeatsFromResponse(response);
+ worker.heartbeatSnapshotStale = false;
+ worker.heartbeatSnapshotAt = Date.now();
+ } else {
+ this.log(`Could not list heartbeats from a worker: ${response.error}`);
+ throw new Error(String(response.error));
+ }
+ }
+ if (!worker.heartbeatSnapshotRefreshQueued) return;
+ }
+ }
+
+ private maybeRefreshAgedWorkerHeartbeatSnapshot(worker: ResidentWorker): void {
+ if (
+ worker.heartbeatSnapshotRefresh ||
+ Date.now() - (worker.heartbeatSnapshotAt ?? 0) < WORKER_HEARTBEAT_SNAPSHOT_MAX_AGE_MS
+ ) {
+ return;
+ }
+ void this.refreshWorkerHeartbeatSnapshot(worker).catch(() => undefined);
+ }
+
/** The uncached disk scan; all statuses load so one row set serves every caller's filter. */
private async scanPassiveScheduledJobs(): Promise<PassiveScheduledJob[]> {
const pendingCancelRoots = new Set<string>();
@@ -1083,6 +1192,7 @@ export class DaemonSupervisor {
pendingCancelRoots.add(canonicalSessionPath(context.sessionFile));
}
}
+ this.ephemeralCancelIntentsPending = pendingCancelRoots.size > 0;
const infos = await this.rlmSpawnLedger().family(SESSION_SCHEDULED_JOBS_FILENAME);
const infoByPath = new Map(infos.map((info) => [canonicalSessionPath(info.path), info] as const));
const storeBySessionId = new Map<string, AgentCronJobStore>();
@@ -1097,23 +1207,6 @@ export class DaemonSupervisor {
infoBySessionId.set(info.id, info);
}
if (infoBySessionId.size === 0) return [];
- const uncoveredRootFor = (info: SessionInfo): string | undefined => {
- let current = info;
- const visited = new Set([canonicalSessionPath(current.path)]);
- while (true) {
- try {
- if (this.findWorkerBySessionFile(current.path)) return undefined;
- } catch {
- return undefined;
- }
- if (!current.parentSessionPath) break;
- const parent = infoByPath.get(canonicalSessionPath(current.parentSessionPath));
- if (!parent || visited.has(canonicalSessionPath(parent.path))) break;
- visited.add(canonicalSessionPath(parent.path));
- current = parent;
- }
- return current.path;
- };
const results: PassiveScheduledJob[] = [];
for (const [artifactSessionId, store] of storeBySessionId) {
let jobs: AgentCronJob[];
@@ -1126,7 +1219,7 @@ export class DaemonSupervisor {
for (const job of jobs) {
const info = infoBySessionId.get(job.sessionId);
if (!info) continue;
- const rootSessionFile = uncoveredRootFor(info);
+ const rootSessionFile = this.uncoveredRootForInfo(info, infoByPath);
if (rootSessionFile === undefined) continue;
if (pendingCancelRoots.has(canonicalSessionPath(rootSessionFile))) continue;
results.push({ rootSessionFile, job, info });
@@ -1135,9 +1228,104 @@ export class DaemonSupervisor {
return results;
}
+ /** Shared by the disk scan and the residency drop so both agree on coverage. */
+ private uncoveredRootForInfo(info: SessionInfo, infoByPath: Map<string, SessionInfo>): string | undefined {
+ let current = info;
+ const visited = new Set([canonicalSessionPath(current.path)]);
+ while (true) {
+ if (this.sessionFileCovered(current.path)) return undefined;
+ if (!current.parentSessionPath) break;
+ const parent = infoByPath.get(canonicalSessionPath(current.parentSessionPath));
+ if (!parent || visited.has(canonicalSessionPath(parent.path))) break;
+ visited.add(canonicalSessionPath(parent.path));
+ current = parent;
+ }
+ return current.path;
+ }
+
+ /** Wake-timer candidates: serve the snapshot whenever present; only invalidating events force a rescan. */
+ private async scheduledWakeCandidates(): Promise<PassiveScheduledJob[]> {
+ const cached = this.passiveScheduledJobs;
+ if (cached && !this.ephemeralCancelIntentsPending) return activeScheduledJobs(cached.rows);
+ return activeScheduledJobs(await this.collectPassiveScheduledJobs());
+ }
+
+ /**
+ * Drops now-covered snapshot rows without a disk scan. The snapshot is re-read
+ * after the ledger await, so rows patched meanwhile derive from their current
+ * form and never revert to a pre-await copy.
+ */
+ private async dropPassiveRowsCoveredByWorkers(): Promise<void> {
+ const infos = await this.rlmSpawnLedger().family(SESSION_SCHEDULED_JOBS_FILENAME);
+ const infoByPath = new Map(infos.map((info) => [canonicalSessionPath(info.path), info] as const));
+ const current = this.passiveScheduledJobs;
+ if (!current) return;
+ const rows: PassiveScheduledJob[] = [];
+ for (const row of current.rows) {
+ const rootSessionFile = this.uncoveredRootForInfo(row.info, infoByPath);
+ if (rootSessionFile === undefined) continue;
+ rows.push({ rootSessionFile, job: row.job, info: row.info });
+ }
+ this.passiveScheduledJobs = { rows, scannedAt: current.scannedAt };
+ }
+
+ /** The reporting worker's tree is resident, so no passive-row recompute is needed. */
+ private onWorkerHeartbeatsChanged(worker: ResidentWorker): void {
+ if (worker.heartbeatSnapshotRefresh) {
+ // The in-flight pass may have been answered before this change; run one more.
+ worker.heartbeatSnapshotRefreshQueued = true;
+ } else {
+ void this.refreshWorkerHeartbeatSnapshot(worker).catch(() => undefined);
+ }
+ this.notifyHeartbeatsChanged();
+ }
+
+ /**
+ * Marks the old incarnation's snapshot suspect and drops now-covered rows before
+ * re-arming; a failed drop forces a scan, since covered rows would pin the wake
+ * timer at delay 0.
+ */
+ private onWorkerResidencyGained(worker: ResidentWorker): void {
+ worker.heartbeatSnapshotStale = true;
+ void this.dropPassiveRowsCoveredByWorkers()
+ .then(() => {
+ this.scheduleScheduledSessionWakeRecompute();
+ this.notifyHeartbeatsChanged();
+ })
+ .catch((error) => {
+ this.log(`Failed to refresh scheduled-jobs coverage after a worker became resident: ${String(error)}`);
+ this.broadcastHeartbeatsChanged();
+ });
+ }
+
+ /** Snapshot first; a miss falls back to one fresh scan so externally created jobs still resolve. */
+ private async findPassiveScheduledJob(
+ match: (job: AgentCronJob) => boolean,
+ ): Promise<PassiveScheduledJob | undefined> {
+ const cached = this.passiveScheduledJobs;
+ if (cached) {
+ const row = cached.rows.find(({ job }) => match(job));
+ if (row) return row;
+ }
+ return (await this.collectPassiveScheduledJobs()).find(({ job }) => match(job));
+ }
+
+ /** Patches the mutated row in place instead of dropping the snapshot; then re-arms the wake timer. */
+ private onPassiveScheduledJobMutated(job: AgentCronJob): void {
+ const cached = this.passiveScheduledJobs;
+ if (cached) {
+ this.passiveScheduledJobs = {
+ ...cached,
+ rows: cached.rows.map((row) => (row.job.id === job.id ? { ...row, job } : row)),
+ };
+ }
+ this.scheduleScheduledSessionWakeRecompute();
+ this.notifyHeartbeatsChanged();
+ }
+
private async recomputeScheduledSessionWake(): Promise<void> {
if (this.shuttingDown || this.updateRestartPhase !== undefined) return;
- const candidates = await this.collectPassiveScheduledJobs();
+ const candidates = await this.scheduledWakeCandidates();
this.clearScheduledWakeTimer();
if (this.shuttingDown || this.updateRestartPhase !== undefined) return;
const now = Date.now();
@@ -1173,7 +1361,7 @@ export class DaemonSupervisor {
}
try {
const due = new Map<string, string>();
- for (const { rootSessionFile, job } of await this.collectPassiveScheduledJobs()) {
+ for (const { rootSessionFile, job } of await this.scheduledWakeCandidates()) {
if (job.status !== "active" || job.nextRunAt === undefined) continue;
const runAt = Date.parse(job.nextRunAt);
if (!Number.isFinite(runAt) || runAt > now) continue;
@@ -2052,6 +2240,9 @@ export class DaemonSupervisor {
command: DaemonCommand,
cancellationAdmission?: SupervisorPromptAdmission,
): Promise<DaemonResponse | undefined> {
+ if (HEARTBEAT_TRACKING_COMMANDS.has(command.type)) {
+ client.tracksHeartbeats = true;
+ }
switch (command.type) {
case "cancel_prompt_admission": {
const admission =
@@ -2148,6 +2339,9 @@ export class DaemonSupervisor {
return success(command.id, "create", this.publicSummary(worker, sessionSummaryFromRosterEntry(root)));
}
case "attach": {
+ if (command.capabilities?.includes("heartbeat_catalog")) {
+ client.tracksHeartbeats = true;
+ }
const attached = await this.attachClient(client, command);
if (client.capabilities.has("chunked_snapshot")) {
const transcript = attached.transcript;
@@ -2178,6 +2372,9 @@ export class DaemonSupervisor {
return success(command.id, "attach", attached.result);
}
case "reattach": {
+ if (command.capabilities?.includes("heartbeat_catalog")) {
+ client.tracksHeartbeats = true;
+ }
const target = await this.findWorkerForClient(client, command.targetActiveSessionId);
const targetActiveSessionId = target.summary.activeSessionId ?? target.summary.id;
if (targetActiveSessionId === command.activeSessionId) {
@@ -2438,7 +2635,7 @@ export class DaemonSupervisor {
jobs.set(job.id, job);
}
}
- for (const { job } of await this.collectPassiveScheduledJobs(command.includeInactive === true)) {
+ for (const { job } of await this.catalogPassiveScheduledJobs(command.includeInactive === true)) {
if (!jobs.has(job.id)) jobs.set(job.id, job);
}
return success(command.id, "cron_list", { jobs: sortCronJobs([...jobs.values()]) });
@@ -2482,32 +2679,7 @@ export class DaemonSupervisor {
);
const heartbeats = new Map<string, AgentConnectionHeartbeat>();
const snapshots: Array<{ heartbeats?: AgentConnectionHeartbeat[]; response?: DaemonResponse }> =
- await Promise.all(
- workers.map(async (worker) => {
- if (worker.client && worker.descriptor.lifecycle === "ready") {
- const response = await this.forwardToWorker(worker, command, 5000).catch((error: unknown) =>
- failure(command.id, command.type, error, serializeDaemonError(error)),
- );
- if (response.success) {
- const snapshot = heartbeatsFromResponse(response);
- worker.heartbeatSnapshot = snapshot;
- worker.heartbeatSnapshotStale = false;
- return { heartbeats: snapshot };
- }
- this.log(`Could not list heartbeats from a worker: ${response.error}`);
- if (worker.heartbeatSnapshot === undefined || worker.heartbeatSnapshotStale === true) {
- return { response };
- }
- }
- if (worker.heartbeatSnapshot !== undefined && worker.heartbeatSnapshotStale !== true) {
- return { heartbeats: worker.heartbeatSnapshot };
- }
- const state =
- worker.descriptor.lifecycle === "ready" ? "disconnected" : worker.descriptor.lifecycle;
- const error = new Error(`Cannot list heartbeats while session worker is ${state}`);
- return { response: failure(command.id, command.type, error, serializeDaemonError(error)) };
- }),
- );
+ await Promise.all(workers.map((worker) => this.workerHeartbeatsForList(worker, command)));
const failed = snapshots.find((snapshot) => snapshot.response)?.response;
if (failed) {
return failed;
@@ -2537,8 +2709,8 @@ export class DaemonSupervisor {
);
if (!cachedWorker) {
// Passive jobs are managed against their durable store; no wake just to flip a status.
- const passive = (await this.collectPassiveScheduledJobs()).find(
- ({ job }) => job.id === command.jobId && job.activeSessionId === command.activeSessionId,
+ const passive = await this.findPassiveScheduledJob(
+ (job) => job.id === command.jobId && job.activeSessionId === command.activeSessionId,
);
if (passive) {
const store = AgentCronJobStore.forSessionArtifacts();
@@ -2546,9 +2718,18 @@ export class DaemonSupervisor {
passive.info.id,
getSessionArtifactPathForFile(resolve(passive.info.path), passive.info.id),
);
- const heartbeat = store.manageHeartbeat(command.activeSessionId, command.jobId, command.action);
+ // Re-checked under the artifact lock; a covered session falls through to the worker forward below.
+ const heartbeat = await store.manageHeartbeat(
+ command.activeSessionId,
+ command.jobId,
+ command.action,
+ new Date(),
+ () =>
+ !this.sessionFileCovered(passive.rootSessionFile) &&
+ !this.sessionFileCovered(passive.info.path),
+ );
if (heartbeat) {
- this.broadcastHeartbeatsChanged();
+ this.onPassiveScheduledJobMutated(heartbeat);
return success(command.id, "heartbeat_manage", { heartbeat });
}
}
@@ -2609,16 +2790,16 @@ export class DaemonSupervisor {
return this.forwardToWorker(candidate.worker, command);
}
}
- const passive = (await this.collectPassiveScheduledJobs()).find(({ job }) => job.id === command.jobId);
+ const passive = await this.findPassiveScheduledJob((job) => job.id === command.jobId);
if (passive) {
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact(
passive.info.id,
getSessionArtifactPathForFile(resolve(passive.info.path), passive.info.id),
);
- const job = store.cancel(command.jobId);
+ const job = await store.cancel(command.jobId);
if (job) {
- this.broadcastHeartbeatsChanged();
+ this.onPassiveScheduledJobMutated(job);
return success(command.id, "cron_cancel", { job });
}
}
@@ -3478,7 +3659,7 @@ export class DaemonSupervisor {
worker.launchEnv = undefined;
worker.transientCreateCommand = undefined;
}
- this.broadcastHeartbeatsChanged();
+ this.onWorkerResidencyGained(worker);
return worker;
} catch (error) {
if (isSupervisorGenerationStale(error)) {
@@ -3669,7 +3850,7 @@ export class DaemonSupervisor {
worker.descriptor.consecutiveFailures = 0;
worker.deferredRecoveryRounds = 0;
this.persistWorker(worker);
- this.broadcastHeartbeatsChanged();
+ this.onWorkerResidencyGained(worker);
} catch (error) {
if (isSupervisorRecoveryCancelled(error)) {
return;
@@ -4137,7 +4318,7 @@ export class DaemonSupervisor {
worker.descriptor.consecutiveFailures = 0;
worker.deferredRecoveryRounds = 0;
this.persistWorker(worker);
- this.broadcastHeartbeatsChanged();
+ this.onWorkerResidencyGained(worker);
return;
} catch (error) {
if (isSupervisorRecoveryCancelled(error)) {
@@ -5185,6 +5366,15 @@ export class DaemonSupervisor {
});
}
+ /** Ambiguous residency counts as covered: never treat a contested session as passive. */
+ private sessionFileCovered(sessionFile: string, exclude?: ResidentWorker): boolean {
+ try {
+ return this.findWorkerBySessionFile(sessionFile, exclude) !== undefined;
+ } catch {
+ return true;
+ }
+ }
+
private findWorkerBySessionFile(sessionFile: string, exclude?: ResidentWorker): ResidentWorker | undefined {
const target = canonicalSessionPath(sessionFile);
const targetEntry = this.roster().bySessionFile(target);
@@ -5799,7 +5989,7 @@ export class DaemonSupervisor {
}
if (outboundType === "heartbeats_changed") {
worker.heartbeatSnapshotStale = true;
- this.broadcastHeartbeatsChanged();
+ this.onWorkerHeartbeatsChanged(worker);
return;
}
if (outboundType === "session_snapshot_begin" && activeSessionId) {
@@ -6991,7 +7181,7 @@ export class DaemonSupervisor {
if (worker.descriptor.rootSessionId) {
const cronStore = AgentCronJobStore.forSessionArtifacts();
cronStore.registerSessionArtifact(worker.descriptor.rootSessionId, context.artifactDir);
- cronStore.cancelJobsForSession({
+ await cronStore.cancelJobsForSession({
sessionId: worker.descriptor.rootSessionId,
sessionFile: context.sessionFile,
});
@@ -7025,13 +7215,10 @@ export class DaemonSupervisor {
}
}
// A worker covering any tree member owns its stores again; a stale intent must not kill new schedules.
- for (const { sessionFile } of sessions) {
- try {
- if (this.findWorkerBySessionFile(sessionFile, exclude)) return;
- } catch {
- return;
- }
- }
+ const cancelStillWanted = (): boolean =>
+ (stillWanted === undefined || stillWanted()) &&
+ sessions.every(({ sessionFile }) => !this.sessionFileCovered(sessionFile, exclude));
+ if (!cancelStillWanted()) return;
let registered = false;
for (const { sessionId, sessionFile } of sessions) {
const artifactDir = getSessionArtifactPathForFile(sessionFile, sessionId);
@@ -7040,10 +7227,9 @@ export class DaemonSupervisor {
registered = true;
}
if (!registered) return;
- // Re-checked in the same synchronous turn as the walk: a promotion committed during the family read keeps its schedules.
- if (stillWanted && !stillWanted()) return;
for (const { sessionFile } of sessions) {
- store.cancelJobsForSession({ sessionFile });
+ if (!cancelStillWanted()) return;
+ await store.cancelJobsForSession({ sessionFile }, new Date(), cancelStillWanted);
}
}
@@ -7105,13 +7291,40 @@ export class DaemonSupervisor {
}
private broadcastHeartbeatsChanged(): void {
- // Every daemon-owned scheduled-job mutation and worker residency change
- // lands here: drop the shared snapshot so the next catalog read (or the
- // recompute armed below) rescans instead of serving pre-mutation rows.
+ // Worker stops and externally-mutated artifacts land here: drop the
+ // shared snapshot so the next catalog read (or the recompute armed
+ // below) rescans instead of serving pre-mutation rows.
this.invalidatePassiveScheduledJobs();
this.scheduleScheduledSessionWakeRecompute();
- for (const client of this.clients) {
- this.write(client, { type: "heartbeats_changed" });
+ this.notifyHeartbeatsChanged();
+ }
+
+ /**
+ * Advisory pushes: clients also re-list on their own schedules, so delivery is
+ * coalesced per burst window and restricted to clients that track the catalog.
+ */
+ private notifyHeartbeatsChanged(): void {
+ if (this.shuttingDown || this.heartbeatsChangedBroadcastTimer) return;
+ this.heartbeatsChangedBroadcastTimer = setTimeout(() => {
+ this.heartbeatsChangedBroadcastTimer = undefined;
+ if (this.shuttingDown) return;
+ // A detached timer callback must never crash the supervisor: a delivery failure only logs.
+ try {
+ for (const client of this.clients) {
+ if (client.tracksHeartbeats !== true) continue;
+ this.write(client, { type: "heartbeats_changed" });
+ }
+ } catch (error) {
+ this.log(`Could not deliver heartbeats_changed: ${String(error)}`);
+ }
+ }, HEARTBEATS_CHANGED_COALESCE_MS);
+ this.heartbeatsChangedBroadcastTimer.unref?.();
+ }
+
+ private clearHeartbeatsChangedBroadcast(): void {
+ if (this.heartbeatsChangedBroadcastTimer) {
+ clearTimeout(this.heartbeatsChangedBroadcastTimer);
+ this.heartbeatsChangedBroadcastTimer = undefined;
}
}
@@ -7205,6 +7418,7 @@ export class DaemonSupervisor {
private async cleanupSupervisorResourcesOnce(): Promise<void> {
this.shuttingDown = true;
this.clearIdleEvictionTimer();
+ this.clearHeartbeatsChangedBroadcast();
this.clearScheduledWakeTimer();
this.clearRosterWatchdogTimer();
await this.idleEvictionSweep?.catch(() => undefined);
@@ -7307,6 +7521,7 @@ export class DaemonSupervisor {
}
this.shuttingDown = true;
this.clearIdleEvictionTimer();
+ this.clearHeartbeatsChangedBroadcast();
this.clearScheduledWakeTimer();
this.clearRosterWatchdogTimer();
await this.idleEvictionSweep?.catch(() => undefined);
diff --git a/packages/coding-agent/test/agent-connection-daemon.test.ts b/packages/coding-agent/test/agent-connection-daemon.test.ts
index 1b8bebf229..0a876358ab 100644
--- a/packages/coding-agent/test/agent-connection-daemon.test.ts
+++ b/packages/coding-agent/test/agent-connection-daemon.test.ts
@@ -2718,6 +2718,20 @@ describe("DaemonAgentConnection", () => {
resumeCursor: { generation: "generation-new", sequence: 1 },
});
});
+
+ it("advertises the heartbeat_catalog capability on attach only when it tracks heartbeats", async () => {
+ const attach = async (tracksHeartbeats?: boolean) => {
+ const client = new FakeDaemonClient();
+ const connection = await DaemonAgentConnection.attach(asDaemonClient(client), "active-1", {
+ closeClientOnDispose: true,
+ tracksHeartbeats,
+ });
+ await connection.dispose();
+ return client.requests.find((request) => request.type === "attach") as { capabilities?: string[] };
+ };
+ expect((await attach(true)).capabilities).toContain("heartbeat_catalog");
+ expect((await attach()).capabilities).not.toContain("heartbeat_catalog");
+ });
});
const DEFERRAL_EVENT_BASE_SEQUENCE = 13;
diff --git a/packages/coding-agent/test/cron-jobs-state-snapshot.test.ts b/packages/coding-agent/test/cron-jobs-state-snapshot.test.ts
index 928c1d7092..cbe19ed959 100644
--- a/packages/coding-agent/test/cron-jobs-state-snapshot.test.ts
+++ b/packages/coding-agent/test/cron-jobs-state-snapshot.test.ts
@@ -1,5 +1,5 @@
import type * as FsModule from "node:fs";
-import { mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
+import { mkdtempSync, readFileSync, renameSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -78,10 +78,10 @@ describe("AgentCronJobStore state snapshots", () => {
renameAfterHook.after = undefined;
});
- it("serves repeated reads of an unchanged file from the in-memory snapshot", () => {
+ it("serves repeated reads of an unchanged file from the in-memory snapshot", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -101,10 +101,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(0);
});
- it("publishes the mutated state so the next read does not re-parse the file", () => {
+ it("publishes the mutated state so the next read does not re-parse the file", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -121,10 +121,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(0);
});
- it("re-reads the file when an external writer changes it and caches the new parse", () => {
+ it("re-reads the file when an external writer changes it and caches the new parse", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -155,10 +155,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(1);
});
- it("re-reads when the file is replaced by a fresh inode with the same size and mtime", () => {
+ it("re-reads when the file is replaced by a fresh inode with the same size and mtime", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -173,19 +173,23 @@ describe("AgentCronJobStore state snapshots", () => {
const bytes = readFileSync(storePath, "utf-8");
readCounts.clear();
+ // A fresh inode is guaranteed by creating the replacement while the
+ // original still exists; in-place recreation can reuse the freed inode.
+ const replacementPath = `${storePath}.replaced`;
+ writeFileSync(replacementPath, bytes);
rmSync(storePath);
- writeFileSync(storePath, bytes);
+ renameSync(replacementPath, storePath);
utimesSync(storePath, pinned, pinned);
expect(store.list()).toHaveLength(1);
expect(jobsReads(storePath)).toBe(1);
});
- it("keeps concurrent store instances on one file consistent", () => {
+ it("keeps concurrent store instances on one file consistent", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const first = new AgentCronJobStore(storePath);
const second = new AgentCronJobStore(storePath);
- const jobA = first.create({
+ const jobA = await first.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -196,7 +200,7 @@ describe("AgentCronJobStore state snapshots", () => {
});
expect(second.list().map((candidate) => candidate.id)).toEqual([jobA.id]);
- const jobB = second.create({
+ const jobB = await second.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/session-2.jsonl",
@@ -232,10 +236,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(second.list().map((candidate) => candidate.id)).toEqual(["external-1"]);
});
- it("drops the snapshot when a mutation fails to persist and re-reads the disk", () => {
+ it("drops the snapshot when a mutation fails to persist and re-reads the disk", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- const first = store.create({
+ const first = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -250,7 +254,7 @@ describe("AgentCronJobStore state snapshots", () => {
renameFault.remaining = 1;
renameFault.path = storePath;
- expect(() =>
+ await expect(
store.create({
activeSessionId: "active-2",
sessionId: "session-2",
@@ -260,7 +264,7 @@ describe("AgentCronJobStore state snapshots", () => {
prompt: "never lands",
now: start,
}),
- ).toThrow();
+ ).rejects.toThrow();
readCounts.clear();
expect(store.list().map((candidate) => candidate.id)).toEqual([first.id]);
@@ -289,14 +293,14 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(0);
});
- it("caches per registered session artifact and re-reads only the changed file", () => {
+ it("caches per registered session artifact and re-reads only the changed file", async () => {
const root = makeTempDir(tempDirs);
const store = AgentCronJobStore.forSessionArtifacts();
const firstDir = join(root, "artifacts-1");
const secondDir = join(root, "artifacts-2");
store.registerSessionArtifact("session-1", firstDir);
store.registerSessionArtifact("session-2", secondDir);
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: join(root, "session-1.jsonl"),
@@ -305,7 +309,7 @@ describe("AgentCronJobStore state snapshots", () => {
prompt: "first heartbeat",
now: start,
});
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: join(root, "session-2.jsonl"),
@@ -341,10 +345,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(secondPath)).toBe(0);
});
- it("serves read-only views: an in-place edit of a listed job cannot change store behavior or reach disk", () => {
+ it("serves read-only views: an in-place edit of a listed job cannot change store behavior or reach disk", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -369,7 +373,7 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(0);
// An unrelated mutation must persist the true state, not a leaked in-place edit.
- store.create({
+ await store.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/session-2.jsonl",
@@ -383,10 +387,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(store.list()).toHaveLength(2);
});
- it("keeps the published snapshot intact when a mutator throws mid-edit", () => {
+ it("keeps the published snapshot intact when a mutator throws mid-edit", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -397,20 +401,23 @@ describe("AgentCronJobStore state snapshots", () => {
});
readCounts.clear();
- expect(() =>
+ // The mutator runs once lock-free to probe and once under the lock; both passes throw.
+ await expect(
(
- store as unknown as { mutateStates: (mutator: (state: MutableJobsState) => unknown[]) => unknown[] }
+ store as unknown as {
+ mutateStates: (mutator: (state: MutableJobsState) => unknown[]) => Promise<unknown[]>;
+ }
).mutateStates((state) => {
state.jobs = state.jobs.map((candidate) => ({ ...candidate, status: "cancelled" as const }));
throw new Error("mutator crashed mid-edit");
}),
- ).toThrow("mutator crashed mid-edit");
+ ).rejects.toThrow("mutator crashed mid-edit");
// The unpersisted partial edit never reaches reads...
expect(store.list().map((candidate) => candidate.status)).toEqual(["active"]);
expect(jobsReads(storePath)).toBe(0);
// ...nor does a later unrelated mutation resurrect it.
- store.create({
+ await store.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/session-2.jsonl",
@@ -424,10 +431,10 @@ describe("AgentCronJobStore state snapshots", () => {
expect(store.list().map((candidate) => candidate.status)).toEqual(["active", "active"]);
});
- it("re-reads instead of caching an external replacement that lands right after our own write", () => {
+ it("re-reads instead of caching an external replacement that lands right after our own write", async () => {
const storePath = join(makeTempDir(tempDirs), "cron-jobs.json");
const store = new AgentCronJobStore(storePath);
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session-1.jsonl",
@@ -454,7 +461,7 @@ describe("AgentCronJobStore state snapshots", () => {
new Date("2026-01-08T00:00:00.000Z"),
);
};
- store.create({
+ await store.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/session-2.jsonl",
@@ -470,7 +477,7 @@ describe("AgentCronJobStore state snapshots", () => {
expect(jobsReads(storePath)).toBe(1);
// A later mutation must operate on the external state, not our stale write.
- const cancelled = store.cancel("external-raced");
+ const cancelled = await store.cancel("external-raced");
expect(cancelled?.status).toBe("cancelled");
expect(store.list()[0]?.status).toBe("cancelled");
});
diff --git a/packages/coding-agent/test/cron-jobs.test.ts b/packages/coding-agent/test/cron-jobs.test.ts
index 2f697a5e76..569242d6f6 100644
--- a/packages/coding-agent/test/cron-jobs.test.ts
+++ b/packages/coding-agent/test/cron-jobs.test.ts
@@ -1,6 +1,7 @@
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { lock } from "proper-lockfile";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CreateAgentSessionRuntimeFactory } from "../src/core/agent-session-runtime.js";
import {
@@ -17,6 +18,7 @@ import {
} from "../src/core/cron-jobs.js";
import type { ActiveSessionState } from "../src/modes/daemon/active-session-state.js";
import { AgentDaemon } from "../src/modes/daemon/daemon-mode.js";
+import * as atomicFile from "../src/utils/atomic-file.js";
import { createHarness, type Harness } from "./suite/harness.js";
const start = new Date("2026-01-01T12:34:00.000Z");
@@ -168,10 +170,10 @@ describe("AgentCronJobStore", () => {
}
});
- it("persists, reloads, and cancels jobs", () => {
+ it("persists, reloads, and cancels jobs", async () => {
const storePath = makeStorePath(tempDirs);
const store = new AgentCronJobStore(storePath);
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -190,21 +192,21 @@ describe("AgentCronJobStore", () => {
},
]);
- const cancelled = store.cancel(job.id, new Date("2026-01-01T12:40:00.000Z"));
+ const cancelled = await store.cancel(job.id, new Date("2026-01-01T12:40:00.000Z"));
expect(cancelled).toMatchObject({ id: job.id, status: "cancelled" });
expect(store.list()[0]).toMatchObject({ id: job.id, status: "cancelled" });
expect(store.list()[0]).not.toHaveProperty("nextRunAt");
});
- it("isolates worker-owned jobs in registered session artifact stores", () => {
+ it("isolates worker-owned jobs in registered session artifact stores", async () => {
const root = makeTempDir(tempDirs);
const firstArtifactDir = join(root, "session-artifacts", "session-1");
const secondArtifactDir = join(root, "session-artifacts", "session-2");
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact("session-1", firstArtifactDir);
store.registerSessionArtifact("session-2", secondArtifactDir);
- const first = store.createHeartbeat({
+ const first = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: join(root, "sessions", "session-1.jsonl"),
@@ -213,7 +215,7 @@ describe("AgentCronJobStore", () => {
prompt: "first heartbeat",
now: start,
});
- const second = store.createHeartbeat({
+ const second = await store.createHeartbeat({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: join(root, "sessions", "session-2.jsonl"),
@@ -234,14 +236,14 @@ describe("AgentCronJobStore", () => {
);
});
- it("moves active-session jobs to the replacement session artifact store", () => {
+ it("moves active-session jobs to the replacement session artifact store", async () => {
const root = makeTempDir(tempDirs);
const firstArtifactDir = join(root, "session-artifacts", "session-1");
const secondArtifactDir = join(root, "session-artifacts", "session-2");
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact("session-1", firstArtifactDir);
store.registerSessionArtifact("session-2", secondArtifactDir);
- const heartbeat = store.createHeartbeat({
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: join(root, "sessions", "session-1.jsonl"),
@@ -251,7 +253,7 @@ describe("AgentCronJobStore", () => {
now: start,
});
- store.rebindSessionJobs({
+ await store.rebindSessionJobs({
activeSessionId: "active-1",
sessionId: "session-2",
sessionFile: join(root, "sessions", "session-2.jsonl"),
@@ -268,11 +270,11 @@ describe("AgentCronJobStore", () => {
]);
});
- it("migrates the legacy global store into per-session artifact stores", () => {
+ it("migrates the legacy global store into per-session artifact stores", async () => {
const root = makeTempDir(tempDirs);
const legacyPath = join(root, "cron-jobs.json");
const legacy = new AgentCronJobStore(legacyPath);
- const first = legacy.create({
+ const first = await legacy.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: join(root, "sessions", "session-1.jsonl"),
@@ -281,7 +283,7 @@ describe("AgentCronJobStore", () => {
prompt: "first migrated job",
now: start,
});
- const second = legacy.create({
+ const second = await legacy.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: join(root, "sessions", "session-2.jsonl"),
@@ -299,11 +301,11 @@ describe("AgentCronJobStore", () => {
expect(migrated.list().map((job) => job.id)).toEqual(expect.arrayContaining([first.id, second.id]));
});
- it("marks in-flight legacy dispatches interrupted during migration", () => {
+ it("marks in-flight legacy dispatches interrupted during migration", async () => {
const root = makeTempDir(tempDirs);
const legacyPath = join(root, "cron-jobs.json");
const legacy = new AgentCronJobStore(legacyPath);
- const heartbeat = legacy.createHeartbeat({
+ const heartbeat = await legacy.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: join(root, "sessions", "session-1.jsonl"),
@@ -312,7 +314,7 @@ describe("AgentCronJobStore", () => {
prompt: "migrated heartbeat",
now: start,
});
- legacy.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ await legacy.claimDue(new Date("2026-01-01T12:34:10.000Z"));
expect(
migrateLegacyCronJobsToSessionArtifacts(legacyPath, {
@@ -330,9 +332,9 @@ describe("AgentCronJobStore", () => {
]);
});
- it("keeps overdue jobs eligible for the scheduler after restart", () => {
+ it("keeps overdue jobs eligible for the scheduler after restart", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -345,123 +347,92 @@ describe("AgentCronJobStore", () => {
expect(store.nextActiveRunAt()?.toISOString()).toBe("2026-01-01T12:35:00.000Z");
});
- it("preserves concurrent cron store writes when a stale snapshot is written", () => {
+ it("keeps one persistent heartbeat per active session", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const first = store.create({
+ const first = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
cwd: "/tmp/project",
- scheduleText: "in 1h",
- prompt: "first",
+ scheduleText: "every 30s",
+ prompt: "check on me",
now: start,
});
- const staleSnapshot = [first];
- const second = store.create({
+ const second = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
cwd: "/tmp/project",
- scheduleText: "in 2h",
- prompt: "second",
+ scheduleText: "every 5m",
+ prompt: "continue the work",
now: new Date("2026-01-01T12:35:00.000Z"),
});
- writeJobsForTest(store, staleSnapshot);
-
- expect(store.list()).toEqual(
- expect.arrayContaining([
- expect.objectContaining({ id: first.id, prompt: "first" }),
- expect.objectContaining({ id: second.id, prompt: "second" }),
- ]),
- );
+ expect(store.getHeartbeat("active-1")).toMatchObject({ id: second.id, prompt: "continue the work" });
+ expect(store.list().find((job) => job.id === first.id)).toMatchObject({ status: "cancelled" });
});
- it("keeps newer cron store state when a stale snapshot is written", () => {
- const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const heartbeat = store.createHeartbeat({
+ it("keeps one heartbeat when two creates for a session overlap", async () => {
+ const root = makeTempDir(tempDirs);
+ const store = AgentCronJobStore.forSessionArtifacts();
+ store.registerSessionArtifact("session-1", join(root, "artifacts", "session-1"));
+ const input = {
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
cwd: "/tmp/project",
scheduleText: "every 5m",
- prompt: "check on me",
- now: start,
- });
- store.pauseHeartbeat("active-1", new Date("2026-01-01T12:35:00.000Z"));
-
- writeJobsForTest(store, [heartbeat]);
+ };
+ await Promise.all([
+ store.createHeartbeat({ ...input, prompt: "first", now: start }),
+ store.createHeartbeat({ ...input, prompt: "second", now: new Date("2026-01-01T12:35:00.000Z") }),
+ ]);
- expect(store.getHeartbeat("active-1")).toMatchObject({
- id: heartbeat.id,
- status: "paused",
- updatedAt: "2026-01-01T12:35:00.000Z",
- });
+ expect(
+ store
+ .list()
+ .map((job) => job.status)
+ .sort(),
+ ).toEqual(["active", "cancelled"]);
});
- it("preserves session-artifact dispatches when a stale snapshot is written", () => {
+ it("keeps a claimed dispatch with its job when another catalog write lands", async () => {
const root = makeTempDir(tempDirs);
- const artifactDir = join(root, "session-artifacts", "session-1");
- const artifactPath = join(artifactDir, SESSION_SCHEDULED_JOBS_FILENAME);
- const writer = AgentCronJobStore.forSessionArtifacts();
- const dispatcher = AgentCronJobStore.forSessionArtifacts();
- writer.registerSessionArtifact("session-1", artifactDir);
- dispatcher.registerSessionArtifact("session-1", artifactDir);
- const heartbeat = writer.createHeartbeat({
+ const store = AgentCronJobStore.forSessionArtifacts();
+ store.registerSessionArtifact("session-1", join(root, "artifacts", "session-1"));
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
- sessionFile: join(root, "sessions", "session-1.jsonl"),
- cwd: root,
+ sessionFile: "/tmp/session.jsonl",
+ cwd: "/tmp/project",
scheduleText: "every 10s",
prompt: "check progress",
now: start,
});
- const staleCancellation = {
- ...heartbeat,
- status: "cancelled" as const,
- nextRunAt: undefined,
- updatedAt: "2026-01-01T12:34:20.000Z",
- };
-
- expect(dispatcher.claimDue(new Date("2026-01-01T12:34:10.000Z"))).toHaveLength(1);
- writeJobsForTest(writer, [staleCancellation]);
-
- const state = JSON.parse(readFileSync(artifactPath, "utf8")) as {
- jobs: AgentCronJob[];
- dispatches: Array<{ jobId: string }>;
- };
- expect(state.jobs).toContainEqual(expect.objectContaining({ id: heartbeat.id, status: "cancelled" }));
- expect(state.dispatches).toContainEqual(expect.objectContaining({ jobId: heartbeat.id }));
- });
-
- it("keeps one persistent heartbeat per active session", () => {
- const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const first = store.createHeartbeat({
- activeSessionId: "active-1",
+ const oneShot = await store.create({
+ activeSessionId: "active-2",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
cwd: "/tmp/project",
- scheduleText: "every 30s",
- prompt: "check on me",
+ scheduleText: "in 1h",
+ prompt: "one shot",
now: start,
});
- const second = store.createHeartbeat({
- activeSessionId: "active-1",
- sessionId: "session-1",
- sessionFile: "/tmp/session.jsonl",
- cwd: "/tmp/project",
- scheduleText: "every 5m",
- prompt: "continue the work",
- now: new Date("2026-01-01T12:35:00.000Z"),
- });
- expect(store.getHeartbeat("active-1")).toMatchObject({ id: second.id, prompt: "continue the work" });
- expect(store.list().find((job) => job.id === first.id)).toMatchObject({ status: "cancelled" });
+ const [dispatch] = await store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ if (!dispatch) throw new Error("Expected heartbeat dispatch");
+ await store.cancel(oneShot.id);
+
+ expect(store.getClaimedJob(heartbeat.id)).toBeDefined();
+ expect(await store.recordDispatchResult(dispatch.id, { outcome: "ran" })).toMatchObject({
+ id: heartbeat.id,
+ runCount: 1,
+ });
});
- it("defaults heartbeats to steer delivery and persists an explicit follow_up opt-out", () => {
+ it("defaults heartbeats to steer delivery and persists an explicit follow_up opt-out", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const steerHeartbeat = store.createHeartbeat({
+ const steerHeartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -472,7 +443,7 @@ describe("AgentCronJobStore", () => {
});
expect(steerHeartbeat.deliveryMode).toBe("steer");
- const followUpHeartbeat = store.createHeartbeat({
+ const followUpHeartbeat = await store.createHeartbeat({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/session-2.jsonl",
@@ -486,9 +457,9 @@ describe("AgentCronJobStore", () => {
expect(store.getHeartbeat("active-2")).toMatchObject({ deliveryMode: "follow_up" });
});
- it("defaults RLM heartbeats to steer delivery and updates the delivery mode", () => {
+ it("defaults RLM heartbeats to steer delivery and updates the delivery mode", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const rlmHeartbeat = store.createRlmHeartbeat({
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -499,7 +470,7 @@ describe("AgentCronJobStore", () => {
});
expect(rlmHeartbeat.deliveryMode).toBe("steer");
- const updated = store.updateRlmHeartbeat("active-1", rlmHeartbeat.id, {
+ const updated = await store.updateRlmHeartbeat("active-1", rlmHeartbeat.id, {
deliveryMode: "follow_up",
now: new Date("2026-01-01T12:35:00.000Z"),
});
@@ -507,9 +478,9 @@ describe("AgentCronJobStore", () => {
expect(store.listRlmHeartbeats("active-1")[0]).toMatchObject({ deliveryMode: "follow_up" });
});
- it("pauses, resumes, and clears heartbeat state", () => {
+ it("pauses, resumes, and clears heartbeat state", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const job = store.createHeartbeat({
+ const job = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -519,17 +490,17 @@ describe("AgentCronJobStore", () => {
now: start,
});
- expect(store.pauseHeartbeat("active-1", new Date("2026-01-01T12:34:10.000Z"))).toMatchObject({
+ expect(await store.pauseHeartbeat("active-1", new Date("2026-01-01T12:34:10.000Z"))).toMatchObject({
id: job.id,
status: "paused",
});
expect(store.getHeartbeat("active-1")).not.toHaveProperty("nextRunAt");
- expect(store.resumeHeartbeat("active-1", new Date("2026-01-01T12:35:00.000Z"))).toMatchObject({
+ expect(await store.resumeHeartbeat("active-1", new Date("2026-01-01T12:35:00.000Z"))).toMatchObject({
id: job.id,
status: "active",
nextRunAt: "2026-01-01T12:35:30.000Z",
});
- expect(store.clearHeartbeat("active-1", new Date("2026-01-01T12:36:00.000Z"))).toMatchObject({
+ expect(await store.clearHeartbeat("active-1", new Date("2026-01-01T12:36:00.000Z"))).toMatchObject({
id: job.id,
status: "cancelled",
});
@@ -537,10 +508,10 @@ describe("AgentCronJobStore", () => {
expect(store.getLatestHeartbeat("active-1")).toMatchObject({ id: job.id, status: "cancelled" });
});
- it("rejects one-shot heartbeat schedules", () => {
+ it("rejects one-shot heartbeat schedules", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- expect(() =>
+ await expect(
store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
@@ -550,12 +521,12 @@ describe("AgentCronJobStore", () => {
prompt: "check on me",
now: start,
}),
- ).toThrow("Heartbeat schedule must be recurring");
+ ).rejects.toThrow("Heartbeat schedule must be recurring");
});
- it("rebinds persisted session jobs to a new daemon active session id", () => {
+ it("rebinds persisted session jobs to a new daemon active session id", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const userHeartbeat = store.createHeartbeat({
+ const userHeartbeat = await store.createHeartbeat({
activeSessionId: "old-active",
sessionId: "old-session",
sessionFile: "/tmp/session.jsonl",
@@ -564,7 +535,7 @@ describe("AgentCronJobStore", () => {
prompt: "check on the user",
now: start,
});
- const rlmHeartbeat = store.createRlmHeartbeat({
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "old-active",
sessionId: "old-session",
sessionFile: "/tmp/session.jsonl",
@@ -574,7 +545,7 @@ describe("AgentCronJobStore", () => {
prompt: "review the latest output",
now: start,
});
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: "other-active",
sessionId: "other-session",
sessionFile: "/tmp/other-session.jsonl",
@@ -584,7 +555,7 @@ describe("AgentCronJobStore", () => {
now: start,
});
- const rebound = store.rebindSessionJobs({
+ const rebound = await store.rebindSessionJobs({
activeSessionId: "new-active",
sessionId: "new-session",
sessionFile: "/tmp/session.jsonl",
@@ -606,9 +577,9 @@ describe("AgentCronJobStore", () => {
expect(store.getHeartbeat("other-active")).toMatchObject({ prompt: "check on a different session" });
});
- it("moves live session jobs to a replacement session file", () => {
+ it("moves live session jobs to a replacement session file", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const cronJob = store.create({
+ const cronJob = await store.create({
activeSessionId: "active-1",
sessionId: "old-session",
sessionFile: "/tmp/old-session.jsonl",
@@ -617,7 +588,7 @@ describe("AgentCronJobStore", () => {
prompt: "continue the audit",
now: start,
});
- const userHeartbeat = store.createHeartbeat({
+ const userHeartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "old-session",
sessionFile: "/tmp/old-session.jsonl",
@@ -626,7 +597,7 @@ describe("AgentCronJobStore", () => {
prompt: "check on the user",
now: start,
});
- const rlmHeartbeat = store.createRlmHeartbeat({
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "active-1",
sessionId: "old-session",
sessionFile: "/tmp/old-session.jsonl",
@@ -637,7 +608,7 @@ describe("AgentCronJobStore", () => {
now: start,
});
- const rebound = store.rebindSessionJobs({
+ const rebound = await store.rebindSessionJobs({
activeSessionId: "active-1",
sessionId: "new-session",
sessionFile: "/tmp/new-session.jsonl",
@@ -668,9 +639,9 @@ describe("AgentCronJobStore", () => {
);
});
- it("keeps multiple RLM heartbeats separate from the single user heartbeat", () => {
+ it("keeps multiple RLM heartbeats separate from the single user heartbeat", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const userHeartbeat = store.createHeartbeat({
+ const userHeartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -679,7 +650,7 @@ describe("AgentCronJobStore", () => {
prompt: "check on the user",
now: start,
});
- const firstRlmHeartbeat = store.createRlmHeartbeat({
+ const firstRlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "rlm-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -689,7 +660,7 @@ describe("AgentCronJobStore", () => {
prompt: "rerun focused tests",
now: start,
});
- const secondRlmHeartbeat = store.createRlmHeartbeat({
+ const secondRlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "rlm-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -711,9 +682,9 @@ describe("AgentCronJobStore", () => {
expect(store.getHeartbeat("active-1")).toMatchObject({ id: userHeartbeat.id, status: "active" });
});
- it("cancels active RLM heartbeats for a released session", () => {
+ it("cancels active RLM heartbeats for a released session", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const active = store.createRlmHeartbeat({
+ const active = await store.createRlmHeartbeat({
activeSessionId: "subagent-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -724,7 +695,7 @@ describe("AgentCronJobStore", () => {
prompt: "continue active work",
now: start,
});
- const paused = store.createRlmHeartbeat({
+ const paused = await store.createRlmHeartbeat({
activeSessionId: "subagent-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -735,8 +706,8 @@ describe("AgentCronJobStore", () => {
prompt: "continue paused work",
now: start,
});
- store.updateRlmHeartbeat("subagent-1", paused.id, { status: "pause", now: start });
- store.createRlmHeartbeat({
+ await store.updateRlmHeartbeat("subagent-1", paused.id, { status: "pause", now: start });
+ await store.createRlmHeartbeat({
activeSessionId: "top-level-1",
sessionId: "top-level-session",
sessionFile: "/tmp/top-level.jsonl",
@@ -748,7 +719,7 @@ describe("AgentCronJobStore", () => {
now: start,
});
- const cancelled = store.cancelRlmHeartbeatsForSession("subagent-1", new Date("2026-01-01T12:40:00.000Z"));
+ const cancelled = await store.cancelRlmHeartbeatsForSession("subagent-1", new Date("2026-01-01T12:40:00.000Z"));
expect(cancelled.map((job) => job.id)).toEqual(expect.arrayContaining([active.id, paused.id]));
expect(store.listRlmHeartbeats("subagent-1")).toEqual([]);
@@ -761,10 +732,10 @@ describe("AgentCronJobStore", () => {
expect(store.listRlmHeartbeats("top-level-1")[0]).toMatchObject({ status: "active" });
});
- it("cancels active and paused jobs for a removed session", () => {
+ it("cancels active and paused jobs for a removed session", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
const sessionFile = "/tmp/session-to-remove.jsonl";
- const cron = store.create({
+ const cron = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile,
@@ -773,7 +744,7 @@ describe("AgentCronJobStore", () => {
prompt: "check the long run",
now: start,
});
- const heartbeat = store.createHeartbeat({
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile,
@@ -782,8 +753,8 @@ describe("AgentCronJobStore", () => {
prompt: "continue the session",
now: start,
});
- store.pauseHeartbeat("active-1", new Date("2026-01-01T12:35:00.000Z"));
- const rlmHeartbeat = store.createRlmHeartbeat({
+ await store.pauseHeartbeat("active-1", new Date("2026-01-01T12:35:00.000Z"));
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile,
@@ -794,7 +765,7 @@ describe("AgentCronJobStore", () => {
prompt: "continue internal work",
now: start,
});
- const unrelated = store.create({
+ const unrelated = await store.create({
activeSessionId: "active-2",
sessionId: "session-2",
sessionFile: "/tmp/other-session.jsonl",
@@ -804,7 +775,10 @@ describe("AgentCronJobStore", () => {
now: start,
});
- const cancelled = store.cancelJobsForSession({ sessionId: "session-1" }, new Date("2026-01-01T12:40:00.000Z"));
+ const cancelled = await store.cancelJobsForSession(
+ { sessionId: "session-1" },
+ new Date("2026-01-01T12:40:00.000Z"),
+ );
expect(cancelled.map((job) => job.id)).toEqual(expect.arrayContaining([cron.id, heartbeat.id, rlmHeartbeat.id]));
for (const id of [cron.id, heartbeat.id, rlmHeartbeat.id]) {
@@ -817,9 +791,9 @@ describe("AgentCronJobStore", () => {
expect(store.list().find((job) => job.id === unrelated.id)).toMatchObject({ status: "active" });
});
- it("returns undefined when updating inactive RLM heartbeats", () => {
+ it("returns undefined when updating inactive RLM heartbeats", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const rlmHeartbeat = store.createRlmHeartbeat({
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "rlm-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -829,10 +803,10 @@ describe("AgentCronJobStore", () => {
prompt: "rerun focused tests",
now: start,
});
- store.deleteRlmHeartbeat("rlm-1", rlmHeartbeat.id, new Date("2026-01-01T12:35:00.000Z"));
+ await store.deleteRlmHeartbeat("rlm-1", rlmHeartbeat.id, new Date("2026-01-01T12:35:00.000Z"));
expect(
- store.updateRlmHeartbeat("rlm-1", rlmHeartbeat.id, {
+ await store.updateRlmHeartbeat("rlm-1", rlmHeartbeat.id, {
prompt: "try to update cancelled heartbeat",
now: new Date("2026-01-01T12:36:00.000Z"),
}),
@@ -844,9 +818,9 @@ describe("AgentCronJobStore", () => {
});
});
- it("updates and deletes only RLM heartbeats in the matching RLM session", () => {
+ it("updates and deletes only RLM heartbeats in the matching RLM session", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const userHeartbeat = store.createHeartbeat({
+ const userHeartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -855,7 +829,7 @@ describe("AgentCronJobStore", () => {
prompt: "check on the user",
now: start,
});
- const rlmHeartbeat = store.createRlmHeartbeat({
+ const rlmHeartbeat = await store.createRlmHeartbeat({
activeSessionId: "rlm-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -867,19 +841,19 @@ describe("AgentCronJobStore", () => {
});
expect(
- store.updateRlmHeartbeat("active-1", userHeartbeat.id, {
+ await store.updateRlmHeartbeat("active-1", userHeartbeat.id, {
prompt: "try to mutate user heartbeat",
now: new Date("2026-01-01T12:35:00.000Z"),
}),
).toBeUndefined();
expect(
- store.updateRlmHeartbeat("rlm-2", rlmHeartbeat.id, {
+ await store.updateRlmHeartbeat("rlm-2", rlmHeartbeat.id, {
prompt: "try wrong RLM session",
now: new Date("2026-01-01T12:35:00.000Z"),
}),
).toBeUndefined();
- const updated = store.updateRlmHeartbeat("rlm-1", rlmHeartbeat.id, {
+ const updated = await store.updateRlmHeartbeat("rlm-1", rlmHeartbeat.id, {
label: "focused-tests",
prompt: "rerun focused tests and inspect failures",
scheduleText: "every 10m",
@@ -901,8 +875,10 @@ describe("AgentCronJobStore", () => {
status: "active",
});
- expect(store.deleteRlmHeartbeat("active-1", userHeartbeat.id)).toBeUndefined();
- expect(store.deleteRlmHeartbeat("rlm-1", rlmHeartbeat.id, new Date("2026-01-01T12:36:00.000Z"))).toMatchObject({
+ expect(await store.deleteRlmHeartbeat("active-1", userHeartbeat.id)).toBeUndefined();
+ expect(
+ await store.deleteRlmHeartbeat("rlm-1", rlmHeartbeat.id, new Date("2026-01-01T12:36:00.000Z")),
+ ).toMatchObject({
id: rlmHeartbeat.id,
status: "cancelled",
});
@@ -925,7 +901,7 @@ describe("AgentCronScheduler", () => {
it("runs due one-shot jobs and marks them completed", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -957,7 +933,7 @@ describe("AgentCronScheduler", () => {
it("does not claim new jobs after a started scheduler is stopped", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const job = store.create({
+ const job = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -978,7 +954,7 @@ describe("AgentCronScheduler", () => {
it("releases leases and recovers the whole claimed batch when setup fails", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const unrelated = store.create({
+ const unrelated = await store.create({
activeSessionId: "unrelated",
sessionId: "unrelated",
sessionFile: "/tmp/unrelated.jsonl",
@@ -987,9 +963,9 @@ describe("AgentCronScheduler", () => {
prompt: "unrelated",
now: start,
});
- const [unrelatedDispatch] = store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ const [unrelatedDispatch] = await store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
if (!unrelatedDispatch) throw new Error("Expected unrelated dispatch");
- const batch = ["active-1", "active-2", "active-3"].map((activeSessionId) =>
+ const createBatchJob = (activeSessionId: string) =>
store.create({
activeSessionId,
sessionId: activeSessionId,
@@ -998,8 +974,12 @@ describe("AgentCronScheduler", () => {
scheduleText: "in 1m",
prompt: activeSessionId,
now: start,
- }),
- );
+ });
+ const batch = [
+ await createBatchJob("active-1"),
+ await createBatchJob("active-2"),
+ await createBatchJob("active-3"),
+ ];
const endDispatch = vi.fn();
const beginDispatch = vi.fn((dispatch) => {
if (dispatch.job.activeSessionId === "active-2") throw new Error("lease setup failed");
@@ -1022,14 +1002,14 @@ describe("AgentCronScheduler", () => {
});
}
expect(store.getClaimedJob(unrelated.id)).toMatchObject({ id: unrelated.id });
- expect(store.recordDispatchResult(unrelatedDispatch.id, { outcome: "ran" })).toMatchObject({
+ expect(await store.recordDispatchResult(unrelatedDispatch.id, { outcome: "ran" })).toMatchObject({
id: unrelated.id,
});
});
it("reschedules recurring jobs after each run", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1055,7 +1035,7 @@ describe("AgentCronScheduler", () => {
it("reschedules interval heartbeats after each run", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1081,7 +1061,7 @@ describe("AgentCronScheduler", () => {
it("does not run an RLM heartbeat that was deleted before it became due", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const job = store.createRlmHeartbeat({
+ const job = await store.createRlmHeartbeat({
activeSessionId: "rlm-1",
sessionId: "session-rlm-1",
sessionFile: "/tmp/session-rlm.jsonl",
@@ -1091,7 +1071,7 @@ describe("AgentCronScheduler", () => {
prompt: "this should never run",
now: start,
});
- store.deleteRlmHeartbeat("rlm-1", job.id, new Date("2026-01-01T12:34:10.000Z"));
+ await store.deleteRlmHeartbeat("rlm-1", job.id, new Date("2026-01-01T12:34:10.000Z"));
const prompts: string[] = [];
const scheduler = new AgentCronScheduler(store, {
now: () => new Date("2026-01-01T12:34:31.000Z"),
@@ -1115,7 +1095,7 @@ describe("AgentCronScheduler", () => {
it("reschedules skipped jobs without recording a run", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const job = store.createHeartbeat({
+ const job = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1144,7 +1124,7 @@ describe("AgentCronScheduler", () => {
it("skips jobs cancelled while earlier due jobs are running", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const first = store.create({
+ const first = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1153,7 +1133,7 @@ describe("AgentCronScheduler", () => {
prompt: "first",
now: start,
});
- const second = store.create({
+ const second = await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1168,7 +1148,7 @@ describe("AgentCronScheduler", () => {
runJob: async (dueJob) => {
prompts.push(dueJob.prompt);
if (dueJob.id === first.id) {
- store.cancel(second.id, new Date("2026-01-01T12:35:00.000Z"));
+ await store.cancel(second.id, new Date("2026-01-01T12:35:00.000Z"));
}
return undefined;
},
@@ -1189,7 +1169,7 @@ describe("AgentCronScheduler", () => {
it("dispatches different sessions concurrently and advances their schedules before completion", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
for (const index of [1, 2]) {
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: `active-${index}`,
sessionId: `session-${index}`,
sessionFile: `/tmp/session-${index}.jsonl`,
@@ -1201,17 +1181,23 @@ describe("AgentCronScheduler", () => {
}
const started: string[] = [];
const releases = new Map<string, () => void>();
+ const startedSignals = new Map<string, () => void>();
+ const whenBothStarted = Promise.all(
+ ["active-1", "active-2"].map((id) => new Promise<void>((resolve) => startedSignals.set(id, resolve))),
+ );
const scheduler = new AgentCronScheduler(store, {
now: () => new Date("2026-01-01T12:34:10.000Z"),
runJob: async (job) => {
started.push(job.activeSessionId);
+ startedSignals.get(job.activeSessionId)?.();
await new Promise<void>((resolve) => releases.set(job.activeSessionId, resolve));
return undefined;
},
});
const run = scheduler.runDue(new Date("2026-01-01T12:34:10.000Z"));
- await new Promise<void>((resolve) => setImmediate(resolve));
+ // The async lock spans a few I/O ticks; wait for both runJob invocations.
+ await whenBothStarted;
expect(started).toEqual(expect.arrayContaining(["active-1", "active-2"]));
expect(store.list().map((job) => job.nextRunAt)).toEqual([
@@ -1227,7 +1213,7 @@ describe("AgentCronScheduler", () => {
it("serializes simultaneous jobs that target the same session", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
for (const prompt of ["first", "second"]) {
- store.create({
+ await store.create({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1239,11 +1225,13 @@ describe("AgentCronScheduler", () => {
}
const started: string[] = [];
let releaseFirst: () => void = () => {};
+ let firstStarted: () => void = () => {};
const scheduler = new AgentCronScheduler(store, {
now: () => new Date("2026-01-01T12:35:00.000Z"),
runJob: async (job) => {
started.push(job.prompt);
if (job.prompt === "first") {
+ firstStarted();
await new Promise<void>((resolve) => {
releaseFirst = resolve;
});
@@ -1253,16 +1241,19 @@ describe("AgentCronScheduler", () => {
});
const run = scheduler.runDue(new Date("2026-01-01T12:35:00.000Z"));
- await new Promise<void>((resolve) => setImmediate(resolve));
+ // The async lock spans a few I/O ticks; wait for the first runJob invocation.
+ await new Promise<void>((resolve) => {
+ firstStarted = resolve;
+ });
expect(started).toEqual(["first"]);
releaseFirst();
await run;
expect(started).toEqual(["first", "second"]);
});
- it("coalesces a missed interval while its previous dispatch is still active", () => {
+ it("coalesces a missed interval while its previous dispatch is still active", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const heartbeat = store.createHeartbeat({
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1272,8 +1263,8 @@ describe("AgentCronScheduler", () => {
now: start,
});
- expect(store.claimDue(new Date("2026-01-01T12:34:10.000Z"))).toHaveLength(1);
- expect(store.claimDue(new Date("2026-01-01T12:34:20.000Z"))).toEqual([]);
+ expect(await store.claimDue(new Date("2026-01-01T12:34:10.000Z"))).toHaveLength(1);
+ expect(await store.claimDue(new Date("2026-01-01T12:34:20.000Z"))).toEqual([]);
expect(store.list().find((job) => job.id === heartbeat.id)).toMatchObject({
nextRunAt: "2026-01-01T12:34:30.000Z",
lastSkippedAt: "2026-01-01T12:34:20.000Z",
@@ -1281,9 +1272,9 @@ describe("AgentCronScheduler", () => {
});
});
- it("reschedules a skipped dispatch from the skip time", () => {
+ it("reschedules a skipped dispatch from the skip time", async () => {
const store = new AgentCronJobStore(makeStorePath(tempDirs));
- const heartbeat = store.createHeartbeat({
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1292,12 +1283,12 @@ describe("AgentCronScheduler", () => {
prompt: "check progress",
now: start,
});
- const [dispatch] = store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ const [dispatch] = await store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
if (!dispatch) {
throw new Error("Expected heartbeat dispatch");
}
- store.recordDispatchResult(dispatch.id, {
+ await store.recordDispatchResult(dispatch.id, {
now: new Date("2026-01-01T12:34:17.000Z"),
outcome: "skipped",
});
@@ -1309,10 +1300,10 @@ describe("AgentCronScheduler", () => {
});
});
- it("does not replay an uncertain claimed dispatch after recovery", () => {
+ it("does not replay an uncertain claimed dispatch after recovery", async () => {
const storePath = makeStorePath(tempDirs);
const store = new AgentCronJobStore(storePath);
- const heartbeat = store.createHeartbeat({
+ const heartbeat = await store.createHeartbeat({
activeSessionId: "active-1",
sessionId: "session-1",
sessionFile: "/tmp/session.jsonl",
@@ -1321,15 +1312,45 @@ describe("AgentCronScheduler", () => {
prompt: "check progress",
now: start,
});
- store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ await store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
const recovered = new AgentCronJobStore(storePath);
- expect(recovered.recoverInterruptedDispatches(new Date("2026-01-01T12:34:11.000Z"))).toEqual([
+ expect(await recovered.recoverInterruptedDispatches(new Date("2026-01-01T12:34:11.000Z"))).toEqual([
expect.objectContaining({ id: heartbeat.id, lastError: "Interrupted before scheduled operation completion" }),
]);
expect(recovered.getClaimedJob(heartbeat.id)).toBeUndefined();
expect(recovered.getDueJob(heartbeat.id, new Date("2026-01-01T12:34:11.000Z"))).toBeUndefined();
});
+
+ it("retries a failed startup recovery before the next claim instead of skipping the stale dispatch", async () => {
+ vi.useFakeTimers();
+ try {
+ const store = new AgentCronJobStore(makeStorePath(tempDirs));
+ const heartbeat = await store.createHeartbeat({
+ activeSessionId: "active-1",
+ sessionId: "session-1",
+ sessionFile: "/tmp/session.jsonl",
+ cwd: "/tmp/project",
+ scheduleText: "every 10s",
+ prompt: "check progress",
+ now: start,
+ });
+ await store.claimDue(new Date("2026-01-01T12:34:10.000Z"));
+ vi.spyOn(store, "recoverInterruptedDispatches").mockRejectedValueOnce(new Error("store busy"));
+ const runJob = vi.fn(async () => undefined);
+ const scheduler = new AgentCronScheduler(store, { now: () => new Date("2026-01-01T12:34:30.000Z"), runJob });
+ scheduler.start();
+
+ await expect(scheduler.runDue(new Date("2026-01-01T12:34:30.000Z"))).rejects.toThrow("store busy");
+ expect(store.getClaimedJob(heartbeat.id)).toBeDefined();
+
+ expect(await scheduler.runDue(new Date("2026-01-01T12:34:30.000Z"))).toBe(1);
+ expect(runJob).toHaveBeenCalledOnce();
+ scheduler.stop();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
});
describe("shouldDeferHeartbeatCronJob", () => {
@@ -1482,14 +1503,6 @@ describe("heartbeat delivery mode", () => {
});
});
-function writeJobsForTest(store: AgentCronJobStore, jobs: readonly AgentCronJob[]): void {
- (
- store as unknown as {
- writeJobs(jobs: readonly AgentCronJob[]): void;
- }
- ).writeJobs(jobs);
-}
-
function makeStorePath(tempDirs: string[]): string {
const dir = makeTempDir(tempDirs);
return join(dir, "cron-jobs.json");
@@ -1545,7 +1558,7 @@ describe("AgentCronScheduler session resurrection guard (ENG-4519)", () => {
createRuntime,
});
const internals = daemon as unknown as CronDaemonInternals;
- const job = internals.cronStore.createHeartbeat({
+ const job = await internals.cronStore.createHeartbeat({
activeSessionId: "old-active",
sessionId: options.sessionId(harness),
sessionFile: options.sessionFile(harness),
@@ -1563,7 +1576,7 @@ describe("AgentCronScheduler session resurrection guard (ENG-4519)", () => {
sessionFile: (h) => h.session.sessionFile!,
persistStates: ["active", "archived"],
});
- const pausedHeartbeat = internals.cronStore.createRlmHeartbeat({
+ const pausedHeartbeat = await internals.cronStore.createRlmHeartbeat({
activeSessionId: "old-active",
sessionId: harness.session.sessionId,
sessionFile: harness.session.sessionFile!,
@@ -1572,7 +1585,7 @@ describe("AgentCronScheduler session resurrection guard (ENG-4519)", () => {
prompt: "check internal work",
now: new Date(),
});
- internals.cronStore.updateRlmHeartbeat("old-active", pausedHeartbeat.id, { status: "pause" });
+ await internals.cronStore.updateRlmHeartbeat("old-active", pausedHeartbeat.id, { status: "pause" });
await expect(internals.cronScheduler.runDue(new Date())).resolves.toBe(0);
@@ -1598,3 +1611,77 @@ describe("AgentCronScheduler session resurrection guard (ENG-4519)", () => {
expect(internals.cronStore.list().find((job) => job.id === jobId)).toMatchObject({ status: "cancelled" });
});
});
+
+describe("AgentCronJobStore mutation locks", () => {
+ const tempDirs: string[] = [];
+
+ afterEach(() => {
+ for (const dir of tempDirs.splice(0)) {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ const holdLock = (path: string) => lock(path, { realpath: false, lockfilePath: `${path}.lock`, stale: 30_000 });
+
+ it("locks only the path a mutation writes", async () => {
+ const root = makeTempDir(tempDirs);
+ const store = AgentCronJobStore.forSessionArtifacts();
+ const seed = async (sessionId: string) => {
+ mkdirSync(join(root, "artifacts", sessionId), { recursive: true });
+ store.registerSessionArtifact(sessionId, join(root, "artifacts", sessionId));
+ return store.create({
+ activeSessionId: `active-${sessionId.split("-")[1]}`,
+ sessionId,
+ sessionFile: `/tmp/${sessionId}.jsonl`,
+ cwd: "/tmp/project",
+ scheduleText: "every 5m",
+ prompt: sessionId,
+ });
+ };
+ const first = await seed("session-1");
+ await Promise.all([seed("session-2"), seed("session-3")]);
+ const held = await Promise.all(
+ ["session-2", "session-3"].map((sessionId) =>
+ holdLock(join(root, "artifacts", sessionId, SESSION_SCHEDULED_JOBS_FILENAME)),
+ ),
+ );
+ try {
+ await store.cancel(first.id);
+ expect(store.list().find((job) => job.id === first.id)?.status).toBe("cancelled");
+ expect(store.list().find((job) => job.id !== first.id)?.status).toBe("active");
+ } finally {
+ await Promise.all(held.map((release) => release()));
+ }
+ });
+
+ it("retries a contended lock without blocking the event loop and fsyncs once per write", async () => {
+ vi.useFakeTimers();
+ try {
+ const root = makeTempDir(tempDirs);
+ const store = new AgentCronJobStore(join(root, "cron-jobs.json"));
+ const writeFileAtomicSync = vi.spyOn(atomicFile, "writeFileAtomicSync");
+ const job = await store.create({
+ activeSessionId: "active-1",
+ sessionId: "session-1",
+ sessionFile: "/tmp/session.jsonl",
+ cwd: "/tmp/project",
+ scheduleText: "every 5m",
+ prompt: "contended",
+ });
+ const held = await holdLock(join(root, "cron-jobs.json"));
+ let settled = false;
+ const cancelling = store.cancel(job.id).finally(() => (settled = true));
+ await held();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(settled).toBe(false);
+ await vi.advanceTimersByTimeAsync(50);
+ await expect(cancelling).resolves.toMatchObject({ id: job.id, status: "cancelled" });
+ for (const options of writeFileAtomicSync.mock.calls.map((call) => call[2])) {
+ expect(options).toMatchObject({ fsync: true, mode: 0o600 });
+ expect(options).not.toHaveProperty("fsyncDir");
+ }
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/packages/coding-agent/test/daemon-mode.test.ts b/packages/coding-agent/test/daemon-mode.test.ts
index 0a4d18f29c..9f3d6239bd 100644
--- a/packages/coding-agent/test/daemon-mode.test.ts
+++ b/packages/coding-agent/test/daemon-mode.test.ts
@@ -38,6 +38,7 @@ import {
} from "../src/core/session-manager.js";
import { SettingsManager } from "../src/core/settings-manager.js";
import type { ActiveSessionState, DaemonSocketClient } from "../src/modes/daemon/active-session-state.js";
+import { bindActiveSessionState } from "../src/modes/daemon/daemon-extension-binding.js";
import {
AgentDaemon,
finishClientSnapshotStreaming,
@@ -284,7 +285,7 @@ describe("daemon mode helpers", () => {
).releaseRlmChildSession = vi.fn(() => vi.fn());
const passivation = internals.passivateIdleChildren(90, Date.parse("2036-08-01T12:00:00Z"), 1);
await disposeStarted;
- const job = internals.cronStore.create({
+ const job = await internals.cronStore.create({
activeSessionId: childState.activeSessionId,
sessionId: childState.runtime.session.sessionId,
sessionFile: fixture.childSessionFile,
@@ -1528,8 +1529,8 @@ describe("daemon mode helpers", () => {
grandchildSessionId,
getSessionArtifactPathForFile(fixture.grandchildSessionFile, grandchildSessionId),
);
- const makeJob = (sessionId: string, sessionFile: string) =>
- seedStore.createRlmHeartbeat({
+ const makeJob = async (sessionId: string, sessionFile: string) =>
+ await seedStore.createRlmHeartbeat({
activeSessionId: `stale-${sessionId}`,
sessionId,
sessionFile,
@@ -1538,8 +1539,8 @@ describe("daemon mode helpers", () => {
scheduleText: "every 30s",
prompt: "report exactly: hi",
});
- makeJob(childSessionId, fixture.childSessionFile);
- const grandchildHeartbeat = makeJob(grandchildSessionId, fixture.grandchildSessionFile);
+ await makeJob(childSessionId, fixture.childSessionFile);
+ const grandchildHeartbeat = await makeJob(grandchildSessionId, fixture.grandchildSessionFile);
const workerDaemon = new AgentDaemon(join(tempDir, "worker-daemon.sock"), {
defaultSessionConfig: { agentDir: tempDir, cwd: tempDir, sessionDir: join(tempDir, "sessions") },
@@ -2923,7 +2924,7 @@ describe("daemon mode helpers", () => {
// A retried delete of the now-tombstoned child still resolves the
// session path and cancels its scheduled jobs.
const cronStore = (fixture.daemon as unknown as { cronStore: AgentCronJobStore }).cronStore;
- const retryJob = cronStore.create({
+ const retryJob = await cronStore.create({
activeSessionId: "gone",
sessionId: "gone",
sessionFile: fixture.childSessionFile,
@@ -3118,7 +3119,7 @@ describe("daemon mode helpers", () => {
rmSync(join(tempDir, "rlm-ledger"), { recursive: true, force: true });
(fixture.daemon as unknown as { rlmSpawnLedgerInstance?: unknown }).rlmSpawnLedgerInstance =
new RlmSpawnLedger(tempDir, join(tempDir, "sessions"));
- const job = internals.cronStore.create({
+ const job = await internals.cronStore.create({
activeSessionId: "gone",
sessionId: "gone",
sessionFile: fixture.childSessionFile,
@@ -4180,3 +4181,45 @@ describe("daemon snapshot transfers", () => {
socket.destroy();
});
});
+
+describe("session replacement binding", () => {
+ it("broadcasts session_replaced only after the replaced bookkeeping settles, surfacing its failures", async () => {
+ const setRebindSession = vi.fn();
+ const session = {
+ messages: [],
+ setExecEnvProvider: vi.fn(),
+ subscribe: () => () => {},
+ bindExtensions: vi.fn(async () => {}),
+ };
+ const broadcasts: string[] = [];
+ const bindWith = (sessionReplaced: (state: object) => void | Promise<void>) =>
+ bindActiveSessionState(
+ {
+ runtime: { session, setRuntimeEnvScope: vi.fn(), setSubagentRuntimeHost: vi.fn(), setRebindSession },
+ } as never,
+ {
+ broadcast: (_state, message) => void broadcasts.push(message.type),
+ sessionReplaced,
+ shutdown: () => {},
+ createConnectionState: (() => ({})) as never,
+ },
+ );
+ // Always invoke the latest registration: bindActiveSessionState re-registers per replacement.
+ const rebind = () => setRebindSession.mock.calls.at(-1)![0] as (session: unknown) => Promise<void>;
+
+ let releaseReplaced: (() => void) | undefined;
+ await bindWith(() => new Promise<void>((resolve) => (releaseReplaced = resolve)));
+ const settling = rebind()(session);
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(broadcasts).toEqual([]);
+ releaseReplaced!();
+ await settling;
+ expect(broadcasts).toEqual(["session_replaced"]);
+
+ await bindWith(() => {
+ throw new Error("cron store contention");
+ });
+ await expect(rebind()(session)).rejects.toThrow("cron store contention");
+ expect(broadcasts).toEqual(["session_replaced"]);
+ });
+});
diff --git a/packages/coding-agent/test/daemon-supervisor-eviction.test.ts b/packages/coding-agent/test/daemon-supervisor-eviction.test.ts
index 212d0b428b..71b034734c 100644
--- a/packages/coding-agent/test/daemon-supervisor-eviction.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-eviction.test.ts
@@ -1,6 +1,7 @@
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { lock } from "proper-lockfile";
import { afterEach, describe, expect, it, vi } from "vitest";
import { getSessionsDir } from "../src/config.js";
import { type AgentCronJob, AgentCronJobStore, SESSION_SCHEDULED_JOBS_FILENAME } from "../src/core/cron-jobs.js";
@@ -58,6 +59,8 @@ interface SupervisorInternals {
log: ReturnType<typeof vi.fn>;
scheduledWakeTimer?: ReturnType<typeof setTimeout>;
scheduledWakeRecompute?: Promise<void>;
+ onWorkerResidencyGained(worker: object): void;
+ broadcastHeartbeatsChanged(): void;
scheduleIdleEvictionSweep(): void;
runIdleEvictionSweep(now?: number): Promise<void>;
recomputeScheduledSessionWake(): Promise<void>;
@@ -84,6 +87,10 @@ afterEach(() => {
for (const directory of tempDirs.splice(0)) rmSync(directory, { recursive: true, force: true });
});
+async function settleAsyncWork(): Promise<void> {
+ await new Promise((resolve) => setImmediate(resolve));
+}
+
function makeSummary(id: string, now: number, overrides: Partial<SessionSummary> = {}): SessionSummary {
return {
id,
@@ -774,8 +781,13 @@ describe("daemon supervisor scheduled-session wake", () => {
return { sessionFile, store };
}
- function armHeartbeat(store: AgentCronJobStore, sessionId: string, sessionFile: string, at: number): AgentCronJob {
- return store.createHeartbeat({
+ async function armHeartbeat(
+ store: AgentCronJobStore,
+ sessionId: string,
+ sessionFile: string,
+ at: number,
+ ): Promise<AgentCronJob> {
+ return await store.createHeartbeat({
activeSessionId: "stale-active",
sessionId,
sessionFile,
@@ -790,7 +802,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
const root = makeScheduledSessionFile("wake-root");
const child = makeScheduledSessionFile("wake-child");
- armHeartbeat(child.store, "wake-child", child.sessionFile, now - 10 * 60_000);
+ await armHeartbeat(child.store, "wake-child", child.sessionFile, now - 10 * 60_000);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [
makeSavedInfo(root.sessionFile, "wake-root"),
@@ -821,7 +833,7 @@ describe("daemon supervisor scheduled-session wake", () => {
supervisor.createOrReuseWorker = vi.fn();
const root = makeScheduledSessionFile("covered-mid-root");
const mid = makeScheduledSessionFile("covered-mid");
- armHeartbeat(mid.store, "covered-mid", mid.sessionFile, now - 10 * 60_000);
+ await armHeartbeat(mid.store, "covered-mid", mid.sessionFile, now - 10 * 60_000);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [
makeSavedInfo(root.sessionFile, "covered-mid-root"),
@@ -844,8 +856,8 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
const healthy = makeScheduledSessionFile("healthy-root");
const corrupt = makeScheduledSessionFile("corrupt-root");
- armHeartbeat(healthy.store, "healthy-root", healthy.sessionFile, now - 10 * 60_000);
- armHeartbeat(corrupt.store, "corrupt-root", corrupt.sessionFile, now - 10 * 60_000);
+ await armHeartbeat(healthy.store, "healthy-root", healthy.sessionFile, now - 10 * 60_000);
+ await armHeartbeat(corrupt.store, "corrupt-root", corrupt.sessionFile, now - 10 * 60_000);
writeFileSync(
join(getSessionArtifactPathForFile(corrupt.sessionFile, "corrupt-root"), SESSION_SCHEDULED_JOBS_FILENAME),
"{ not json",
@@ -874,7 +886,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const { sessionFile, store } = makeScheduledSessionFile("armed-root");
// Real-clock epochs keep nextRunAt in the future so the armed timer never fires mid-test.
const armedAt = Date.now();
- const job = armHeartbeat(store, "armed-root", sessionFile, armedAt);
+ const job = await armHeartbeat(store, "armed-root", sessionFile, armedAt);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "armed-root")]),
liveEdges: vi.fn(async () => []),
@@ -883,27 +895,34 @@ describe("daemon supervisor scheduled-session wake", () => {
await supervisor.recomputeScheduledSessionWake();
expect(supervisor.scheduledWakeTimer).toBeDefined();
+ // A covering worker drops the covered rows in memory: the residency event, not a scan, converges the timer.
const resident = makeWorker("resident", []);
resident.descriptor.sessionFile = sessionFile;
supervisor.workers.set("resident", resident);
- await supervisor.recomputeScheduledSessionWake();
+ supervisor.onWorkerResidencyGained(resident);
+ await settleAsyncWork();
+ await supervisor.scheduledWakeRecompute;
expect(supervisor.scheduledWakeTimer).toBeUndefined();
supervisor.workers.delete("resident");
- store.pauseHeartbeat("stale-active");
- await supervisor.recomputeScheduledSessionWake();
+ // The store mutations below are external writes; signal each with the invalidating broadcast.
+ await store.pauseHeartbeat("stale-active");
+ supervisor.broadcastHeartbeatsChanged();
+ await supervisor.scheduledWakeRecompute;
expect(supervisor.scheduledWakeTimer).toBeUndefined();
await supervisor.wakeDueScheduledSessions(armedAt + 60 * 60_000);
expect(supervisor.createOrReuseWorker).not.toHaveBeenCalled();
// Settle the recompute the wake pass queued before mutating the store again.
await supervisor.scheduledWakeRecompute;
- store.resumeHeartbeat("stale-active");
- await supervisor.recomputeScheduledSessionWake();
+ await store.resumeHeartbeat("stale-active");
+ supervisor.broadcastHeartbeatsChanged();
+ await supervisor.scheduledWakeRecompute;
expect(supervisor.scheduledWakeTimer).toBeDefined();
- store.cancel(job.id);
- await supervisor.recomputeScheduledSessionWake();
+ await store.cancel(job.id);
+ supervisor.broadcastHeartbeatsChanged();
+ await supervisor.scheduledWakeRecompute;
expect(supervisor.scheduledWakeTimer).toBeUndefined();
});
@@ -911,7 +930,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
supervisor.createOrReuseWorker = vi.fn();
const { sessionFile, store } = makeScheduledSessionFile("restart-root");
- armHeartbeat(store, "restart-root", sessionFile, now - 10 * 60_000);
+ await armHeartbeat(store, "restart-root", sessionFile, now - 10 * 60_000);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "restart-root")]),
liveEdges: vi.fn(async () => []),
@@ -938,8 +957,8 @@ describe("daemon supervisor scheduled-session wake", () => {
const root = makeScheduledSessionFile("owned-root");
// The child's persisted id differs from its filename; the artifact keys on the id.
const child = makeScheduledSessionFile("owned-child", "owned-child-real");
- armHeartbeat(root.store, "owned-root", root.sessionFile, now);
- armHeartbeat(child.store, "owned-child-real", child.sessionFile, now);
+ await armHeartbeat(root.store, "owned-root", root.sessionFile, now);
+ await armHeartbeat(child.store, "owned-child-real", child.sessionFile, now);
const owned = makeWorker("owned", []);
owned.descriptor.ownerClientId = "owner";
owned.descriptor.rootSessionId = "owned-root";
@@ -975,7 +994,7 @@ describe("daemon supervisor scheduled-session wake", () => {
it("keeps schedules that were promoted public while the ephemeral cancel was in flight", async () => {
const supervisor = makeSupervisor();
const { sessionFile, store } = makeScheduledSessionFile("promoted-root");
- armHeartbeat(store, "promoted-root", sessionFile, now);
+ await armHeartbeat(store, "promoted-root", sessionFile, now);
let releaseFamily = () => {};
const familyGate = new Promise<void>((resolve) => {
releaseFamily = resolve;
@@ -1008,7 +1027,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
supervisor.createOrReuseWorker = vi.fn();
const { sessionFile, store } = makeScheduledSessionFile("promoted-parked-root");
- armHeartbeat(store, "promoted-parked-root", sessionFile, now - 10 * 60_000);
+ await armHeartbeat(store, "promoted-parked-root", sessionFile, now - 10 * 60_000);
let failFamily: (error: Error) => void = () => {};
const familyGate = new Promise<never>((_, reject) => {
failFamily = reject;
@@ -1052,7 +1071,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
supervisor.createOrReuseWorker = vi.fn();
const { sessionFile, store } = makeScheduledSessionFile("failed-cancel-root");
- armHeartbeat(store, "failed-cancel-root", sessionFile, now - 10 * 60_000);
+ await armHeartbeat(store, "failed-cancel-root", sessionFile, now - 10 * 60_000);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "failed-cancel-root")]),
liveEdges: vi.fn(async () => []),
@@ -1097,6 +1116,65 @@ describe("daemon supervisor scheduled-session wake", () => {
expect(supervisor.scheduledWakeTimer).toBeUndefined();
});
+ it("stops cancelling when a promotion lands mid-loop between per-session cancels", async () => {
+ const supervisor = makeSupervisor();
+ const root = makeScheduledSessionFile("promoted-mid-root");
+ const child = makeScheduledSessionFile("promoted-mid-child");
+ await armHeartbeat(root.store, "promoted-mid-root", root.sessionFile, now);
+ await armHeartbeat(child.store, "promoted-mid-child", child.sessionFile, now);
+ supervisor.rlmSpawnLedgerInstance = {
+ family: vi.fn(async () => [
+ makeSavedInfo(root.sessionFile, "promoted-mid-root"),
+ makeSavedInfo(child.sessionFile, "promoted-mid-child", {
+ parentSessionPath: root.sessionFile,
+ rlmDepth: 1,
+ }),
+ ]),
+ liveEdges: vi.fn(async () => []),
+ };
+ const originalCancel = AgentCronJobStore.prototype.cancelJobsForSession;
+ const cancel = vi.spyOn(AgentCronJobStore.prototype, "cancelJobsForSession");
+ cancel.mockImplementationOnce(async function (this: AgentCronJobStore, input, cancelNow) {
+ const cancelled = await originalCancel.call(this, input, cancelNow);
+ const resident = makeWorker("mid-loop-worker", []);
+ resident.descriptor.sessionFile = child.sessionFile;
+ supervisor.workers.set("mid-loop-worker", resident);
+ return cancelled;
+ });
+
+ await supervisor.cancelScheduledJobsForSessionTree("promoted-mid-root", root.sessionFile);
+
+ expect(cancel).toHaveBeenCalledTimes(1);
+ expect(root.store.list().map((job) => job.status)).toEqual(["cancelled"]);
+ expect(child.store.list().map((job) => job.status)).toEqual(["active"]);
+ cancel.mockRestore();
+ });
+
+ it("keeps schedules when a worker covers the tree while the cancel waits for the artifact lock", async () => {
+ const supervisor = makeSupervisor();
+ const root = makeScheduledSessionFile("locked-root");
+ await armHeartbeat(root.store, "locked-root", root.sessionFile, now);
+ supervisor.rlmSpawnLedgerInstance = {
+ family: vi.fn(async () => [makeSavedInfo(root.sessionFile, "locked-root")]),
+ liveEdges: vi.fn(async () => []),
+ };
+ const jobsPath = join(
+ getSessionArtifactPathForFile(root.sessionFile, "locked-root"),
+ SESSION_SCHEDULED_JOBS_FILENAME,
+ );
+ const release = await lock(jobsPath, { realpath: false, lockfilePath: `${jobsPath}.lock`, stale: 30_000 });
+
+ const cancel = supervisor.cancelScheduledJobsForSessionTree("locked-root", root.sessionFile);
+ await settleAsyncWork();
+ const resident = makeWorker("late-worker", []);
+ resident.descriptor.sessionFile = root.sessionFile;
+ supervisor.workers.set("late-worker", resident);
+ await release();
+ await cancel;
+
+ expect(root.store.list().map((job) => job.status)).toEqual(["active"]);
+ });
+
it("keeps the tombstoned descriptor after a failed ephemeral cancel so the next boot finishes it", async () => {
const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-durable-cancel-"));
tempDirs.push(directory);
@@ -1105,7 +1183,7 @@ describe("daemon supervisor scheduled-session wake", () => {
mkdirSync(workersDir, { recursive: true });
const socketPath = join(directory, "daemon.sock");
const { sessionFile, store } = makeScheduledSessionFile("durable-cancel-root");
- armHeartbeat(store, "durable-cancel-root", sessionFile, now - 10 * 60_000);
+ await armHeartbeat(store, "durable-cancel-root", sessionFile, now - 10 * 60_000);
const bootSupervisor = (): SupervisorInternals => {
const supervisor = new DaemonSupervisor(socketPath, {
defaultSessionConfig: { agentDir: directory, cwd: directory },
@@ -1172,7 +1250,7 @@ describe("daemon supervisor scheduled-session wake", () => {
const supervisor = makeSupervisor();
supervisor.createOrReuseWorker = vi.fn();
const { sessionFile, store } = makeScheduledSessionFile("managed-root");
- const job = armHeartbeat(store, "managed-root", sessionFile, now - 10 * 60_000);
+ const job = await armHeartbeat(store, "managed-root", sessionFile, now - 10 * 60_000);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "managed-root")]),
liveEdges: vi.fn(async () => []),
@@ -1193,7 +1271,7 @@ describe("daemon supervisor scheduled-session wake", () => {
it("lists and cancels a passive scheduled job without a resident worker", async () => {
const supervisor = makeSupervisor();
const { sessionFile, store } = makeScheduledSessionFile("unscoped-root");
- const job = armHeartbeat(store, "unscoped-root", sessionFile, now);
+ const job = await armHeartbeat(store, "unscoped-root", sessionFile, now);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "unscoped-root")]),
liveEdges: vi.fn(async () => []),
@@ -1221,7 +1299,7 @@ describe("daemon supervisor scheduled-session wake", () => {
it("drops a stale ephemeral-cancel intent once a worker covers the tree again", async () => {
const supervisor = makeSupervisor();
const { sessionFile, store } = makeScheduledSessionFile("reopened-root");
- armHeartbeat(store, "reopened-root", sessionFile, now);
+ await armHeartbeat(store, "reopened-root", sessionFile, now);
supervisor.rlmSpawnLedgerInstance = {
family: vi.fn(async () => [makeSavedInfo(sessionFile, "reopened-root")]),
liveEdges: vi.fn(async () => []),
diff --git a/packages/coding-agent/test/daemon-supervisor-heartbeats.test.ts b/packages/coding-agent/test/daemon-supervisor-heartbeats.test.ts
index 9c398f09a0..15fc51f6c9 100644
--- a/packages/coding-agent/test/daemon-supervisor-heartbeats.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-heartbeats.test.ts
@@ -1,23 +1,40 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { lock } from "proper-lockfile";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { AgentCronJobStore, SESSION_SCHEDULED_JOBS_FILENAME } from "../src/core/cron-jobs.js";
+import { type SessionInfo, SessionManager } from "../src/core/session-manager.js";
import type { DaemonSocketClient } from "../src/modes/daemon/active-session-state.js";
import { type DaemonCommand, type DaemonResponse, failure, success } from "../src/modes/daemon/daemon-protocol.js";
import {
DaemonSupervisor,
HEARTBEAT_LIST_FORWARD_TIMEOUT_MS,
HEARTBEAT_LIST_LAUNCH_WAIT_MS,
+ HEARTBEATS_CHANGED_COALESCE_MS,
+ WORKER_HEARTBEAT_SNAPSHOT_MAX_AGE_MS,
} from "../src/modes/daemon/daemon-supervisor.js";
+interface PassiveScheduledJobRow {
+ rootSessionFile: string;
+ job: { id: string; status: string; nextRunAt?: string };
+ info: SessionInfo;
+}
+
interface SupervisorHarness {
workers: Map<string, unknown>;
+ clients: Set<{ socket: { destroyed: boolean; write: ReturnType<typeof vi.fn> }; tracksHeartbeats?: boolean }>;
openingWorkers: Map<string, Promise<unknown>>;
catalogOpeningWorkers: Map<string, Promise<unknown>>;
+ passiveScheduledJobs?: { rows: PassiveScheduledJobRow[]; scannedAt: number };
findWorkerForClient(client: DaemonSocketClient, selector: string): Promise<{ worker: unknown }>;
+ attachClient(client: unknown, command: unknown): Promise<unknown>;
forwardToWorker(worker: unknown, command: DaemonCommand, timeoutMs?: number): Promise<DaemonResponse>;
handleCommand(client: DaemonSocketClient, command: DaemonCommand): Promise<DaemonResponse | undefined>;
handleWorkerFrame(worker: unknown, frame: unknown): void;
+ rlmSpawnLedger(): { family: (filename?: string) => Promise<SessionInfo[]> };
+ scheduleScheduledSessionWakeRecompute(): void;
+ onWorkerResidencyGained(worker: object): void;
}
const tempDirs: string[] = [];
@@ -28,16 +45,51 @@ afterEach(() => {
}
});
-function createSupervisorHarness(): SupervisorHarness {
- const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-heartbeats-"));
- tempDirs.push(directory);
- return new DaemonSupervisor(join(directory, "daemon.sock"), {
- defaultSessionConfig: { agentDir: directory, cwd: directory },
- descriptorDir: join(directory, "workers"),
+function createSupervisorHarness(directory?: string): SupervisorHarness {
+ const agentDir = directory ?? mkdtempSync(join(tmpdir(), "prime-supervisor-heartbeats-"));
+ if (directory === undefined) tempDirs.push(agentDir);
+ return new DaemonSupervisor(join(agentDir, "daemon.sock"), {
+ defaultSessionConfig: { agentDir, cwd: agentDir },
+ descriptorDir: join(agentDir, "workers"),
}) as unknown as SupervisorHarness;
}
-function worker(lifecycle: "starting" | "ready" | "recovering" | "failed", connected = true) {
+function sessionInfoFor(path: string, id: string): SessionInfo {
+ return {
+ path,
+ id,
+ cwd: path,
+ rlmDepth: 0,
+ created: new Date(),
+ modified: new Date(),
+ messageCount: 0,
+ firstMessage: "",
+ allMessagesText: "",
+ };
+}
+
+function passiveRow(sessionFile: string, id: string): PassiveScheduledJobRow {
+ return { rootSessionFile: sessionFile, job: { id, status: "active" }, info: sessionInfoFor(sessionFile, id) };
+}
+
+const HEARTBEATS_CHANGED_FRAME = {
+ header: { kind: "outbound" as const, outboundType: "heartbeats_changed" as const },
+ payload: Buffer.alloc(0),
+};
+
+async function flushAsyncWork(): Promise<void> {
+ await new Promise((resolve) => setImmediate(resolve));
+}
+
+function worker(
+ lifecycle: "starting" | "ready" | "recovering" | "failed",
+ connected = true,
+): {
+ descriptor: { lifecycle: "starting" | "ready" | "recovering" | "failed" };
+ client?: object;
+ heartbeatSnapshot?: Array<{ job: { id: string } }>;
+ heartbeatSnapshotStale?: boolean;
+} {
return {
descriptor: { lifecycle },
...(connected ? { client: {} } : {}),
@@ -235,7 +287,8 @@ describe("daemon supervisor heartbeat aggregation", () => {
success: true,
data: { heartbeats: [{ job: { id: "heartbeat-1" } }, { job: { id: "heartbeat-2" } }] },
});
- expect(supervisor.forwardToWorker).toHaveBeenCalledTimes(3);
+ // The second list serves both workers from their cached snapshots: no re-forward per request.
+ expect(supervisor.forwardToWorker).toHaveBeenCalledTimes(2);
});
it("returns a worker failure instead of a partial catalog", async () => {
@@ -259,31 +312,6 @@ describe("daemon supervisor heartbeat aggregation", () => {
expect(supervisor.forwardToWorker).toHaveBeenCalledTimes(2);
});
- it("does not fall back to a snapshot after the worker reports heartbeat changes", async () => {
- const supervisor = createSupervisorHarness();
- const target = {
- ...worker("ready"),
- heartbeatSnapshot: [{ job: { id: "heartbeat-1" } }],
- heartbeatSnapshotStale: false,
- };
- supervisor.workers.set("target", target);
- supervisor.forwardToWorker = vi.fn(async (_worker, command) =>
- failure(command.id, command.type, "worker unavailable"),
- );
-
- supervisor.handleWorkerFrame(target, {
- header: { kind: "outbound", outboundType: "heartbeats_changed" },
- payload: Buffer.alloc(0),
- });
- const response = await supervisor.handleCommand({} as DaemonSocketClient, {
- id: "list-stale",
- type: "heartbeats_list",
- });
-
- expect(target.heartbeatSnapshotStale).toBe(true);
- expect(response).toMatchObject({ success: false, error: "worker unavailable" });
- });
-
it("fails rather than returning a partial catalog without a cached snapshot", async () => {
const supervisor = createSupervisorHarness();
supervisor.workers.set("ready", worker("ready"));
@@ -352,4 +380,312 @@ describe("daemon supervisor heartbeat aggregation", () => {
);
expect(target.heartbeatSnapshot).toEqual([]);
});
+
+ it("lets concurrent lists join one refresh without queuing a trailing pass", async () => {
+ const supervisor = createSupervisorHarness();
+ const target = worker("ready");
+ supervisor.workers.set("target", target);
+ let release = () => {};
+ const firstAnswer = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ let forwards = 0;
+ supervisor.forwardToWorker = vi.fn(async (_worker, command) => {
+ if (++forwards === 1) await firstAnswer;
+ return success(command.id, command.type, { heartbeats: [{ job: { id: "heartbeat-1" } }] });
+ });
+
+ const lists = ["list-1", "list-2"].map((id) =>
+ supervisor.handleCommand({} as DaemonSocketClient, { id, type: "heartbeats_list" }),
+ );
+ await flushAsyncWork();
+ release();
+ await Promise.all(lists);
+
+ expect(supervisor.forwardToWorker).toHaveBeenCalledOnce();
+ });
+
+ it("refreshes only the reporting worker, coalesces frames, ages out, and never serves a stale snapshot", async () => {
+ vi.useFakeTimers();
+ try {
+ const supervisor = createSupervisorHarness();
+ const first = worker("ready");
+ const second = worker("ready");
+ supervisor.workers.set("first", first);
+ supervisor.workers.set("second", second);
+ let firstHeartbeatId = "heartbeat-1";
+ let secondHeartbeatId = "heartbeat-2";
+ supervisor.forwardToWorker = vi.fn(async (target, command) =>
+ success(command.id, command.type, {
+ heartbeats: [{ job: { id: target === first ? firstHeartbeatId : secondHeartbeatId } }],
+ }),
+ );
+ const family = vi.spyOn(supervisor.rlmSpawnLedger(), "family");
+ const list = (id: string) =>
+ supervisor.handleCommand({} as DaemonSocketClient, { id, type: "heartbeats_list" });
+
+ // One forward per worker populates the snapshots; repeated lists serve them.
+ await list("list-1");
+ await list("list-2");
+ expect(supervisor.forwardToWorker).toHaveBeenCalledTimes(2);
+
+ secondHeartbeatId = "heartbeat-2-updated";
+ for (let frame = 0; frame < 3; frame++) {
+ supervisor.handleWorkerFrame(second, HEARTBEATS_CHANGED_FRAME);
+ }
+ await vi.advanceTimersByTimeAsync(0);
+ expect(supervisor.forwardToWorker).toHaveBeenCalledTimes(4);
+ expect(family).toHaveBeenCalledTimes(1);
+
+ const updated = (await list("list-3")) as { data?: { heartbeats?: Array<{ job: { id: string } }> } };
+ expect(updated.data?.heartbeats?.map((heartbeat) => heartbeat.job.id)).toEqual([
+ "heartbeat-1",
+ "heartbeat-2-updated",
+ ]);
+
+ firstHeartbeatId = "heartbeat-1-updated";
+ await vi.advanceTimersByTimeAsync(WORKER_HEARTBEAT_SNAPSHOT_MAX_AGE_MS);
+ const served = (await list("list-4")) as { data?: { heartbeats?: Array<{ job: { id: string } }> } };
+ expect(served.data?.heartbeats?.[0]?.job.id).toBe("heartbeat-1");
+ await vi.advanceTimersByTimeAsync(0);
+ expect(first.heartbeatSnapshot).toEqual([{ job: { id: "heartbeat-1-updated" } }]);
+
+ supervisor.forwardToWorker = vi.fn(async (_worker, command) =>
+ failure(command.id, command.type, "worker unavailable"),
+ );
+ supervisor.handleWorkerFrame(second, HEARTBEATS_CHANGED_FRAME);
+ await vi.advanceTimersByTimeAsync(0);
+ const failedRefresh = await list("list-5");
+ expect(second.heartbeatSnapshotStale).toBe(true);
+ expect(failedRefresh).toMatchObject({ success: false, error: "worker unavailable" });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
+
+describe("daemon supervisor passive scheduled-jobs snapshot", () => {
+ it("keeps the passive snapshot honest across residency gains without disk scans", async () => {
+ const supervisor = createSupervisorHarness();
+ const family = vi.spyOn(supervisor.rlmSpawnLedger(), "family");
+ const scan = vi.spyOn(
+ supervisor as unknown as { scanPassiveScheduledJobs: () => Promise<unknown[]> },
+ "scanPassiveScheduledJobs",
+ );
+ const coverableSession = join("tmp", "coverable.jsonl");
+ const freshSession = join("tmp", "fresh.jsonl");
+ const covering = {
+ descriptor: {
+ workerId: "worker-1",
+ lifecycle: "ready",
+ sessionFile: coverableSession,
+ createCommand: { sessionPath: coverableSession },
+ },
+ } as never;
+ supervisor.workers.set("worker-1", covering);
+ const restarted = {
+ ...worker("ready"),
+ heartbeatSnapshot: [{ job: { id: "heartbeat-1" } }],
+ heartbeatSnapshotStale: false,
+ };
+
+ supervisor.passiveScheduledJobs = {
+ scannedAt: Date.now(),
+ rows: [passiveRow(coverableSession, "job-1")],
+ };
+ supervisor.scheduleScheduledSessionWakeRecompute();
+ await flushAsyncWork();
+ expect(scan).not.toHaveBeenCalled();
+
+ supervisor.onWorkerResidencyGained(restarted as never);
+ expect(restarted.heartbeatSnapshotStale).toBe(true);
+ let releaseFamily: (() => void) | undefined;
+ family.mockImplementationOnce(() => new Promise((resolve) => (releaseFamily = () => resolve([]))));
+ supervisor.onWorkerResidencyGained(covering);
+ supervisor.passiveScheduledJobs = {
+ scannedAt: Date.now(),
+ rows: [passiveRow(freshSession, "job-2")],
+ };
+ releaseFamily!();
+ await flushAsyncWork();
+ const rows = supervisor.passiveScheduledJobs?.rows ?? [];
+ expect(rows.map((row) => [row.rootSessionFile, row.job.id])).toEqual([[freshSession, "job-2"]]);
+
+ family.mockRejectedValueOnce(new Error("ledger read failed")).mockResolvedValueOnce([]);
+ supervisor.onWorkerResidencyGained(covering);
+ await flushAsyncWork();
+ expect(scan).toHaveBeenCalledTimes(1);
+ expect(supervisor.passiveScheduledJobs?.rows).toEqual([]);
+ });
+
+ it("daemon-owned manage patches the snapshot row instead of rescanning", async () => {
+ const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-passive-"));
+ tempDirs.push(directory);
+ const supervisor = createSupervisorHarness(directory);
+ const manager = SessionManager.create(directory, join(directory, "sessions"));
+ manager.newSession();
+ manager.flushNow();
+ const store = AgentCronJobStore.forSessionArtifacts();
+ store.registerSessionArtifact(manager.getSessionId(), manager.getSessionArtifactDir()!);
+ const job = await store.createHeartbeat({
+ activeSessionId: manager.getSessionId(),
+ sessionId: manager.getSessionId(),
+ sessionFile: manager.getSessionFile()!,
+ cwd: directory,
+ scheduleText: "every 1h",
+ prompt: "continue",
+ });
+ const family = vi.spyOn(supervisor.rlmSpawnLedger(), "family");
+
+ await supervisor.handleCommand({} as DaemonSocketClient, { id: "list-1", type: "heartbeats_list" });
+ expect(family).toHaveBeenCalledTimes(1);
+ const managed = await supervisor.handleCommand({} as DaemonSocketClient, {
+ id: "manage-1",
+ type: "heartbeat_manage",
+ activeSessionId: manager.getSessionId(),
+ jobId: job.id,
+ action: "pause",
+ });
+ expect(managed).toMatchObject({ success: true, data: { heartbeat: { id: job.id, status: "paused" } } });
+
+ const paused = await supervisor.handleCommand({} as DaemonSocketClient, {
+ id: "list-2",
+ type: "heartbeats_list",
+ });
+ expect(
+ (paused as { data?: { heartbeats?: Array<{ job: { id: string; status: string } }> } }).data?.heartbeats,
+ ).toMatchObject([{ job: { id: job.id, status: "paused" } }]);
+ expect(family).toHaveBeenCalledTimes(1);
+ });
+
+ it("forwards a manage to the worker that became resident while the passive write waited for the lock", async () => {
+ const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-passive-"));
+ tempDirs.push(directory);
+ const supervisor = createSupervisorHarness(directory);
+ const manager = SessionManager.create(directory, join(directory, "sessions"));
+ manager.newSession();
+ manager.flushNow();
+ const store = AgentCronJobStore.forSessionArtifacts();
+ store.registerSessionArtifact(manager.getSessionId(), manager.getSessionArtifactDir()!);
+ const job = await store.createHeartbeat({
+ activeSessionId: manager.getSessionId(),
+ sessionId: manager.getSessionId(),
+ sessionFile: manager.getSessionFile()!,
+ cwd: directory,
+ scheduleText: "every 1h",
+ prompt: "continue",
+ });
+ await supervisor.handleCommand({} as DaemonSocketClient, { id: "list-1", type: "heartbeats_list" });
+ const covering = {
+ descriptor: {
+ workerId: "worker-1",
+ lifecycle: "ready",
+ sessionFile: manager.getSessionFile(),
+ createCommand: { sessionPath: manager.getSessionFile() },
+ },
+ } as never;
+ supervisor.findWorkerForClient = vi.fn(async () => ({ worker: covering }));
+ supervisor.forwardToWorker = vi.fn(async (_worker, command) =>
+ success(command.id, command.type, { heartbeat: { ...job, status: "paused" } }),
+ );
+ const jobsPath = join(manager.getSessionArtifactDir()!, SESSION_SCHEDULED_JOBS_FILENAME);
+ const release = await lock(jobsPath, { realpath: false, lockfilePath: `${jobsPath}.lock`, stale: 30_000 });
+
+ const managed = supervisor.handleCommand({} as DaemonSocketClient, {
+ id: "manage-1",
+ type: "heartbeat_manage",
+ activeSessionId: manager.getSessionId(),
+ jobId: job.id,
+ action: "pause",
+ });
+ await flushAsyncWork();
+ supervisor.workers.set("worker-1", covering);
+ await release();
+
+ await expect(managed).resolves.toMatchObject({ success: true });
+ expect(supervisor.forwardToWorker).toHaveBeenCalledWith(
+ covering,
+ expect.objectContaining({ type: "heartbeat_manage", jobId: job.id }),
+ );
+ expect(store.list().map((candidate) => candidate.status)).toEqual(["active"]);
+ });
+});
+
+describe("daemon supervisor heartbeats_changed delivery", () => {
+ function socketClient(): {
+ socket: { destroyed: boolean; write: ReturnType<typeof vi.fn> };
+ tracksHeartbeats?: boolean;
+ capabilities?: Set<string>;
+ } {
+ return {
+ socket: { destroyed: false, write: vi.fn(() => true) },
+ capabilities: new Set(["attach_snapshot", "event_sequence"]),
+ };
+ }
+
+ function heartbeatsChangedWrites(client: { socket: { write: ReturnType<typeof vi.fn> } }): number {
+ return client.socket.write.mock.calls.filter((args) => String(args[0]).includes('"type":"heartbeats_changed"'))
+ .length;
+ }
+
+ it("delivers one coalesced push only to clients that track heartbeats", async () => {
+ vi.useFakeTimers();
+ try {
+ const supervisor = createSupervisorHarness();
+ const tracked = socketClient();
+ const other = socketClient();
+ supervisor.clients.add(tracked);
+ supervisor.clients.add(other);
+ await supervisor.handleCommand(tracked as never, { id: "list-1", type: "heartbeats_list" });
+ expect(tracked.tracksHeartbeats).toBe(true);
+ expect(other.tracksHeartbeats).toBeUndefined();
+ const target = worker("ready");
+ supervisor.workers.set("target", target);
+
+ // Three frames in the window collapse into one push, not one per frame.
+ for (let frame = 0; frame < 3; frame++) {
+ supervisor.handleWorkerFrame(target, HEARTBEATS_CHANGED_FRAME);
+ }
+ await vi.advanceTimersByTimeAsync(HEARTBEATS_CHANGED_COALESCE_MS);
+ expect(heartbeatsChangedWrites(tracked)).toBe(1);
+ expect(heartbeatsChangedWrites(other)).toBe(0);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("marks clients that attach with the heartbeat_catalog capability and pushes to them", async () => {
+ vi.useFakeTimers();
+ try {
+ const supervisor = createSupervisorHarness();
+ supervisor.attachClient = vi.fn(async () => ({ result: { activeSessionId: "active-1" } })) as never;
+ const acp = socketClient();
+ const plain = socketClient();
+ supervisor.clients.add(acp);
+ supervisor.clients.add(plain);
+ await supervisor.handleCommand(acp as never, {
+ id: "attach-1",
+ type: "attach",
+ activeSessionId: "active-1",
+ capabilities: ["attach_snapshot", "event_sequence", "heartbeat_catalog"],
+ });
+ await supervisor.handleCommand(plain as never, {
+ id: "attach-2",
+ type: "attach",
+ activeSessionId: "active-1",
+ capabilities: ["attach_snapshot", "event_sequence"],
+ });
+ expect(acp.tracksHeartbeats).toBe(true);
+ expect(plain.tracksHeartbeats).toBeUndefined();
+
+ const target = worker("ready");
+ supervisor.workers.set("target", target);
+ supervisor.handleWorkerFrame(target, HEARTBEATS_CHANGED_FRAME);
+ await vi.advanceTimersByTimeAsync(HEARTBEATS_CHANGED_COALESCE_MS);
+ expect(heartbeatsChangedWrites(acp)).toBe(1);
+ expect(heartbeatsChangedWrites(plain)).toBe(0);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
});
diff --git a/packages/coding-agent/test/daemon-supervisor-process.test.ts b/packages/coding-agent/test/daemon-supervisor-process.test.ts
index 6949449ebc..038da8c422 100644
--- a/packages/coding-agent/test/daemon-supervisor-process.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-process.test.ts
@@ -812,7 +812,7 @@ describe("daemon supervisor resident workers", () => {
throw new Error("Fixture session did not persist");
}
const cronStore = new AgentCronJobStore(getCronJobsPath(agentDir));
- const heartbeat = cronStore.createHeartbeat({
+ const heartbeat = await cronStore.createHeartbeat({
activeSessionId: "old-active-session",
sessionId: sessionManager.getSessionId(),
sessionFile,
@@ -863,7 +863,7 @@ describe("daemon supervisor resident workers", () => {
throw new Error("Fixture session did not persist");
}
const cronStore = new AgentCronJobStore(getCronJobsPath(agentDir));
- const heartbeat = cronStore.createHeartbeat({
+ const heartbeat = await cronStore.createHeartbeat({
activeSessionId: "deleted-worker",
sessionId: sessionManager.getSessionId(),
sessionFile,
diff --git a/packages/coding-agent/test/daemon-worker-windows-timeouts.test.ts b/packages/coding-agent/test/daemon-worker-windows-timeouts.test.ts
index b2c19c8066..f23d5e912c 100644
--- a/packages/coding-agent/test/daemon-worker-windows-timeouts.test.ts
+++ b/packages/coding-agent/test/daemon-worker-windows-timeouts.test.ts
@@ -100,6 +100,7 @@ describe("Windows worker connection timing", () => {
refreshWorkerSummaries: async () => {},
persistWorker: () => {},
broadcastHeartbeatsChanged: () => {},
+ onWorkerResidencyGained: () => {},
}) as { adoptOrRecoverWorker(candidate: typeof worker): Promise<void> };
await supervisor.adoptOrRecoverWorker(worker);
expect(connectWorker).toHaveBeenCalledWith(worker, 90_000);
diff --git a/packages/coding-agent/test/main-interactive-routing.test.ts b/packages/coding-agent/test/main-interactive-routing.test.ts
index bc03c82aaa..5d170c800a 100644
--- a/packages/coding-agent/test/main-interactive-routing.test.ts
+++ b/packages/coding-agent/test/main-interactive-routing.test.ts
@@ -264,8 +264,8 @@ describe("runtime session option resolution", () => {
createRlmHeartbeat: () => {
throw new Error("not used");
},
- updateRlmHeartbeat: () => undefined,
- deleteRlmHeartbeat: () => undefined,
+ updateRlmHeartbeat: async () => undefined,
+ deleteRlmHeartbeat: async () => undefined,
};
const resolved = resolveRuntimeSessionOptions(
diff --git a/packages/coding-agent/test/proper-lockfile-compromise.test.ts b/packages/coding-agent/test/proper-lockfile-compromise.test.ts
index ae57fe805a..50fed286c1 100644
--- a/packages/coding-agent/test/proper-lockfile-compromise.test.ts
+++ b/packages/coding-agent/test/proper-lockfile-compromise.test.ts
@@ -136,13 +136,13 @@ describe("proper-lockfile compromise boundaries", () => {
expect(existsSync(join(root, "agent", "settings.json"))).toBe(false);
});
- it("does not mutate scheduled jobs after a lock compromise", () => {
- lockState.compromiseSync = true;
+ it("does not mutate scheduled jobs after a lock compromise", async () => {
+ lockState.compromiseAsync = true;
const root = tempDir("pa-lock-cron-");
const artifactDir = join(root, "artifact");
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact("session-1", artifactDir);
- expect(() => store.recoverSessionArtifact("session-1")).toThrow(/Cron jobs lock compromised/);
+ await expect(store.recoverSessionArtifact("session-1")).rejects.toThrow(/Cron jobs lock compromised/);
expect(existsSync(join(artifactDir, "scheduled-jobs.json"))).toBe(false);
});
});
diff --git a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts b/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
index 11c4f8d535..07c3930948 100644
--- a/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
+++ b/packages/coding-agent/test/suite/agent-session-retry-events.test.ts
@@ -897,7 +897,12 @@ describe("AgentSession retry and event characterization", () => {
};
const wakeQuotaProbe = (harness: Harness): (() => Promise<AssistantMessage>) => {
const turn = assistantTurns(harness);
- return () => fireQuotaWake(harness).then(turn);
+ const internals = harness.session as unknown as QuotaParkInternals;
+ return async () => {
+ const message = await fireQuotaWake(harness).then(turn);
+ await internals._agentEventQueue;
+ return message;
+ };
};
const readQuotaWakeJob = (harness: Harness, jobId: string | undefined) =>
@@ -972,8 +977,8 @@ describe("AgentSession retry and event characterization", () => {
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact(harness.sessionManager.getSessionId(), artifactDir);
- const [dispatch] = store.claimDue(new Date(Date.now() + 60_000));
- store.recordDispatchResult(dispatch!.id, { outcome: "ran" });
+ const [dispatch] = await store.claimDue(new Date(Date.now() + 60_000));
+ await store.recordDispatchResult(dispatch!.id, { outcome: "ran" });
await fireQuotaWake(harness);
expect([quotaPark(harness)?.waking, harness.faux.state.callCount]).toEqual([true, 1]);
expect(readQuotaWakeJob(harness, jobId)?.status).toBe("completed");
@@ -990,6 +995,7 @@ describe("AgentSession retry and event characterization", () => {
const aborted = assistantTurns(abortedHarness);
await fireQuotaWake(abortedHarness);
expect((await aborted()).stopReason).toBe("aborted");
+ await (abortedHarness.session as unknown as QuotaParkInternals)._agentEventQueue;
const reArmed = quotaPark(abortedHarness);
expect([reArmed?.waking, (reArmed?.resumeAtMs ?? 0) > parkedAtMs]).toEqual([false, true]);
@@ -1073,7 +1079,7 @@ describe("AgentSession retry and event characterization", () => {
restarted.sessionManager.getSessionId(),
restarted.sessionManager.getSessionArtifactDir()!,
);
- store.cancel(quotaPark(restarted)!.jobId!);
+ await store.cancel(quotaPark(restarted)!.jobId!);
await restarted.session.navigateTree(lastUserEntryId(restarted));
await restarted.session.navigateTree(restarted.sessionManager.getLeafId()!);
expect(restarted.session.isQuotaParked).toBe(false);
diff --git a/packages/coding-agent/test/suite/regressions/2259-scheduled-catalog-metadata.test.ts b/packages/coding-agent/test/suite/regressions/2259-scheduled-catalog-metadata.test.ts
index 3715f77d17..e1309cdfc6 100644
--- a/packages/coding-agent/test/suite/regressions/2259-scheduled-catalog-metadata.test.ts
+++ b/packages/coding-agent/test/suite/regressions/2259-scheduled-catalog-metadata.test.ts
@@ -58,7 +58,7 @@ describe("scheduled catalog metadata scans", () => {
const store = AgentCronJobStore.forSessionArtifacts();
for (const manager of [child!, archived!]) {
store.registerSessionArtifact(manager.getSessionId(), manager.getSessionArtifactDir()!);
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: manager.getSessionId(),
sessionId: manager.getSessionId(),
sessionFile: manager === child ? childFile : manager.getSessionFile()!,
@@ -85,7 +85,7 @@ describe("scheduled catalog metadata scans", () => {
expect(readInfo).not.toHaveBeenCalledWith(unrelated!.getSessionFile());
const late = unrelated!;
store.registerSessionArtifact(late.getSessionId(), late.getSessionArtifactDir()!);
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: late.getSessionId(),
sessionId: late.getSessionId(),
sessionFile: late.getSessionFile()!,
diff --git a/packages/coding-agent/test/suite/regressions/4257-update-restart-resume.test.ts b/packages/coding-agent/test/suite/regressions/4257-update-restart-resume.test.ts
index 670301d2d5..02509f6e52 100644
--- a/packages/coding-agent/test/suite/regressions/4257-update-restart-resume.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4257-update-restart-resume.test.ts
@@ -360,7 +360,7 @@ describe("update restart queue recovery over the daemon RPC boundary (issue #425
const activeSessionId = "restored-active";
const persistedStore = AgentCronJobStore.forSessionArtifacts();
persistedStore.registerSessionArtifact(harness.session.sessionId, artifactDir);
- const heartbeat = persistedStore.createHeartbeat({
+ const heartbeat = await persistedStore.createHeartbeat({
activeSessionId,
sessionId: harness.session.sessionId,
sessionFile,
@@ -376,8 +376,8 @@ describe("update restart queue recovery over the daemon RPC boundary (issue #425
vi.useFakeTimers();
internals.cronScheduler.start();
try {
- internals.registerCronStoreForState(state);
- internals.rebindCronJobsToState(state);
+ await internals.registerCronStoreForState(state);
+ await internals.rebindCronJobsToState(state);
await internals.cronScheduler.runDue();
expect(internals.cronStore.list().find((job) => job.id === heartbeat.id)?.runCount).toBe(1);
diff --git a/packages/coding-agent/test/suite/regressions/4527-worker-heartbeat-scheduling.test.ts b/packages/coding-agent/test/suite/regressions/4527-worker-heartbeat-scheduling.test.ts
index 063289217f..400185991a 100644
--- a/packages/coding-agent/test/suite/regressions/4527-worker-heartbeat-scheduling.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4527-worker-heartbeat-scheduling.test.ts
@@ -33,7 +33,7 @@ describe("ENG-4527 worker heartbeat scheduling", () => {
const sessionFile = join(harness.tempDir, "sessions", `${sessionId}.jsonl`);
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact(sessionId, artifactDir);
- store.createHeartbeat({
+ await store.createHeartbeat({
activeSessionId: `worker-${index}`,
sessionId,
sessionFile,
diff --git a/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts b/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts
index 702dc39a3c..443b2470b5 100644
--- a/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts
+++ b/packages/coding-agent/test/suite/regressions/heartbeats-list-response-latency.test.ts
@@ -44,14 +44,16 @@ describe("heartbeats_list response latency", () => {
manager.flushNow();
const store = AgentCronJobStore.forSessionArtifacts();
store.registerSessionArtifact(manager.getSessionId(), manager.getSessionArtifactDir()!);
- const job = store.createHeartbeat({
- activeSessionId: manager.getSessionId(),
- sessionId: manager.getSessionId(),
- sessionFile: manager.getSessionFile()!,
- cwd: directory,
- scheduleText: "every 1h",
- prompt: "continue",
- }).id;
+ const job = (
+ await store.createHeartbeat({
+ activeSessionId: manager.getSessionId(),
+ sessionId: manager.getSessionId(),
+ sessionFile: manager.getSessionFile()!,
+ cwd: directory,
+ scheduleText: "every 1h",
+ prompt: "continue",
+ })
+ ).id;
const family = vi.spyOn(supervisor.rlmSpawnLedger(), "family");
const responses = await Promise.all(["1", "2", "3", "4", "5"].map((id) => listHeartbeats(supervisor, id)));
expect(responses.map(heartbeatIds)).toEqual([[job], [job], [job], [job], [job]]); // one scan served five lists
@@ -73,10 +75,10 @@ describe("heartbeats_list response latency", () => {
releaseScan.resolve();
await siblings;
}
- store.manageHeartbeat(manager.getSessionId(), job, "pause");
+ await store.manageHeartbeat(manager.getSessionId(), job, "pause");
supervisor.broadcastHeartbeatsChanged();
expect(heartbeatIds(await listHeartbeats(supervisor, "7"))).toEqual([job]); // broadcast dropped the snapshot
- store.manageHeartbeat(manager.getSessionId(), job, "stop");
+ await store.manageHeartbeat(manager.getSessionId(), job, "stop");
supervisor.broadcastHeartbeatsChanged();
expect(heartbeatIds(await listHeartbeats(supervisor, "8"))).toEqual([]);
});