Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

feat(scheduler): retain run outcomes and require explicit interrupted-work replay

Seth Wilson113290d8ead5
diff --git a/src/bots/bot-manager.test.ts b/src/bots/bot-manager.test.ts
index abbe4fda..85f9a3b1 100644
--- a/src/bots/bot-manager.test.ts
+++ b/src/bots/bot-manager.test.ts
@@ -218,6 +218,51 @@ describe('BotManager queue + turn lifecycle', () => {
}
});
+ it('does not automatically retry a failed scheduled bot run and pauses its routine', async () => {
+ mockedGenerateText.mockRejectedValue(new Error('HTTP 429: too many requests'));
+ seedBot(store, 'scheduled');
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] });
+ const settle = async () => { for (let i = 0; i < 20; i++) await new Promise(r => setImmediate(r)); };
+ try {
+ const { jobId } = manager.enqueue('scheduled', { trigger: 'cron', prompt: 'Send the report', routineId: 'bot:scheduled:report' });
+ await settle();
+ expect(manager.getJournal('scheduled').map((r) => r.state)).toEqual(['failed']);
+ expect(manager.queue.pendingJobs('scheduled')).toEqual([]);
+ expect(manager.queue.listDlq('scheduled').map((j) => j.id)).toEqual([jobId]);
+ expect(store.readRoutineState('scheduled').paused['bot:scheduled:report']).toBeDefined();
+ await vi.advanceTimersByTimeAsync(30_000);
+ await settle();
+ expect(manager.getJournal('scheduled')).toHaveLength(1);
+ expect(manager.enqueue('scheduled', { trigger: 'cron', prompt: 'Send the report again', routineId: 'bot:scheduled:report' })).toMatchObject({ accepted: false, reasonCode: 'routine_paused' });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('pauses future routines on quick restart and retains the DLQ until explicit replay is accepted', async () => {
+ seedBot(store, 'crashed');
+ manager.queue.enqueue({ id: 'previous-run', botId: 'crashed', trigger: 'cron', prompt: 'Send once', attempts: 0, createdAt: Date.now(), idempotencyKey: 'previous-schedule' });
+ manager.queue.claim('previous-run', 3600);
+ manager.dispose();
+ const restarted = makeManager(root);
+ expect(restarted.queue.listDlq('crashed').map((j) => j.id)).toEqual(['previous-run']);
+ expect(store.readRoutineState('crashed').paused['*']).toBeDefined();
+ expect(restarted.enqueue('crashed', { trigger: 'cron', prompt: 'Send once', routineId: 'bot:crashed:any-name' })).toMatchObject({ accepted: false, reasonCode: 'routine_paused' });
+ expect(mockedGenerateText).not.toHaveBeenCalled();
+ restarted.setEnabled('crashed', false);
+ expect(restarted.replayDlq('crashed', 'previous-run')).toMatchObject({ accepted: false, reasonCode: 'target_disabled' });
+ expect(restarted.queue.listDlq('crashed')).toHaveLength(1);
+ restarted.setEnabled('crashed', true);
+ mockedGenerateText.mockResolvedValue({ text: 'done', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as never);
+ expect(restarted.replayDlq('crashed', 'previous-run')).toMatchObject({ accepted: true });
+ expect(restarted.queue.listDlq('crashed')).toEqual([]);
+ await vi.waitFor(() => expect(restarted.getJournal('crashed')).toHaveLength(1));
+ // Explicit replay repeats just this job, not all future ticks.
+ expect(store.readRoutineState('crashed').paused['*']).toBeDefined();
+ restarted.start('crashed');
+ expect(store.readRoutineState('crashed').paused).toEqual({});
+ });
+
it('a retried job never runs twice when the due-sweep and the backoff timer both fire (P0.7)', async () => {
let calls = 0;
let releaseSecond: () => void = () => {};
diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.ts
index 02a28444..8dcf81a6 100644
--- a/src/bots/bot-manager.ts
+++ b/src/bots/bot-manager.ts
@@ -273,6 +273,14 @@ export class BotManager {
this.queue = deps.queue ?? new BotQueue(this.store.botsRoot, this.config.bots?.retention?.dlqCap);
this.tasks = new BotTaskStore(this.store.botsRoot);
this.inbox = new BotInbox(this.store.botsRoot);
+ this.queue.onInterruptedSchedule = (job) => {
+ if (!this.store.exists(job.botId)) return;
+ const routines = this.store.readRoutineState(job.botId);
+ routines.paused['*'] = { since: new Date().toISOString(), reason: 'A scheduled run was interrupted. Inspect its effects before /bots start or manual DLQ replay.' };
+ this.store.writeRoutineState(job.botId, routines);
+ this.needsYou.add(job.botId);
+ this.inbox.add({ botId: job.botId, botName: this.store.get(job.botId)?.name ?? job.botId, runId: job.id, files: [], kind: 'needs-you', title: 'Scheduled run interrupted', summary: routines.paused['*'].reason, reasonCode: 'scheduled_interrupted' });
+ };
// A stop survives a restart (it used to be in memory only, so a restart
// quietly undid /bots stop).
for (const id of this.store.readHeld()) if (this.store.exists(id)) this.held.add(id);
@@ -316,7 +324,7 @@ export class BotManager {
for (const m of this.store.list()) {
const drained = this.queue.drainMail(m.id);
if (drained.length > 0) this.mailboxes.set(m.id, drained);
- if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou) this.needsYou.add(m.id);
+ if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou || this.store.readRoutineState(m.id).paused['*']) this.needsYou.add(m.id);
}
// A profile relocation (re-parent or fleet-layout migration) invalidates
// the cached journal handle (it is bound to the old dir) and the bot's
@@ -435,7 +443,7 @@ export class BotManager {
return this.journalFor(botId);
}
- enqueue(botId: string, job: { trigger: BotTrigger; prompt: string; fromBot?: string; source?: { channelType: string; channelId: string }; attempts?: number; routineId?: string; attachments?: ChannelAttachment[] }): { jobId: string; accepted: boolean; reasonCode?: string; duplicated?: boolean } {
+ enqueue(botId: string, job: { trigger: BotTrigger; prompt: string; fromBot?: string; source?: { channelType: string; channelId: string }; attempts?: number; routineId?: string; attachments?: ChannelAttachment[]; manualReplay?: boolean }): { jobId: string; accepted: boolean; reasonCode?: string; duplicated?: boolean } {
const manifest = this.store.get(botId);
if (!manifest) return { jobId: '', accepted: false, reasonCode: 'target_unknown' };
if (!manifest.enabled || this.disabled.has(botId)) return { jobId: '', accepted: false, reasonCode: 'target_disabled' };
@@ -447,12 +455,12 @@ export class BotManager {
// It is skipped — never stacked — when the bot is still busy, when the
// routine ran too recently, or when the routine was paused for
// producing nothing. The next tick tries again.
- if (job.trigger === 'cron') {
+ if (job.trigger === 'cron' && !job.manualReplay) {
// A stopped bot's routines don't queue up behind the stop.
if (this.held.has(botId)) return { jobId: '', accepted: false, reasonCode: 'stopped' };
const key = job.routineId ?? `bot:${botId}:cron`;
const routineState = this.store.readRoutineState(botId);
- const pausedEntry = routineState.paused[key];
+ const pausedEntry = routineState.paused[key] ?? routineState.paused['*'];
if (pausedEntry) {
const until = pausedEntry.until ? Date.parse(pausedEntry.until) : Number.POSITIVE_INFINITY;
if (Date.now() < until) return { jobId: '', accepted: false, reasonCode: 'routine_paused' };
@@ -953,7 +961,7 @@ export class BotManager {
this.providers.markAuthFailure?.(usedProvider);
authSwitch = resolveProvider(this.providers, manifest, this.config).name !== usedProvider;
}
- const willRetry = output.status === 'failed' && !!output.reasonCode
+ const willRetry = job.trigger !== 'cron' && output.status === 'failed' && !!output.reasonCode
&& (isTransientFailure(output.reasonCode) || authSwitch) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS;
const turnLimited = output.status === 'failed' && isTurnLimitFailure(output.reasonCode);
const routineKey = job.trigger === 'cron' ? (job.routineId ?? `bot:${botId}:cron`) : undefined;
@@ -1008,6 +1016,9 @@ export class BotManager {
} else if (output.status === 'completed' && state.failStreak?.[routineKey]) {
delete state.failStreak[routineKey];
}
+ if (output.status !== 'completed') {
+ state.paused[routineKey] = { since: new Date().toISOString(), reason: 'Scheduled run did not finish. Inspect its effects before /bots start or manual DLQ replay.' };
+ }
this.store.writeRoutineState(botId, state);
}
@@ -1015,7 +1026,7 @@ export class BotManager {
// three runs in a row is paused — the owner is told once, and
// /bots start resumes it. This is what stops a bot that has
// talked itself into a loop of records about records.
- if (routineKey && this.expectationFor(manifest, job) === 'work' && output.status !== 'halted') {
+ if (routineKey && this.expectationFor(manifest, job) === 'work' && output.status === 'completed') {
const state = this.store.readRoutineState(botId);
if (output.outcome === 'none') {
const streak = (state.noOutcomeStreak[routineKey] ?? 0) + 1;
@@ -1099,6 +1110,10 @@ export class BotManager {
if (routineFailStreak <= 1) {
await this.alertOwner(botId, `❌ **${manifest.name}** failed permanently [reason: ${output.reasonCode}] — replay with \`/bots replay ${botId} ${job.id}\``);
}
+ } else if (output.status === 'paused' && job.trigger === 'cron') {
+ this.queue.settle(job.id, 'dead', 'scheduled_interrupted');
+ this.needsYou.add(botId);
+ this.inbox.add({ botId, botName: manifest.name, runId: job.id, files: output.deliverables, kind: 'needs-you', title: 'Scheduled run paused', summary: 'Inspect the existing effects before /bots start or manually replaying this run from the DLQ.', reasonCode: 'scheduled_interrupted' });
} else if (output.status === 'paused') {
// Step-budget pause: work continues next turn — same job requeues in
// place (durable), no attempts bump.
@@ -1232,6 +1247,11 @@ export class BotManager {
}
} catch (err: any) {
logger.error({ botId, jobId: job.id, err: err?.message }, 'Bot turn crashed');
+ if (job.trigger === 'cron') {
+ const state = this.store.readRoutineState(botId);
+ state.paused[job.routineId ?? `bot:${botId}:cron`] = { since: new Date().toISOString(), reason: 'Scheduled run crashed. Inspect its effects before /bots start or manual DLQ replay.' };
+ this.store.writeRoutineState(botId, state);
+ }
this.queue.settle(job.id, 'dead', 'unknown_error');
this.needsYou.add(botId);
void this.alertOwner(botId, `💥 **${botId}** run ${job.id} crashed: ${String(err?.message ?? err).slice(0, 150)} — see journal; replay from DLQ.`);
@@ -2086,14 +2106,20 @@ export class BotManager {
// then reported not_found).
const entry = this.queue.peekDlq(jobId);
if (!entry || entry.botId !== botId) return { accepted: false, reasonCode: 'not_found' };
- this.queue.removeFromDlq(jobId);
- return this.enqueue(botId, {
+ const result = this.enqueue(botId, {
trigger: entry.trigger,
+ // The owner explicitly chose this job. Bypass routine timing/pause gates
+ // for this admission only; future ticks stay paused and a new crash still
+ // has cron recovery semantics.
+ manualReplay: true,
+ attachments: entry.attachments,
prompt: entry.prompt,
fromBot: entry.fromBot,
source: entry.source,
attempts: 0,
});
+ if (result.accepted) this.queue.removeFromDlq(jobId);
+ return result;
}
getStorage(): Array<{ id: string; bytes: number; journalBytes: number }> {
diff --git a/src/bots/queue.test.ts b/src/bots/queue.test.ts
index 5c30448f..deae18d4 100644
--- a/src/bots/queue.test.ts
+++ b/src/bots/queue.test.ts
@@ -46,6 +46,7 @@ mockedStreamText.mockImplementation(((opts: any) => {
}) as any);
import { BotQueue, JsonFileQueueBackend, SqliteQueueBackend, idempotencyKeyFor, LEASE_SECONDS } from './queue.js';
import type { DurableBotJob } from './queue.js';
+import { openSqlite } from '../utils/sqlite-driver.js';
function job(id: string, overrides: Partial<DurableBotJob> = {}): Omit<DurableBotJob, 'state'> {
return {
@@ -181,6 +182,62 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
expect(backend.counts().claimed).toBe(1);
});
+ it('quarantines interrupted scheduled jobs while preserving safe first attempts and chat recovery', () => {
+ const backend = mkBackend(name, root);
+ const queue = Object.create(BotQueue.prototype) as BotQueue;
+ const interrupted = vi.fn();
+ Object.assign(queue, { backend, onInterruptedSchedule: interrupted });
+ backend.enqueue(job('scheduled-claimed', { trigger: 'cron' }));
+ backend.claim('scheduled-claimed', 0);
+ backend.enqueue(job('scheduled-fresh', { trigger: 'cron' }));
+ backend.enqueue(job('scheduled-retry', { trigger: 'cron' }));
+ backend.claim('scheduled-retry', 0);
+ backend.retry('scheduled-retry', 1, Date.now() - 1);
+ backend.enqueue(job('scheduled-continuation', { trigger: 'cron' }));
+ backend.claim('scheduled-continuation', 0);
+ backend.retry('scheduled-continuation', 0, Date.now() - 1);
+ backend.enqueue(job('chat-claimed'));
+ backend.claim('chat-claimed', 0);
+ expect(queue.resumeJobs().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);
+ expect(interrupted).toHaveBeenCalledTimes(3);
+ expect(backend.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-claimed', 'scheduled-continuation', 'scheduled-retry']);
+ expect(backend.listDlq().every((j) => j.reasonCode === 'scheduled_interrupted')).toBe(true);
+ const restarted = mkBackend(name, root);
+ expect(restarted.rehydratable().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);
+ });
+
+ it('quarantines a scheduled run on quick restart before its lease expires', () => {
+ const backend = mkBackend(name, root);
+ backend.enqueue(job('scheduled-live-lease', { trigger: 'cron' }));
+ backend.claim('scheduled-live-lease', 3600);
+ backend.enqueue(job('chat-live-lease'));
+ backend.claim('chat-live-lease', 3600);
+ backend.enqueue(job('scheduled-future-retry', { trigger: 'cron' }));
+ backend.retry('scheduled-future-retry', 1, Date.now() + 3600_000);
+ const restarted = mkBackend(name, root);
+ const queue = Object.create(BotQueue.prototype) as BotQueue;
+ const pause = vi.fn();
+ Object.assign(queue, { backend: restarted, onInterruptedSchedule: pause });
+ expect(queue.resumeJobs()).toEqual([]);
+ expect(pause).toHaveBeenCalledTimes(2);
+ expect(restarted.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-future-retry', 'scheduled-live-lease']);
+ expect(restarted.counts()).toMatchObject({ claimed: 1 });
+ });
+
+ it('does not discard the interrupted job when saving its routine pause fails', () => {
+ const backend = mkBackend(name, root);
+ const queue = Object.create(BotQueue.prototype) as BotQueue;
+ Object.assign(queue, { backend, onInterruptedSchedule: () => { throw new Error('Disk full'); } });
+ backend.enqueue(job('interrupted', { trigger: 'cron' }));
+ backend.claim('interrupted', 0);
+ expect(() => queue.resumeJobs()).toThrow('Disk full');
+ expect(backend.rehydratable().map((j) => j.id)).toEqual(['interrupted']);
+ expect(backend.listDlq()).toEqual([]);
+ queue.onInterruptedSchedule = vi.fn();
+ expect(queue.resumeJobs()).toEqual([]);
+ expect(backend.listDlq()[0].reasonCode).toBe('scheduled_interrupted');
+ });
+
it('DLQ is capped with oldest-first eviction', () => {
const backend = mkBackend(name, root, 3);
for (let i = 0; i < 5; i++) {
@@ -214,6 +271,22 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
});
}
+ it('rolls back SQLite quarantine if the DLQ insert fails after removing the active row', () => {
+ const db = openSqlite(join(root, 'queue.db'))!;
+ const backend = new SqliteQueueBackend(root, 100, db);
+ openBackends.push(backend);
+ backend.enqueue(job('quarantine', { trigger: 'cron' }));
+ backend.claim('quarantine', 3600);
+ db.exec(`CREATE TRIGGER fail_dlq BEFORE INSERT ON bot_dlq BEGIN SELECT RAISE(ABORT, 'injected disk error'); END`);
+ expect(() => backend.settle('quarantine', 'dead', 'scheduled_interrupted')).toThrow('injected disk error');
+ expect(backend.interruptedSchedules().map((j) => j.id)).toEqual(['quarantine']);
+ expect(backend.listDlq()).toEqual([]);
+ db.exec('DROP TRIGGER fail_dlq');
+ backend.settle('quarantine', 'dead', 'scheduled_interrupted');
+ expect(backend.interruptedSchedules()).toEqual([]);
+ expect(backend.listDlq()[0]).toMatchObject({ id: 'quarantine', prompt: 'task quarantine', reasonCode: 'scheduled_interrupted' });
+ });
+
it('BotQueue facade resumes expired leases and pending jobs', () => {
const dir = join(root, 'bots');
const queue = new BotQueue(dir, 100);
diff --git a/src/bots/queue.ts b/src/bots/queue.ts
index 6cbfcb20..d88d8e8c 100644
--- a/src/bots/queue.ts
+++ b/src/bots/queue.ts
@@ -64,6 +64,8 @@ export interface BotQueueBackend {
retry(jobId: string, attempts: number, runAfterMs?: number): void;
/** Jobs safe to resume: pending (due), or claimed whose lease has expired. */
rehydratable(): DurableBotJob[];
+ /** Startup only: claimed scheduled jobs and old scheduled retry entries, regardless of lease/backoff. */
+ interruptedSchedules(): DurableBotJob[];
/** Pending jobs whose run-after time has arrived (retry backoff elapsed). */
dueJobs(): DurableBotJob[];
/** All pending jobs for one bot, in creation order — explicit resume (/bots start), ignores retry backoff. */
@@ -147,6 +149,7 @@ function normalizeJob(raw: any): DurableBotJob {
idempotencyKey: raw.idempotencyKey ?? '',
reasonCode: raw.reasonCode ?? raw.reason_code ?? undefined,
settledAt: raw.settledAt ?? raw.settled_at ?? undefined,
+ runAfter: raw.runAfter ?? raw.run_after ?? undefined,
};
}
@@ -160,6 +163,8 @@ function normalizeJob(raw: any): DurableBotJob {
*/
export class BotQueue {
readonly backend: BotQueueBackend;
+ /** The manager pauses future routines when a prior scheduled run is uncertain. */
+ onInterruptedSchedule?: (job: DurableBotJob) => void;
/** Post-close guard: in-flight turns' queue access degrades to no-ops. */
private closed = false;
@@ -201,15 +206,29 @@ export class BotQueue {
/** Jobs to resume on startup: due pending, plus expired-lease claimed jobs. */
resumeJobs(): DurableBotJob[] {
+ this.excludeInterruptedSchedules(this.backend.interruptedSchedules());
const expired = this.backend.requeueExpiredLeases();
if (expired > 0) logger.info({ expired }, 'Requeued expired bot job leases');
- return this.backend.rehydratable();
+ return this.excludeInterruptedSchedules(this.backend.rehydratable());
+ }
+
+ private excludeInterruptedSchedules(jobs: DurableBotJob[]): DurableBotJob[] {
+ return jobs.filter((job) => {
+ // A pending first attempt has not acted. Claimed work and old retry/
+ // continuation entries may already have effects and require manual replay.
+ if (job.trigger !== 'cron' || (job.state === 'pending' && job.attempts === 0 && job.runAfter === undefined)) return true;
+ // Persist the routine pause first. If that fails, leave the job eligible
+ // for quarantine again rather than losing the opportunity to stop future ticks.
+ this.onInterruptedSchedule?.(job);
+ this.backend.settle(job.id, 'dead', 'scheduled_interrupted');
+ return false;
+ });
}
/** Pending jobs whose retry backoff has elapsed (swept periodically). */
dueJobs(): DurableBotJob[] {
if (this.closed) return [];
- return this.backend.dueJobs();
+ return this.excludeInterruptedSchedules(this.backend.dueJobs());
}
/** All pending jobs for one bot — explicit resume via /bots start. */
@@ -365,27 +384,30 @@ export class SqliteQueueBackend implements BotQueueBackend {
}
settle(jobId: string, outcome: 'done' | 'dead', reasonCode?: string): void {
- const row = this.db.prepare(`SELECT * FROM bot_jobs WHERE id = ?`).get(jobId) as any;
- if (!row) return;
- if (outcome === 'done') {
+ if (this.closed) return;
+ this.db.transaction(() => {
+ const row = this.db.prepare(`SELECT * FROM bot_jobs WHERE id = ?`).get(jobId) as any;
+ if (!row) return;
+ if (outcome === 'done') {
+ this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);
+ return;
+ }
this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);
- return;
- }
- this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);
- this.db.prepare(
- `INSERT INTO bot_dlq (id, bot_id, trigger, prompt, from_bot, source, attempts, created_at, idempotency_key, reason_code, dead_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
- ).run(
- row.id, row.bot_id, row.trigger, row.prompt, row.from_bot, row.source,
- row.attempts + 1, row.created_at, row.idempotency_key, reasonCode ?? row.reason_code ?? null, Date.now(),
- );
- // Cap the DLQ — evict oldest, bounded by design (§2.10).
- const overflow = this.db.prepare(
- `SELECT id FROM bot_dlq ORDER BY dead_at DESC, created_at DESC, rowid DESC LIMIT -1 OFFSET ?`,
- ).all(this.dlqCap) as any[];
- for (const r of overflow) {
- this.db.prepare(`DELETE FROM bot_dlq WHERE id = ?`).run(r.id);
- }
+ this.db.prepare(
+ `INSERT INTO bot_dlq (id, bot_id, trigger, prompt, from_bot, source, attempts, created_at, idempotency_key, reason_code, dead_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ ).run(
+ row.id, row.bot_id, row.trigger, row.prompt, row.from_bot, row.source,
+ row.attempts + 1, row.created_at, row.idempotency_key, reasonCode ?? row.reason_code ?? null, Date.now(),
+ );
+ // Cap the DLQ — evict oldest, bounded by design (§2.10).
+ const overflow = this.db.prepare(
+ `SELECT id FROM bot_dlq ORDER BY dead_at DESC, created_at DESC, rowid DESC LIMIT -1 OFFSET ?`,
+ ).all(this.dlqCap) as any[];
+ for (const r of overflow) {
+ this.db.prepare(`DELETE FROM bot_dlq WHERE id = ?`).run(r.id);
+ }
+ });
}
requeueExpiredLeases(): number {
@@ -402,6 +424,14 @@ export class SqliteQueueBackend implements BotQueueBackend {
).run(attempts, runAfterMs ?? null, jobId);
}
+ interruptedSchedules(): DurableBotJob[] {
+ if (this.closed) return [];
+ const rows = this.db.prepare(
+ `SELECT * FROM bot_jobs WHERE trigger = 'cron' AND (state = 'claimed' OR attempts > 0 OR run_after IS NOT NULL) ORDER BY created_at`,
+ ).all() as unknown[];
+ return rows.map(normalizeJob);
+ }
+
rehydratable(): DurableBotJob[] {
if (this.closed) return [];
// Due pending jobs + expired-lease claimed jobs — jobs still in their
@@ -618,6 +648,11 @@ export class JsonFileQueueBackend implements BotQueueBackend {
}
}
+ interruptedSchedules(): DurableBotJob[] {
+ return this.data.jobs.filter((job) => job.trigger === 'cron'
+ && (job.state === 'claimed' || job.attempts > 0 || job.runAfter !== undefined));
+ }
+
rehydratable(): DurableBotJob[] {
// Due pending + expired-lease claimed (INCLUSIVE — a heartbeat to an
// instant-expiry lease must be sweepable in the same millisecond;
diff --git a/src/bots/tools/bot-schedule.ts b/src/bots/tools/bot-schedule.ts
index eaa64b54..64797bac 100644
--- a/src/bots/tools/bot-schedule.ts
+++ b/src/bots/tools/bot-schedule.ts
@@ -23,7 +23,7 @@ export interface BotScheduler {
}): void;
persistSchedules(): void;
/** All live manifests, used for the per-bot self-schedule cap. */
- getManifests(): Array<{ id: string; botId?: string }>;
+ getManifests(): Array<{ id: string; botId?: string; lastRun?: { status: string } }>;
}
const SELF_PREFIX = 'self-';
@@ -107,7 +107,7 @@ export function createBotScheduleTool(scheduler: BotScheduler, botId: string) {
}
// One-shot with a pending cap — this is the runaway-loop guard.
- const pending = scheduler.getManifests().filter(m => m.botId === botId && m.id.includes(`:${SELF_PREFIX}`)).length;
+ const pending = scheduler.getManifests().filter(m => m.botId === botId && m.id.includes(`:${SELF_PREFIX}`) && (!m.lastRun || m.lastRun.status === 'running')).length;
if (pending >= MAX_PENDING_SELF_SCHEDULES) {
return `Error: you already have ${MAX_PENDING_SELF_SCHEDULES} pending self-scheduled runs — complete or let them fire before adding more.`;
}
diff --git a/src/capabilities/scheduler/list-tasks.ts b/src/capabilities/scheduler/list-tasks.ts
index 4a158f9e..78b7197c 100644
--- a/src/capabilities/scheduler/list-tasks.ts
+++ b/src/capabilities/scheduler/list-tasks.ts
@@ -4,7 +4,7 @@ import type { Scheduler, ScheduledTaskManifest } from '../../core/scheduler.js';
export function createListTasksTool(scheduler: Scheduler) {
return tool({
- description: 'List all scheduled tasks with their cron expressions and descriptions.',
+ description: 'List scheduled tasks, run outcomes, failures, timezone and missed-run policy. Failed or interrupted tasks remain paused until explicitly rescheduled.',
inputSchema: zodSchema(z.object({})),
execute: async () => {
const manifests = scheduler.getManifests();
@@ -13,7 +13,9 @@ export function createListTasksTool(scheduler: Scheduler) {
}
return manifests.map((m: ScheduledTaskManifest) => {
const trigger = m.skillName ? `skill: ${m.skillName}` : m.prompt ? `prompt: "${m.prompt.slice(0, 40)}"` : 'no action';
- return `${m.id} | ${m.cron} | ${m.description} | ${trigger}`;
+ const timing = m.cron ? `${m.cron} (${m.timezone ?? 'host timezone'})` : `${m.executeAt ?? 'unknown time'} (missed: ${m.missedRunPolicy ?? 'skip'})`;
+ const state = m.lastRun ? `${m.lastRun.status}${m.paused ? ', paused' : ''}${m.lastRun.error ? `: ${m.lastRun.error}` : ''}` : 'pending';
+ return `${m.id} | ${timing} | ${state} | ${m.description} | ${trigger}`;
}).join('\n');
},
});
diff --git a/src/capabilities/scheduler/schedule-task.test.ts b/src/capabilities/scheduler/schedule-task.test.ts
index e090b8c8..ad31b8c3 100644
--- a/src/capabilities/scheduler/schedule-task.test.ts
+++ b/src/capabilities/scheduler/schedule-task.test.ts
@@ -64,3 +64,32 @@ describe('turn grants', () => {
expect(await pm.approvePluginTool('whatsapp__call_owner', 'x')).toBe(false);
});
});
+
+describe('schedule timing policy', () => {
+ it('includes catch-up policy in the approval and saves it', async () => {
+ const { tool, added, asked } = setup();
+ await tool.execute({ delay_seconds: 60, description: 'Brief me', prompt: 'Read the news', missed_run_policy: 'run_once' });
+ expect(asked[0]).toContain('if offline, run once after restart');
+ expect(added[0].missedRunPolicy).toBe('run_once');
+ });
+
+ it('saves and confirms the cron timezone', async () => {
+ const { tool, added, asked } = setup();
+ const result = await tool.execute({ cron: '0 9 * * *', description: 'Brief me', prompt: 'Read the news', timezone: 'America/New_York' });
+ expect(asked[0]).toContain('America/New_York');
+ expect(added[0].timezone).toBe('America/New_York');
+ expect(result).toContain('America/New_York');
+ });
+
+ it.each([
+ { delay_seconds: -1 }, { delay_seconds: Infinity },
+ { cron: 'invalid' }, { cron: '0 9 * * *', timezone: 'Not/A_Zone' },
+ { delay_seconds: 60, missed_run_policy: 'replay_all' },
+ { delay_seconds: 60, timezone: 'UTC' },
+ ])('rejects invalid timing before asking for approval: %o', async (timing) => {
+ const { tool, added, asked } = setup();
+ await tool.execute({ ...timing, description: 'Brief me', prompt: 'Read the news' });
+ expect(asked).toEqual([]);
+ expect(added).toEqual([]);
+ });
+});
diff --git a/src/capabilities/scheduler/schedule-task.ts b/src/capabilities/scheduler/schedule-task.ts
index 05e514a5..11cd444b 100644
--- a/src/capabilities/scheduler/schedule-task.ts
+++ b/src/capabilities/scheduler/schedule-task.ts
@@ -1,6 +1,8 @@
import { tool, zodSchema } from 'ai';
import { z } from 'zod';
import cron from 'node-cron';
+import { randomUUID } from 'node:crypto';
+import { isValidScheduleTimezone } from '../../core/scheduler.js';
import { isPluginToolName } from '../../plugins/tool-bridge.js';
import type { Scheduler, ScheduledTaskManifest } from '../../core/scheduler.js';
import type { PermissionManager } from '../permissions.js';
@@ -10,7 +12,9 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
description: 'Schedule a task. Use "cron" for recurring tasks (e.g. "0 9 * * *" for daily at 9am) or "delay_seconds" for one-shot delayed tasks (e.g. 15 for "remind me in 15 seconds"). Provide exactly one of cron or delay_seconds.',
inputSchema: zodSchema(z.object({
cron: z.string().optional().describe('Cron expression for recurring tasks (e.g. "0 9 * * *" for daily at 9am)'),
- delay_seconds: z.number().optional().describe('Delay in seconds for one-shot tasks (e.g. 15 for "remind me in 15 seconds")'),
+ delay_seconds: z.number().finite().positive().optional().describe('Delay in seconds for one-shot tasks (e.g. 15 for "remind me in 15 seconds")'),
+ timezone: z.string().optional().describe('IANA timezone for recurring cron, e.g. America/New_York. Omit for host local time.'),
+ missed_run_policy: z.enum(['skip', 'run_once']).optional().describe('One-shot behavior if Mercury was offline when due: skip (default) or run_once on restart. Interrupted runs are never retried automatically.'),
description: z.string().describe('Human-readable description of what this task does'),
prompt: z.string().optional().describe('Prompt to send to the agent when the task fires'),
skill_name: z.string().optional().describe('Name of a skill to invoke when the task fires'),
@@ -18,7 +22,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
'Plugin tools the task needs when it fires, approved now so it can run unattended, e.g. ["whatsapp__call_owner"] for "call me in 10 minutes" or "call me when the research is done". Only list tools the task actually needs.',
),
})),
- execute: async ({ cron: cronExpr, delay_seconds, description, prompt, skill_name, allow_tools }) => {
+ execute: async ({ cron: cronExpr, delay_seconds, description, prompt, skill_name, allow_tools, timezone, missed_run_policy }) => {
if (!cronExpr && !delay_seconds) {
return 'Either cron or delay_seconds must be provided.';
}
@@ -26,6 +30,10 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
return 'Provide either cron or delay_seconds, not both.';
}
+ if (delay_seconds !== undefined && (!Number.isFinite(delay_seconds) || delay_seconds <= 0)) return 'delay_seconds must be a positive finite number.';
+ if (cronExpr && !cron.validate(cronExpr)) return `Invalid cron expression: "${cronExpr}".`;
+ if (timezone !== undefined && (!cronExpr || !isValidScheduleTimezone(timezone))) return 'timezone requires a cron schedule and a valid IANA timezone.';
+ if (missed_run_policy !== undefined && (!delay_seconds || !['skip', 'run_once'].includes(missed_run_policy))) return 'missed_run_policy requires a one-shot schedule and skip or run_once.';
if (!prompt && !skill_name) {
return 'Either prompt or skill_name must be provided so the task has something to do.';
}
@@ -42,11 +50,11 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
return 'Only the owner can schedule tasks that act without asking (allow_tools).';
}
const approved = await permissions.requestApproval(
- `Create scheduled task: ${description}${allowTools.length ? ` (when it runs, it may use ${allowTools.join(', ')} without asking again)` : ''}`,
+ `Create scheduled task: ${description}${timezone ? ` (${timezone})` : ''}${missed_run_policy === 'run_once' ? ' (if offline, run once after restart)' : ''}${allowTools.length ? ` (when it runs, it may use ${allowTools.join(', ')} without asking again)` : ''}`,
);
if (!approved) return 'Permission denied: scheduled task creation was not approved.';
- const id = `task-${Date.now().toString(36)}`;
+ const id = `task-${randomUUID()}`;
const ctx = getContext();
if (delay_seconds) {
@@ -56,6 +64,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
prompt,
skillName: skill_name,
delaySeconds: delay_seconds,
+ missedRunPolicy: missed_run_policy ?? 'skip',
executeAt: new Date(Date.now() + delay_seconds * 1000).toISOString(),
createdAt: new Date().toISOString(),
sourceChannelId: ctx.channelId,
@@ -77,6 +86,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
const manifest: ScheduledTaskManifest = {
id,
cron: cronExpr,
+ timezone,
description,
prompt,
skillName: skill_name,
@@ -90,7 +100,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis
scheduler.persistSchedules();
const triggerType = skill_name ? `skill: ${skill_name}` : `prompt: "${prompt!.slice(0, 60)}"`;
- return `Task "${id}" scheduled. Cron: ${cronExpr}. Will execute ${triggerType}. Description: ${description}`;
+ return `Task "${id}" scheduled. Cron: ${cronExpr} (${timezone ?? 'host timezone'}). Will execute ${triggerType}. Description: ${description}`;
},
});
}
diff --git a/src/core/agent.ts b/src/core/agent.ts
index 9c802c2b..8bf26973 100644
--- a/src/core/agent.ts
+++ b/src/core/agent.ts
@@ -12,7 +12,7 @@ import type { ChannelRegistry } from '../channels/registry.js';
import type { MercuryConfig, ProviderName } from '../utils/config.js';
import type { TokenBudget } from '../utils/tokens.js';
import type { CapabilityRegistry } from '../capabilities/registry.js';
-import type { ScheduledTaskManifest } from './scheduler.js';
+import type { ScheduledDispatch, ScheduledTaskManifest } from './scheduler.js';
import { DeepSeekProvider } from '../providers/deepseek.js';
import { ArtifactCollector } from '../artifacts/collector.js';
import { artifactStore, type Artifact } from '../artifacts/store.js';
@@ -671,7 +671,7 @@ export class Agent {
this.notifyBackgroundTaskComplete(task);
});
- this.scheduler.setOnScheduledTask(async (manifest) => this.handleScheduledTask(manifest));
+ this.scheduler.setOnScheduledTask(async (manifest, runId) => this.handleScheduledTask(manifest, runId));
this.channels.onIncomingMessage((msg) => this.enqueueMessage(msg));
this.wireOwnerApprovals();
@@ -2165,6 +2165,18 @@ export class Agent {
}
logger.error({ err, msg: msg.content.slice(0, 50) }, 'Failed to handle message');
} finally {
+ const taskId = msg.metadata?.scheduledTaskId;
+ const runId = msg.metadata?.scheduledRunId;
+ if (typeof taskId === 'string' && typeof runId === 'string') {
+ const entry = item.workKey ? this.workLedger.get(item.workKey) : undefined;
+ try {
+ this.scheduler.finishRun(taskId, runId, entry?.status === 'completed' ? 'completed' : 'failed',
+ entry?.status === 'completed' ? undefined : entry?.error ?? 'Scheduled work ended without confirmed completion');
+ } catch (err) {
+ // Leave the durable running record uncertain, but always release queue state.
+ logger.error({ err, taskId, runId }, 'Could not persist scheduled run outcome');
+ }
+ }
this.currentWorkKey = null;
if (!this.lifecycle.is('idle') && this.lifecycle.canTransitionTo('idle')) {
this.lifecycle.transition('idle');
@@ -4169,7 +4181,7 @@ export class Agent {
return;
}
logger.error({ err: lastError }, errMsg);
- if (this.currentWorkKey && (hasCompletedTool || hasStreamedOutput || requiresContinuationApproval)) {
+ if (this.currentWorkKey && !msg.metadata?.scheduledTaskId && (hasCompletedTool || hasStreamedOutput || requiresContinuationApproval)) {
const continuationAttempt = typeof msg.metadata?.continuationAttempt === 'number' ? msg.metadata.continuationAttempt : 0;
const needsApproval = needsContinuationApproval(continuationAttempt, requiresContinuationApproval);
if (needsApproval) {
@@ -4203,7 +4215,7 @@ export class Agent {
return;
}
}
- if (this.currentWorkKey && !hasCompletedTool && !hasStreamedOutput && this.isRetryableProviderError(lastError)) {
+ if (this.currentWorkKey && !msg.metadata?.scheduledTaskId && !hasCompletedTool && !hasStreamedOutput && this.isRetryableProviderError(lastError)) {
const attempts = this.workLedger.get(this.currentWorkKey)?.attempts ?? 1;
const shouldContinue = !needsRetryApproval(attempts) || (
channel && msg.channelType !== 'internal'
@@ -4969,28 +4981,36 @@ export class Agent {
});
}
- async processInternalPrompt(prompt: string, channelId?: string, channelType?: string, opts: { allowTools?: string[]; scheduledTaskId?: string } = {}): Promise<void> {
+ async processInternalPrompt(prompt: string, channelId?: string, channelType?: string, opts: { allowTools?: string[]; scheduledTaskId?: string; scheduledRunId?: string } = {}): Promise<void> {
const syntheticMsg: ChannelMessage = {
- id: `internal-${Date.now().toString(36)}`,
+ id: opts.scheduledRunId ? `scheduled-${opts.scheduledRunId}` : `internal-${Date.now().toString(36)}`,
channelId: channelId || 'internal',
channelType: (channelType || 'internal') as ChannelType,
senderId: 'system',
content: prompt,
timestamp: Date.now(),
...(opts.allowTools?.length || opts.scheduledTaskId
- ? { metadata: { ...(opts.allowTools?.length ? { allowTools: opts.allowTools } : {}), ...(opts.scheduledTaskId ? { scheduledTaskId: opts.scheduledTaskId } : {}) } }
+ ? { metadata: { ...(opts.allowTools?.length ? { allowTools: opts.allowTools } : {}), ...(opts.scheduledTaskId ? { scheduledTaskId: opts.scheduledTaskId, scheduledRunId: opts.scheduledRunId } : {}) } }
: {}),
};
- this.enqueueMessage(syntheticMsg);
+ if (opts.scheduledTaskId) {
+ // A scheduled prompt is work, never an interactive slash command/dialog answer.
+ const accepted = this.workLedger.accept(syntheticMsg);
+ if (accepted.accepted) {
+ this.queueMessage(syntheticMsg, accepted.entry.key);
+ void this.processQueue();
+ }
+ } else {
+ this.enqueueMessage(syntheticMsg);
+ }
}
- private async handleScheduledTask(manifest: ScheduledTaskManifest): Promise<void> {
+ private async handleScheduledTask(manifest: ScheduledTaskManifest, runId: string): Promise<ScheduledDispatch> {
// Bot routines run on the bot lane, never through the main agent loop
// (BOTS-ARCHITECTURE.md §2.3 — cron lane).
if (manifest.botId) {
if (!this.botManager) {
- logger.warn({ task: manifest.id, botId: manifest.botId }, 'Bot routine fired but BotManager is not wired');
- return;
+ throw new Error('Bot routine fired but BotManager is not wired');
}
logger.info({ task: manifest.id, botId: manifest.botId }, 'Bot routine firing');
const result = this.botManager.enqueue(manifest.botId, {
@@ -5002,8 +5022,10 @@ export class Agent {
// Gate skips (busy, too soon, paused) are the governor working, not faults.
const level = result.reasonCode === 'busy' || result.reasonCode === 'too_soon' || result.reasonCode === 'routine_paused' ? 'info' : 'warn';
logger[level]({ botId: manifest.botId, task: manifest.id, reasonCode: result.reasonCode }, 'Bot routine tick skipped');
+ if (['busy', 'too_soon', 'routine_paused', 'stopped', 'target_disabled'].includes(result.reasonCode ?? '')) return 'skipped';
+ throw new Error(`Bot schedule was not accepted: ${result.reasonCode ?? 'unknown error'}`);
}
- return;
+ return 'dispatched';
}
logger.info({ task: manifest.id, channel: manifest.sourceChannelType }, 'Processing scheduled task');
try {
@@ -5014,7 +5036,7 @@ export class Agent {
if (channel && manifest.sourceChannelType !== 'internal') {
const skillInfo = manifest.skillName ? ` (${manifest.skillName})` : '';
await channel.send(
- ` Scheduled task started${skillInfo}: ${manifest.description}\nAll actions auto-approved for this run.`,
+ ` Scheduled task started${skillInfo}: ${manifest.description}\nExisting tool permissions apply to this run.`,
manifest.sourceChannelId,
).catch((e) => logger.warn({ e }, 'channel send failed'));
}
@@ -5027,9 +5049,11 @@ export class Agent {
if (!prompt) {
prompt = `Execute scheduled task: ${manifest.description}`;
}
- await this.processInternalPrompt(prompt, manifest.sourceChannelId, manifest.sourceChannelType, { allowTools: manifest.allowTools, scheduledTaskId: manifest.id });
+ await this.processInternalPrompt(prompt, manifest.sourceChannelId, manifest.sourceChannelType, { allowTools: manifest.allowTools, scheduledTaskId: manifest.id, scheduledRunId: runId });
+ return 'queued';
} catch (err) {
logger.error({ err, task: manifest.id }, 'Scheduled task execution failed');
+ throw err;
}
}
diff --git a/src/core/scheduled-dispatch.test.ts b/src/core/scheduled-dispatch.test.ts
new file mode 100644
index 00000000..3eb73d52
--- /dev/null
+++ b/src/core/scheduled-dispatch.test.ts
@@ -0,0 +1,66 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { Agent } from './agent.js';
+import { WorkLedger } from './work-ledger.js';
+import { Scheduler, type ScheduledTaskManifest } from './scheduler.js';
+import type { MercuryConfig } from '../utils/config.js';
+
+const dirs: string[] = [];
+afterEach(() => {
+ vi.useRealTimers();
+ vi.unstubAllEnvs();
+ dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }));
+});
+
+/** Exercise the real scheduler -> internal prompt -> durable queue -> finalization path. */
+function harness(fail = false) {
+ const dir = mkdtempSync(join(tmpdir(), 'scheduled-dispatch-'));
+ dirs.push(dir);
+ vi.stubEnv('MERCURY_HOME', dir);
+ const workLedger = new WorkLedger({ filePath: join(dir, 'work-ledger.json') });
+ const agent = Object.create(Agent.prototype) as Agent;
+ const internal = agent as unknown as {
+ currentWorkKey: string;
+ handleScheduledTask(manifest: ScheduledTaskManifest, runId: string): Promise<'queued'>;
+ processQueue(): Promise<void>;
+ };
+ const scheduler = new Scheduler({ heartbeat: { intervalMinutes: 30 } } as MercuryConfig,
+ (manifest, runId) => internal.handleScheduledTask(manifest, runId));
+ const effect = vi.fn();
+ Object.assign(agent, {
+ scheduler, workLedger, messageQueue: [], queuedWorkKeys: new Set(), processing: false, running: false,
+ currentMessage: null, currentWorkKey: null,
+ channels: { get: () => undefined, getNotificationChannel: () => undefined },
+ lifecycle: { is: () => true },
+ announceQueue: () => {},
+ withSessionProgramming: (_msg: unknown, run: () => Promise<void>) => run(),
+ handleMessage: async () => {
+ effect();
+ if (fail) throw new Error('Lost connection after performing an effect');
+ workLedger.markCompleted(internal.currentWorkKey, 'Done');
+ },
+ });
+ return { agent, internal, scheduler, effect, workLedger };
+}
+
+describe('scheduled main-agent dispatch', () => {
+ it.each([false, true])('tracks execution rather than enqueue acceptance (failure=%s)', async (fail) => {
+ vi.useFakeTimers();
+ const { agent, internal, scheduler, effect } = harness(fail);
+ scheduler.addDelayedTask({ id: 'scheduled', description: 'Run work', prompt: '/a prompt is not a slash command', delaySeconds: 1, createdAt: new Date().toISOString() });
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(scheduler.getManifest('scheduled')?.lastRun?.status).toBe('running');
+ expect(effect).not.toHaveBeenCalled();
+ Object.assign(agent, { running: true });
+ await internal.processQueue();
+ expect(effect).toHaveBeenCalledOnce();
+ expect(scheduler.getManifest('scheduled')?.lastRun?.status).toBe(fail ? 'failed' : 'completed');
+ scheduler.stopAll();
+ scheduler.restorePersistedTasks();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(effect).toHaveBeenCalledOnce();
+ scheduler.stopAll();
+ });
+});
diff --git a/src/core/scheduler.test.ts b/src/core/scheduler.test.ts
new file mode 100644
index 00000000..fb40564a
--- /dev/null
+++ b/src/core/scheduler.test.ts
@@ -0,0 +1,156 @@
+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { MercuryConfig } from '../utils/config.js';
+import { loadSchedules, saveSchedules, Scheduler, type ScheduledTaskManifest } from './scheduler.js';
+
+const config = { heartbeat: { intervalMinutes: 30 } } as MercuryConfig;
+const NOW = new Date('2026-11-06T13:59:58Z');
+let home: string;
+let schedulers: Scheduler[];
+function create(handler: ConstructorParameters<typeof Scheduler>[1] = async () => {}) {
+ const scheduler = new Scheduler(config, handler);
+ schedulers.push(scheduler);
+ return scheduler;
+}
+function task(overrides: Partial<ScheduledTaskManifest> = {}): ScheduledTaskManifest {
+ return { id: 'reminder', description: 'Remind me', prompt: 'Notify me', delaySeconds: 1, executeAt: new Date(Date.now() + 1000).toISOString(), createdAt: NOW.toISOString(), ...overrides };
+}
+beforeEach(() => {
+ home = mkdtempSync(join(tmpdir(), 'mercury-scheduler-'));
+ vi.stubEnv('MERCURY_HOME', home);
+ vi.useFakeTimers();
+ vi.setSystemTime(NOW);
+ schedulers = [];
+});
+afterEach(() => {
+ schedulers.forEach((scheduler) => scheduler.stopAll());
+ vi.useRealTimers();
+ vi.unstubAllEnvs();
+ rmSync(home, { recursive: true, force: true });
+});
+
+describe('durable schedules', () => {
+ it('persists running intent before invoking an effect, then retains confirmed completion', async () => {
+ const handler = vi.fn(async () => {
+ expect(loadSchedules()[0].lastRun?.status).toBe('running');
+ });
+ const scheduler = create(handler);
+ scheduler.addDelayedTask(task());
+ scheduler.persistSchedules();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(handler).toHaveBeenCalledOnce();
+ expect(loadSchedules()[0].lastRun).toMatchObject({ status: 'completed', finishedAt: new Date().toISOString() });
+ if (process.platform !== 'win32') expect(statSync(join(home, 'schedules.yaml')).mode & 0o777).toBe(0o600);
+ expect(readFileSync(join(home, 'schedules.yaml'), 'utf8')).toContain('completed');
+ });
+
+ it('retains a failed one-shot and never repeats it on restart', async () => {
+ const handler = vi.fn(async () => { throw new Error('Connection lost after sending'); });
+ const scheduler = create(handler);
+ scheduler.addDelayedTask(task({ missedRunPolicy: 'run_once' }));
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(loadSchedules()[0]).toMatchObject({ paused: true, lastRun: { status: 'failed', error: 'Connection lost after sending' } });
+ scheduler.stopAll();
+ create(handler).restorePersistedTasks();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(handler).toHaveBeenCalledOnce();
+ });
+
+ it('marks unconfirmed work interrupted and pauses it even with catch-up enabled', async () => {
+ const handler = vi.fn(async () => 'queued' as const);
+ const first = create(handler);
+ first.addDelayedTask(task({ missedRunPolicy: 'run_once' }));
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(loadSchedules()[0].lastRun?.status).toBe('running');
+ first.stopAll();
+ const restarted = create(handler);
+ restarted.restorePersistedTasks();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(handler).toHaveBeenCalledOnce();
+ expect(restarted.getManifest('reminder')).toMatchObject({ paused: true, lastRun: { status: 'interrupted' } });
+ // A metadata-only edit must not turn uncertain side effects into a retry.
+ restarted.replaceManifest({ ...restarted.getManifest('reminder')!, description: 'renamed' });
+ await vi.advanceTimersByTimeAsync(2000);
+ expect(handler).toHaveBeenCalledOnce();
+ restarted.replaceManifest(restarted.getManifest('reminder')!, true);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(handler).toHaveBeenCalledTimes(2);
+ });
+
+ it('skips legacy overdue one-shots but catches up explicitly opted-in work exactly once', async () => {
+ saveSchedules([
+ task({ id: 'legacy', executeAt: new Date(Date.now() - 1000).toISOString() }),
+ task({ id: 'catch-up', missedRunPolicy: 'run_once', executeAt: new Date(Date.now() - 1000).toISOString() }),
+ ]);
+ const handler = vi.fn(async () => {});
+ const first = create(handler);
+ first.restorePersistedTasks();
+ await vi.advanceTimersByTimeAsync(1);
+ expect(handler).toHaveBeenCalledOnce();
+ expect(handler).toHaveBeenCalledWith(expect.objectContaining({ id: 'catch-up' }), expect.any(String));
+ expect(first.getManifest('legacy')?.lastRun?.status).toBe('skipped');
+ first.stopAll();
+ create(handler).restorePersistedTasks();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(handler).toHaveBeenCalledOnce();
+ });
+
+ it('does not overlap a main-agent run that has only been queued', async () => {
+ const handler = vi.fn(async () => 'queued' as const);
+ const scheduler = create(handler);
+ scheduler.addPersistedTask(task({ delaySeconds: undefined, executeAt: undefined, cron: '* * * * * *' }));
+ await vi.advanceTimersByTimeAsync(4000);
+ expect(handler).toHaveBeenCalledOnce();
+ const runId = scheduler.getManifest('reminder')!.lastRun!.id;
+ scheduler.finishRun('reminder', runId, 'completed');
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(handler).toHaveBeenCalledTimes(2);
+ scheduler.finishRun('reminder', runId, 'failed', 'stale result');
+ expect(scheduler.getManifest('reminder')!.lastRun!.status).toBe('running');
+ });
+
+ it('fires cron in the selected IANA zone, independent of the host timezone', async () => {
+ const handler = vi.fn(async () => {});
+ create(handler).addPersistedTask(task({ delaySeconds: undefined, executeAt: undefined, cron: '0 9 * * *', timezone: 'America/New_York' }));
+ await vi.advanceTimersByTimeAsync(2000); // 14:00 UTC = 09:00 EST
+ expect(handler).toHaveBeenCalledOnce();
+ });
+
+ it('chunks delays beyond Node’s timer limit without firing early', async () => {
+ const handler = vi.fn(async () => {});
+ const delaySeconds = 30 * 24 * 3600;
+ create(handler).addDelayedTask(task({ delaySeconds, executeAt: new Date(Date.now() + delaySeconds * 1000).toISOString() }));
+ await vi.advanceTimersByTimeAsync(2 ** 31 - 1);
+ expect(handler).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(delaySeconds * 1000 - (2 ** 31 - 1));
+ expect(handler).toHaveBeenCalledOnce();
+ });
+
+ it('bounds resolved one-shot history while retaining active and unresolved work', () => {
+ const finished = Array.from({ length: 105 }, (_, i) => task({ id: `done-${i}`, lastRun: { id: String(i), status: 'completed', startedAt: NOW.toISOString(), finishedAt: NOW.toISOString() } }));
+ saveSchedules([...finished, task({ id: 'pending' }), task({ id: 'failed', paused: true, lastRun: { id: 'f', status: 'failed', startedAt: NOW.toISOString() } })]);
+ const scheduler = create();
+ scheduler.restorePersistedTasks();
+ expect(scheduler.getManifests()).toHaveLength(102);
+ expect(scheduler.getManifest('pending')).not.toBeNull();
+ expect(scheduler.getManifest('failed')).not.toBeNull();
+ });
+
+ it('leaves malformed persisted schedules untouched instead of overwriting recovery evidence', () => {
+ const path = join(home, 'schedules.yaml');
+ const invalid = 'tasks: [malformed';
+ writeFileSync(path, invalid);
+ create().restorePersistedTasks();
+ expect(readFileSync(path, 'utf8')).toBe(invalid);
+ });
+
+ it('validates before changing an existing schedule', () => {
+ const scheduler = create();
+ scheduler.addDelayedTask(task());
+ expect(() => scheduler.replaceManifest(task({ delaySeconds: Infinity }))).toThrow();
+ expect(() => scheduler.replaceManifest(task({ timezone: 'Mars/Olympus' }))).toThrow();
+ expect(scheduler.getManifest('reminder')?.delaySeconds).toBe(1);
+ });
+});
diff --git a/src/core/scheduler.ts b/src/core/scheduler.ts
index b14720cc..f667518e 100644
--- a/src/core/scheduler.ts
+++ b/src/core/scheduler.ts
@@ -1,5 +1,6 @@
import cron from 'node-cron';
-import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { randomUUID } from 'node:crypto';
import { join } from 'node:path';
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
import type { MercuryConfig } from '../utils/config.js';
@@ -11,8 +12,21 @@ export interface ScheduledTask {
cron: string;
handler: () => Promise<void>;
description: string;
+ timezone?: string;
}
+export type ScheduledRunStatus = 'running' | 'completed' | 'failed' | 'interrupted' | 'skipped' | 'dispatched';
+export interface ScheduledRun {
+ id: string;
+ status: ScheduledRunStatus;
+ startedAt: string;
+ finishedAt?: string;
+ error?: string;
+}
+
+/** Queued main-agent work completes via finishRun; bots own a separate work ledger. */
+export type ScheduledDispatch = void | 'queued' | 'dispatched' | 'skipped';
+
export interface ScheduledTaskManifest {
id: string;
cron?: string;
@@ -22,6 +36,12 @@ export interface ScheduledTaskManifest {
delaySeconds?: number;
executeAt?: string;
createdAt: string;
+ /** IANA timezone for cron; omitted means the host timezone for legacy schedules. */
+ timezone?: string;
+ /** Overdue one-shots default to skip, preserving the legacy no-replay behavior. */
+ missedRunPolicy?: 'skip' | 'run_once';
+ paused?: boolean;
+ lastRun?: ScheduledRun;
sourceChannelId?: string;
sourceChannelType?: string;
/** Set for Mercury Bot routines — the run routes to the bot, not the main agent. */
@@ -46,7 +66,7 @@ export function loadSchedules(): ScheduledTaskManifest[] {
try {
const raw = readFileSync(path, 'utf-8');
const data = parseYaml(raw) as { tasks?: ScheduledTaskManifest[] };
- return data.tasks || [];
+ return Array.isArray(data?.tasks) ? data.tasks : [];
} catch (err) {
logger.warn({ err }, 'Failed to load schedules.yaml');
return [];
@@ -55,13 +75,37 @@ export function loadSchedules(): ScheduledTaskManifest[] {
export function saveSchedules(tasks: ScheduledTaskManifest[]): void {
const path = getSchedulesPath();
- const dir = getMercuryHome();
- if (!existsSync(dir)) {
- mkdirSync(dir, { recursive: true });
+ mkdirSync(getMercuryHome(), { recursive: true, mode: 0o700 });
+ const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
+ let fd: number | undefined;
+ try {
+ fd = openSync(temp, 'wx', 0o600);
+ writeFileSync(fd, stringifyYaml({ tasks }), 'utf-8');
+ fsyncSync(fd);
+ closeSync(fd);
+ fd = undefined;
+ renameSync(temp, path);
+ // Directory fsync is unavailable on some platforms (notably Windows).
+ try {
+ fd = openSync(getMercuryHome(), 'r');
+ fsyncSync(fd);
+ } catch { /* the file itself has already been synced */ }
+ } finally {
+ if (fd !== undefined) closeSync(fd);
+ if (existsSync(temp)) unlinkSync(temp);
}
- writeFileSync(path, stringifyYaml({ tasks }), 'utf-8');
}
+export function isValidScheduleTimezone(timezone: string): boolean {
+ try {
+ new Intl.DateTimeFormat('en', { timeZone: timezone }).format();
+ return true;
+ } catch { return false; }
+}
+
+const MAX_TIMER_MS = 2 ** 31 - 1;
+const MAX_FINISHED_ONESHOTS = 100;
+
export class Scheduler {
private tasks: Map<string, cron.ScheduledTask> = new Map();
private delayedTasks: Map<string, NodeJS.Timeout> = new Map();
@@ -72,12 +116,12 @@ export class Scheduler {
constructor(
config: MercuryConfig,
- private onScheduledTask?: (manifest: ScheduledTaskManifest) => Promise<void>,
+ private onScheduledTask?: (manifest: ScheduledTaskManifest, runId: string) => Promise<ScheduledDispatch>,
) {
this.heartbeatIntervalMinutes = config.heartbeat.intervalMinutes;
}
- setOnScheduledTask(handler: (manifest: ScheduledTaskManifest) => Promise<void>): void {
+ setOnScheduledTask(handler: (manifest: ScheduledTaskManifest, runId: string) => Promise<ScheduledDispatch>): void {
this.onScheduledTask = handler;
}
@@ -108,56 +152,93 @@ export class Scheduler {
}
addTask(task: ScheduledTask): void {
- if (this.tasks.has(task.id)) {
- this.removeTask(task.id);
- }
+ this.tasks.get(task.id)?.stop();
+ let running = false;
const scheduled = cron.schedule(task.cron, async () => {
+ if (running) return; // Cron ticks never build an overlapping backlog.
+ running = true;
try {
await task.handler();
} catch (err) {
logger.error({ task: task.id, err }, 'Scheduled task error');
- }
- });
+ } finally { running = false; }
+ }, { timezone: task.timezone });
this.tasks.set(task.id, scheduled);
- logger.info({ id: task.id, cron: task.cron, desc: task.description }, 'Task scheduled');
+ }
+
+ private validate(manifest: ScheduledTaskManifest): void {
+ if (manifest.timezone !== undefined && !isValidScheduleTimezone(manifest.timezone)) {
+ throw new Error('Invalid schedule timezone');
+ }
+ if (manifest.missedRunPolicy !== undefined && !['skip', 'run_once'].includes(manifest.missedRunPolicy)) {
+ throw new Error('Invalid missed-run policy');
+ }
+ if (manifest.cron && cron.validate(manifest.cron) && manifest.delaySeconds === undefined) return;
+ if (!manifest.cron && Number.isFinite(manifest.delaySeconds) && manifest.delaySeconds! > 0
+ && (!manifest.executeAt || Number.isFinite(Date.parse(manifest.executeAt)))) return;
+ throw new Error('Manifest requires a valid cron or positive finite delaySeconds');
}
addPersistedTask(manifest: ScheduledTaskManifest): void {
- this.taskManifests.set(manifest.id, manifest);
+ this.validate(manifest);
+ // Registering a bot's declared routine again must not erase interrupted state.
+ const previous = this.taskManifests.get(manifest.id);
+ const next = { ...manifest, lastRun: manifest.lastRun ?? previous?.lastRun, paused: manifest.paused ?? previous?.paused };
+ this.removeTask(manifest.id);
+ this.taskManifests.set(manifest.id, next);
+ if (next.paused) return;
this.addTask({
- id: manifest.id,
- cron: manifest.cron!,
- description: manifest.description,
- handler: async () => {
- logger.info({ task: manifest.id }, 'Scheduled task firing');
- if (this.onScheduledTask) {
- await this.onScheduledTask(manifest);
- }
- },
+ id: next.id, cron: next.cron!, description: next.description, timezone: next.timezone,
+ handler: () => this.runTask(next.id),
});
}
addDelayedTask(manifest: ScheduledTaskManifest): void {
- this.taskManifests.set(manifest.id, manifest);
- const delayMs = (manifest.delaySeconds || 60) * 1000;
+ this.validate(manifest);
+ const next = { ...manifest, executeAt: manifest.executeAt ?? new Date(Date.now() + manifest.delaySeconds! * 1000).toISOString() };
+ this.removeTask(next.id);
+ this.taskManifests.set(next.id, next);
+ if (!next.paused && !next.lastRun) this.armDelayedTask(next);
+ }
- const timer = setTimeout(async () => {
- try {
- logger.info({ task: manifest.id }, 'Delayed task firing');
- if (this.onScheduledTask) {
- await this.onScheduledTask(manifest);
- }
- } catch (err) {
- logger.error({ task: manifest.id, err }, 'Delayed task error');
- } finally {
- this.delayedTasks.delete(manifest.id);
- this.taskManifests.delete(manifest.id);
- this.persistSchedules();
+ private armDelayedTask(manifest: ScheduledTaskManifest): void {
+ const remaining = Date.parse(manifest.executeAt!) - Date.now();
+ const timer = setTimeout(() => {
+ this.delayedTasks.delete(manifest.id);
+ if (this.taskManifests.get(manifest.id) !== manifest) return;
+ if (Date.parse(manifest.executeAt!) > Date.now()) {
+ this.armDelayedTask(manifest); // Node timers longer than ~24 days otherwise fire immediately.
+ } else {
+ void this.runTask(manifest.id).catch((err) => logger.error({ err, task: manifest.id }, 'Could not persist scheduled run'));
}
- }, delayMs);
-
+ }, Math.min(MAX_TIMER_MS, Math.max(0, remaining)));
this.delayedTasks.set(manifest.id, timer);
- logger.info({ id: manifest.id, delaySeconds: manifest.delaySeconds }, 'Delayed task scheduled');
+ }
+
+ private async runTask(id: string): Promise<void> {
+ const manifest = this.taskManifests.get(id);
+ if (!manifest || manifest.paused || manifest.lastRun?.status === 'running') return;
+ const runId = randomUUID();
+ manifest.lastRun = { id: runId, status: 'running', startedAt: new Date().toISOString() };
+ // Fail closed: no handler runs unless the intent is durable first.
+ this.persistSchedules();
+ try {
+ if (!this.onScheduledTask) throw new Error('Scheduled task handler unavailable');
+ const outcome = await this.onScheduledTask(manifest, runId);
+ if (outcome !== 'queued') this.finishRun(id, runId, outcome ?? 'completed');
+ } catch (err) {
+ this.finishRun(id, runId, 'failed', err instanceof Error ? err.message : String(err));
+ logger.error({ err, task: id }, 'Scheduled task failed; review before resuming');
+ }
+ }
+
+ /** A run identity prevents a late completion from changing an edited/deleted task. */
+ finishRun(id: string, runId: string, status: Exclude<ScheduledRunStatus, 'running'>, error?: string): void {
+ const manifest = this.taskManifests.get(id);
+ if (!manifest || manifest.lastRun?.id !== runId || manifest.lastRun.status !== 'running') return;
+ manifest.lastRun = { ...manifest.lastRun, status, finishedAt: new Date().toISOString(), ...(error ? { error: error.slice(0, 1000) } : {}) };
+ if (status === 'failed' || status === 'interrupted') manifest.paused = true;
+ this.persistSchedules();
}
removeTask(id: string): void {
@@ -182,45 +263,53 @@ export class Scheduler {
return this.taskManifests.get(id) || null;
}
- replaceManifest(manifest: ScheduledTaskManifest): void {
- this.removeTask(manifest.id);
- if (manifest.delaySeconds) {
- const delay = Math.max(1, manifest.delaySeconds);
- manifest.delaySeconds = delay;
- manifest.executeAt = new Date(Date.now() + delay * 1000).toISOString();
- this.addDelayedTask(manifest);
- } else if (manifest.cron) {
- this.addPersistedTask(manifest);
- } else {
- throw new Error('Manifest requires cron or delaySeconds');
+ replaceManifest(manifest: ScheduledTaskManifest, resume = false): void {
+ this.validate(manifest);
+ if (this.getManifest(manifest.id)?.lastRun?.status === 'running') throw new Error('Cannot edit a running schedule');
+ const next = { ...manifest };
+ if (resume) {
+ next.paused = false;
+ next.lastRun = undefined;
+ if (next.delaySeconds) next.executeAt = new Date(Date.now() + next.delaySeconds * 1000).toISOString();
}
+ this.removeTask(next.id);
+ if (next.cron) this.addPersistedTask(next);
+ else this.addDelayedTask(next);
}
restorePersistedTasks(): void {
const persisted = loadSchedules();
- for (const manifest of persisted) {
- if (manifest.delaySeconds) {
- const executeAt = manifest.executeAt ? new Date(manifest.executeAt) : null;
- const now = Date.now();
- if (executeAt && executeAt.getTime() > now) {
- const remainingMs = executeAt.getTime() - now;
- manifest.delaySeconds = Math.ceil(remainingMs / 1000);
- this.addDelayedTask(manifest);
+ for (const stored of persisted) {
+ const manifest = { ...stored };
+ try {
+ this.validate(manifest);
+ if (manifest.lastRun?.status === 'running') {
+ manifest.lastRun = { ...manifest.lastRun, status: 'interrupted', finishedAt: new Date().toISOString(), error: 'Mercury stopped before this run was confirmed. Check its effects before resuming.' };
+ manifest.paused = true;
+ }
+ if (manifest.cron) {
+ this.addPersistedTask(manifest);
} else {
- logger.info({ id: manifest.id }, 'Delayed task already expired, skipping');
+ // Old schedules without executeAt cannot safely be inferred as pending.
+ const overdue = !manifest.executeAt || Date.parse(manifest.executeAt) <= Date.now();
+ if (!manifest.lastRun && overdue && (manifest.missedRunPolicy ?? 'skip') === 'skip') {
+ manifest.lastRun = { id: randomUUID(), status: 'skipped', startedAt: new Date().toISOString(), finishedAt: new Date().toISOString(), error: 'Missed while Mercury was offline (policy: skip).' };
+ }
+ if (overdue && !manifest.lastRun) manifest.executeAt = new Date().toISOString();
+ this.addDelayedTask(manifest);
}
- } else if (manifest.cron && cron.validate(manifest.cron)) {
- this.addPersistedTask(manifest);
- } else {
- logger.warn({ id: manifest.id, cron: manifest.cron }, 'Skipping invalid task');
+ } catch (err) {
+ logger.warn({ err, id: manifest.id }, 'Skipping invalid task');
}
}
- if (persisted.length > 0) {
- logger.info({ count: persisted.length }, 'Restored persisted scheduled tasks');
- }
+ if (persisted.length > 0) this.persistSchedules();
}
persistSchedules(): void {
+ // Keep active work and all unresolved failures. Bound successful/skipped one-shot history.
+ const finished = this.getManifests().filter((m) => !m.cron && m.lastRun && ['completed', 'skipped', 'dispatched'].includes(m.lastRun.status))
+ .sort((a, b) => (b.lastRun!.finishedAt ?? '').localeCompare(a.lastRun!.finishedAt ?? ''));
+ for (const manifest of finished.slice(MAX_FINISHED_ONESHOTS)) this.removeTask(manifest.id);
saveSchedules(this.getManifests());
}
diff --git a/src/core/work-ledger.test.ts b/src/core/work-ledger.test.ts
index 3e91052e..cfaa3fb9 100644
--- a/src/core/work-ledger.test.ts
+++ b/src/core/work-ledger.test.ts
@@ -214,3 +214,17 @@ describe('WorkLedger', () => {
expect(ledger.get(entry.key)?.status).toBe('cancelled');
});
});
+
+describe('scheduled work recovery', () => {
+ it.each(['queued', 'running', 'paused', 'failed'] as const)('never replays %s scheduled work after restart', (state) => {
+ const { ledger, filePath } = setup();
+ const entry = ledger.accept(message('schedule', { scheduledTaskId: 'task-1', scheduledRunId: 'run-1' })).entry;
+ if (state === 'running') ledger.markRunning(entry.key);
+ if (state === 'paused') ledger.markPaused(entry.key, 'Side effects may be partial');
+ if (state === 'failed') ledger.markFailed(entry.key, 'interrupted/ambiguous');
+ const restarted = new WorkLedger({ filePath });
+ expect(restarted.recoverInterrupted()).toEqual([]);
+ expect(restarted.get(entry.key)).toMatchObject({ status: 'failed', error: expect.stringContaining('explicitly reschedule') });
+ expect(new WorkLedger({ filePath }).recoverInterrupted()).toEqual([]);
+ });
+});
diff --git a/src/core/work-ledger.ts b/src/core/work-ledger.ts
index 192e6dd5..02571e5e 100644
--- a/src/core/work-ledger.ts
+++ b/src/core/work-ledger.ts
@@ -239,6 +239,12 @@ export class WorkLedger {
const recoverableFailure = entry.status === 'failed' && /interrupted\/ambiguous|side effects may be partial|after one or more tools completed|partial output/i.test(entry.error || '');
const isPaused = entry.status === 'paused';
if (entry.status !== 'queued' && entry.status !== 'running' && !recoverableFailure && !isPaused) continue;
+ if (typeof entry.message.metadata?.scheduledTaskId === 'string') {
+ // The scheduler owns recovery. Never replay a turn whose external effects
+ // may have happened before the process stopped, even if it was queued.
+ this.markFailed(entry.key, 'Scheduled run interrupted; inspect its effects and explicitly reschedule it.');
+ continue;
+ }
const interrupted = entry.status === 'running' || isPaused;
entry.status = 'queued';
entry.updatedAt = this.now();
diff --git a/src/eval/scenarios/scheduled-work.test.ts b/src/eval/scenarios/scheduled-work.test.ts
new file mode 100644
index 00000000..94772c08
--- /dev/null
+++ b/src/eval/scenarios/scheduled-work.test.ts
@@ -0,0 +1,111 @@
+import { afterAll, describe, expect, it, vi } from 'vitest';
+// Import the harness before production modules to establish its scratch home.
+import { EvalHarness } from '../runner.js';
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { loadSchedules } from '../../core/scheduler.js';
+import { WorkLedger } from '../../core/work-ledger.js';
+import { call, textStep, toolStep } from '../fixture.js';
+
+afterAll(async () => (await import('../env.js')).removeEvalRoot());
+
+async function settleUntil(done: () => boolean): Promise<void> {
+ const deadline = performance.now() + 5_000;
+ while (!done()) {
+ if (performance.now() > deadline) throw new Error('Scheduled agent turn did not settle');
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ vi.advanceTimersByTime(0);
+ }
+}
+
+describe.each([false, true])('eval · scheduled real work (streaming=%s)', (streaming) => {
+ it.each([false, true])('persists the actual agent outcome and never replays the one-shot (provider fails=%s)', async (fails) => {
+ const harness = await EvalHarness.create({
+ name: `scheduled-real-work-${streaming}-${fails}`,
+ channel: { streaming },
+ providers: [{ name: 'primary' }, { name: 'backup' }],
+ turns: [],
+ });
+ const target = join(harness.workDir, 'scheduled-receipt.txt');
+ const observedAfterWrite: Array<{ status: string | undefined; content: string }> = [];
+ const scheduler = harness.agent.scheduler;
+ try {
+ harness.player.load({
+ message: 'Create scheduled-receipt.txt containing accepted.',
+ steps: [toolStep([call('create_file', { path: 'scheduled-receipt.txt', content: 'accepted' })])],
+ onExhausted: () => {
+ // This is the next real SDK model call, after the real filesystem
+ // tool has returned. Enqueueing alone must not complete the run.
+ observedAfterWrite.push({ status: loadSchedules()[0]?.lastRun?.status, content: readFileSync(target, 'utf8') });
+ if (fails) throw new Error('Provider connection lost after scheduled write');
+ return textStep('Created scheduled-receipt.txt with accepted.');
+ },
+ });
+ scheduler.addDelayedTask({
+ id: 'scheduled-receipt',
+ description: 'Write a receipt once',
+ prompt: 'Create scheduled-receipt.txt containing accepted.',
+ delaySeconds: 1,
+ // Due now, so only zero-delay fake timers need to run. No heartbeat,
+ // watchdog, or guessed sleep participates in the test.
+ executeAt: new Date(Date.now() - 1).toISOString(),
+ createdAt: new Date().toISOString(),
+ missedRunPolicy: 'run_once',
+ sourceChannelId: harness.channelId,
+ sourceChannelType: harness.channelType,
+ });
+ scheduler.persistSchedules();
+ vi.advanceTimersByTime(0);
+ await settleUntil(() => {
+ const status = scheduler.getManifest('scheduled-receipt')?.lastRun?.status;
+ return status === 'completed' || status === 'failed';
+ });
+
+ expect(observedAfterWrite).toEqual([{ status: 'running', content: 'accepted' }]);
+ expect(harness.capabilities.toolLog).toHaveLength(1);
+ expect(harness.capabilities.toolLog[0]).toMatchObject({ tool: 'create_file', recorded: false });
+ expect(harness.player.calls).toHaveLength(2);
+ expect(harness.providers[1].callCount).toBe(0);
+ expect(readFileSync(target, 'utf8')).toBe('accepted');
+
+ const saved = loadSchedules()[0];
+ expect(saved.lastRun?.status).toBe(fails ? 'failed' : 'completed');
+ expect(saved.lastRun?.finishedAt).toBeTruthy();
+ if (fails) {
+ expect(saved.paused).toBe(true);
+ expect(saved.lastRun?.error).toContain('interrupted/ambiguous');
+ expect(harness.channel.events.some((event) => event.text.includes('interrupted/ambiguous'))).toBe(true);
+ expect(harness.channel.events.some((event) => event.text.includes('Task complete'))).toBe(false);
+ } else {
+ expect(harness.channel.events.some((event) => event.kind === 'delivered' && event.text.includes('Created scheduled-receipt.txt'))).toBe(true);
+ }
+
+ const workKey = `channel:${harness.channelType}:scheduled-${saved.lastRun!.id}`;
+ const reloadedLedger = new WorkLedger();
+ expect(reloadedLedger.get(workKey)).toMatchObject({
+ status: fails ? 'failed' : 'completed',
+ attempts: 1,
+ message: { metadata: { scheduledTaskId: saved.id, scheduledRunId: saved.lastRun!.id } },
+ });
+ expect(reloadedLedger.get(workKey)?.nextAttemptAt).toBeUndefined();
+ // In particular, the ordinary chat interrupted-work recovery path must
+ // not enqueue a failed scheduled turn for a second side effect.
+ expect(reloadedLedger.recoverInterrupted()).toEqual([]);
+
+ writeFileSync(target, 'later owner edit');
+ scheduler.stopAll();
+ scheduler.restorePersistedTasks();
+ // This passes every ordinary short retry delay without a live heartbeat.
+ // It also catches a restored overdue run_once timer being armed again.
+ vi.advanceTimersByTime(60_000);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(scheduler.getManifest(saved.id)?.lastRun).toEqual(saved.lastRun);
+ expect(harness.player.calls).toHaveLength(2);
+ expect(harness.capabilities.toolLog).toHaveLength(1);
+ expect(readFileSync(target, 'utf8')).toBe('later owner edit');
+ } finally {
+ await harness.dispose();
+ }
+ });
+});
diff --git a/src/web/api/schedules-route.test.ts b/src/web/api/schedules-route.test.ts
index ca41e01f..f99dcf81 100644
--- a/src/web/api/schedules-route.test.ts
+++ b/src/web/api/schedules-route.test.ts
@@ -1,5 +1,10 @@
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
import system, { setScheduler } from './system.js';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Scheduler } from '../../core/scheduler.js';
+import type { MercuryConfig } from '../../utils/config.js';
/** The status bar's schedule list deletes through this route. */
function fakeScheduler(ids: string[]) {
@@ -29,3 +34,42 @@ describe('DELETE /api/schedules/:id', () => {
expect((await system.request('http://127.0.0.1/api/schedules/zzz', { method: 'DELETE' })).status).toBe(404);
});
});
+
+const dirs: string[] = [];
+const realSchedulers: Scheduler[] = [];
+afterEach(() => {
+ realSchedulers.splice(0).forEach((scheduler) => scheduler.stopAll());
+ setScheduler(null);
+ vi.unstubAllEnvs();
+ dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }));
+});
+function realSchedule() {
+ const dir = mkdtempSync(join(tmpdir(), 'schedule-route-'));
+ dirs.push(dir);
+ vi.stubEnv('MERCURY_HOME', dir);
+ const scheduler = new Scheduler({ heartbeat: { intervalMinutes: 30 } } as MercuryConfig);
+ realSchedulers.push(scheduler);
+ scheduler.addDelayedTask({ id: 'failed', description: 'Remind me', prompt: 'Notify', delaySeconds: 60, executeAt: '2026-01-01T00:00:00Z', createdAt: '2025-12-31T00:00:00Z', paused: true, lastRun: { id: 'run', status: 'failed', startedAt: '2026-01-01T00:00:00Z', error: 'Unknown send outcome' } });
+ setScheduler(scheduler);
+ return scheduler;
+}
+function update(body: Record<string, unknown>) {
+ return system.request('http://127.0.0.1/api/schedules/failed', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+}
+describe('schedule policy editing', () => {
+ it('keeps failed work paused on metadata edits; explicit resume rearms it', async () => {
+ const scheduler = realSchedule();
+ expect((await update({ description: 'Renamed', missedRunPolicy: 'run_once' })).status).toBe(200);
+ expect(scheduler.getManifest('failed')).toMatchObject({ paused: true, lastRun: { status: 'failed' }, executeAt: '2026-01-01T00:00:00Z' });
+ expect((await update({ resume: true, delaySeconds: 300 })).status).toBe(200);
+ expect(scheduler.getManifest('failed')).toMatchObject({ paused: false, delaySeconds: 300 });
+ expect(scheduler.getManifest('failed')?.lastRun).toBeUndefined();
+ expect(Date.parse(scheduler.getManifest('failed')!.executeAt!)).toBeGreaterThan(Date.now());
+ });
+
+ it.each([{ timezone: 'Not/A_Zone' }, { missedRunPolicy: 'replay_all' }, { delaySeconds: -1 }, { delaySeconds: '60' }, { resume: 'yes' }, { cron: '0 9 * * *', delaySeconds: 60 }])('rejects invalid options without changing the task: %o', async (body) => {
+ const scheduler = realSchedule();
+ expect((await update(body)).status).toBe(400);
+ expect(scheduler.getManifest('failed')).toMatchObject({ description: 'Remind me', paused: true, lastRun: { status: 'failed' } });
+ });
+});
diff --git a/src/web/api/system.ts b/src/web/api/system.ts
index 37ba594c..56aface6 100644
--- a/src/web/api/system.ts
+++ b/src/web/api/system.ts
@@ -12,7 +12,7 @@ import { SkillStore } from '../../skills/store.js';
import { PermissionManager, type PermissionsManifest } from '../../capabilities/permissions.js';
import { getMercuryHome } from '../../utils/config.js';
import { getStatus } from './status.js';
-import type { Scheduler } from '../../core/scheduler.js';
+import { isValidScheduleTimezone, type Scheduler } from '../../core/scheduler.js';
import cron from 'node-cron';
const system = new Hono();
@@ -286,6 +286,18 @@ system.put('/api/schedules/:id', async (c) => {
if (!existing) return c.json({ success: false, error: 'Schedule not found' }, 404);
const body = await c.req.json();
+ if (existing.lastRun?.status === 'running') return c.json({ success: false, error: 'Cannot edit a running schedule' }, 409);
+ if (body.resume !== undefined && typeof body.resume !== 'boolean') return c.json({ success: false, error: 'resume must be boolean' }, 400);
+ if (body.timezone !== undefined && (typeof body.timezone !== 'string' || (body.timezone !== '' && !isValidScheduleTimezone(body.timezone)))) {
+ return c.json({ success: false, error: 'Invalid timezone' }, 400);
+ }
+ if (body.missedRunPolicy !== undefined && !['skip', 'run_once'].includes(body.missedRunPolicy)) {
+ return c.json({ success: false, error: 'Invalid missed-run policy' }, 400);
+ }
+ if (body.delaySeconds !== undefined && (typeof body.delaySeconds !== 'number' || !Number.isFinite(body.delaySeconds) || body.delaySeconds <= 0)) {
+ return c.json({ success: false, error: 'delaySeconds must be a positive finite number' }, 400);
+ }
+ if (body.cron && body.delaySeconds !== undefined) return c.json({ success: false, error: 'Provide cron or delaySeconds, not both' }, 400);
const description = typeof body.description === 'string' && body.description.trim()
? body.description.trim()
: existing.description;
@@ -320,10 +332,15 @@ system.put('/api/schedules/:id', async (c) => {
skillName,
cron: cronExpr,
delaySeconds,
- executeAt: delaySeconds ? new Date(Date.now() + delaySeconds * 1000).toISOString() : undefined,
+ executeAt: delaySeconds ? existing.executeAt ?? new Date(Date.now() + delaySeconds * 1000).toISOString() : undefined,
+ timezone: cronExpr ? (body.timezone === '' ? undefined : body.timezone ?? existing.timezone) : undefined,
+ missedRunPolicy: cronExpr ? undefined : body.missedRunPolicy ?? existing.missedRunPolicy,
};
- scheduler.replaceManifest(manifest);
+ // Editing a label/policy never silently repeats failed or completed work.
+ // A new delay explicitly rearms a pending one-shot; terminal runs require resume.
+ if (body.delaySeconds !== undefined && !existing.lastRun) manifest.executeAt = new Date(Date.now() + delaySeconds! * 1000).toISOString();
+ scheduler.replaceManifest(manifest, body.resume === true);
scheduler.persistSchedules();
return c.json({ success: true, schedule: scheduler.getManifest(id) });
});
diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts
index c4375d5c..394a1743 100644
--- a/ui/src/lib/api.ts
+++ b/ui/src/lib/api.ts
@@ -478,7 +478,7 @@ export const mcp = {
export const schedules = {
list: () => get<{ schedules: Schedule[]; total: number }>("/api/schedules"),
- update: (id: string, data: Partial<Schedule>) =>
+ update: (id: string, data: Partial<Schedule> & { resume?: boolean }) =>
put<{ success: boolean; schedule: Schedule }>(`/api/schedules/${id}`, data),
delete: (id: string) => del<{ success: boolean }>(`/api/schedules/${id}`),
};
@@ -1036,12 +1036,16 @@ export interface McpServer {
export interface Schedule {
id: string;
+ executeAt?: string;
+ timezone?: string;
+ missedRunPolicy?: 'skip' | 'run_once';
+ paused?: boolean;
+ lastRun?: { id: string; status: 'running' | 'completed' | 'failed' | 'interrupted' | 'skipped' | 'dispatched'; startedAt: string; finishedAt?: string; error?: string };
description: string;
cron?: string;
delaySeconds?: number;
prompt?: string;
skillName?: string;
- lastRun?: string;
nextRun?: string;
}
diff --git a/ui/src/pages/Schedules.tsx b/ui/src/pages/Schedules.tsx
index 3d713a64..e6a08359 100644
--- a/ui/src/pages/Schedules.tsx
+++ b/ui/src/pages/Schedules.tsx
@@ -79,6 +79,9 @@ interface EditForm {
delaySeconds: string;
prompt: string;
skillName: string;
+ timezone: string;
+ missedRunPolicy: 'skip' | 'run_once';
+ resume: boolean;
}
export function SchedulesPage() {
@@ -96,6 +99,9 @@ export function SchedulesPage() {
delaySeconds: "",
prompt: "",
skillName: "",
+ timezone: "",
+ missedRunPolicy: 'skip',
+ resume: false,
});
const [saving, setSaving] = useState(false);
@@ -134,6 +140,9 @@ export function SchedulesPage() {
delaySeconds: schedule.delaySeconds?.toString() ?? "",
prompt: schedule.prompt ?? "",
skillName: schedule.skillName ?? "",
+ timezone: schedule.timezone ?? "",
+ missedRunPolicy: schedule.missedRunPolicy ?? 'skip',
+ resume: false,
});
setEditOpen(true);
}
@@ -142,11 +151,16 @@ export function SchedulesPage() {
if (!editId) return;
setSaving(true);
try {
- const payload: Partial<Schedule> = {
+ const payload: Partial<Schedule> & { resume?: boolean } = {
description: editForm.description,
+ timezone: editForm.timezone.trim(),
+ missedRunPolicy: editForm.missedRunPolicy,
+ resume: editForm.resume,
};
if (editForm.cron.trim()) payload.cron = editForm.cron.trim();
- if (editForm.delaySeconds.trim())
+ const original = schedules.find((s) => s.id === editId);
+ if (!editForm.cron.trim() && editForm.delaySeconds.trim()
+ && (editForm.resume || original?.cron || Number(editForm.delaySeconds) !== original?.delaySeconds))
payload.delaySeconds = Number(editForm.delaySeconds);
if (editForm.prompt.trim()) payload.prompt = editForm.prompt.trim();
if (editForm.skillName.trim())
@@ -286,7 +300,7 @@ export function SchedulesPage() {
{schedule.cron && (
<Badge variant="secondary" className="gap-1">
<Calendar className="h-3 w-3" />
- {humanizeCron(schedule.cron)}
+ {humanizeCron(schedule.cron)} · {schedule.timezone ?? 'host timezone'}
</Badge>
)}
{schedule.delaySeconds != null && (
@@ -295,6 +309,9 @@ export function SchedulesPage() {
{humanizeDelay(schedule.delaySeconds)}
</Badge>
)}
+ <Badge variant={schedule.paused ? 'destructive' : 'outline'}>
+ {schedule.lastRun?.status ?? 'pending'}{schedule.paused ? ' · paused' : ''}
+ </Badge>
{schedule.skillName && (
<Badge variant="default" className="gap-1">
<Sparkles className="h-3 w-3" />
@@ -308,6 +325,8 @@ export function SchedulesPage() {
variant="ghost"
size="icon"
onClick={() => openEdit(schedule)}
+ disabled={schedule.lastRun?.status === 'running'}
+ aria-label="Edit schedule"
>
<Pencil className="h-4 w-4" />
</Button>
@@ -322,6 +341,9 @@ export function SchedulesPage() {
</CardHeader>
<CardContent>
<div className="flex flex-col gap-2 text-sm">
+ {schedule.lastRun?.error && <p className="text-destructive">{schedule.lastRun.error}</p>}
+ {schedule.lastRun?.status === 'dispatched' && <p className="text-muted-foreground">Accepted by the bot queue; see the bot's runs for its outcome.</p>}
+ {!schedule.cron && <p className="text-muted-foreground">Due: {schedule.executeAt ? formatDate(schedule.executeAt) : 'unknown'} · If offline: {schedule.missedRunPolicy === 'run_once' ? 'run once on restart' : 'skip'}</p>}
{schedule.prompt && (
<p className="text-muted-foreground">
<span className="font-medium text-foreground">Prompt: </span>
@@ -329,11 +351,11 @@ export function SchedulesPage() {
</p>
)}
<div className="flex flex-wrap gap-x-6 gap-y-1 text-xs text-muted-foreground">
- {schedule.lastRun && (
+ {schedule.lastRun?.startedAt && (
<span>
Last run:{" "}
<span className="font-medium tabular-nums text-foreground">
- {formatDate(schedule.lastRun)}
+ {formatDate(schedule.lastRun.startedAt)}
</span>
</span>
)}
@@ -360,7 +382,7 @@ export function SchedulesPage() {
<DialogHeader>
<DialogTitle>Edit Schedule</DialogTitle>
<DialogDescription>
- Update the schedule configuration.
+ Update the schedule configuration. Failed or interrupted work stays paused until you choose to schedule it again.
</DialogDescription>
</DialogHeader>
<div className="flex flex-col gap-4 py-2">
@@ -397,6 +419,26 @@ export function SchedulesPage() {
/>
</div>
</div>
+ {editForm.cron.trim() ? (
+ <div className="space-y-1.5">
+ <label htmlFor="schedule-timezone" className="text-sm font-medium">Timezone</label>
+ <Input id="schedule-timezone" value={editForm.timezone} placeholder="Host timezone (e.g. America/New_York)" onChange={(e) => setEditForm((f) => ({ ...f, timezone: e.target.value }))} />
+ </div>
+ ) : (
+ <div className="space-y-1.5">
+ <label htmlFor="schedule-missed" className="text-sm font-medium">If Mercury is offline when due</label>
+ <select id="schedule-missed" className="w-full rounded border bg-background p-2" value={editForm.missedRunPolicy} onChange={(e) => setEditForm((f) => ({ ...f, missedRunPolicy: e.target.value as 'skip' | 'run_once' }))}>
+ <option value="skip">Skip the missed run</option>
+ <option value="run_once">Run once after restart</option>
+ </select>
+ </div>
+ )}
+ {schedules.find((s) => s.id === editId)?.lastRun && (
+ <label className="flex gap-2 text-sm">
+ <input type="checkbox" checked={editForm.resume} onChange={(e) => setEditForm((f) => ({ ...f, resume: e.target.checked }))} />
+ Schedule again. I checked the prior run's effects; repeating the work is intended.
+ </label>
+ )}
<div className="space-y-1.5">
<label className="text-sm font-medium">Prompt</label>
<Input
diff --git a/website/docs/reference/scheduling.mdx b/website/docs/reference/scheduling.mdx
index 2acd031d..87a29d8e 100644
--- a/website/docs/reference/scheduling.mdx
+++ b/website/docs/reference/scheduling.mdx
@@ -17,4 +17,53 @@ Tell Mercury in conversation:
- "Remind me in 15 minutes to take a break"
- "Ping me in 2 hours about the deployment"
-Tasks persist to `~/.mercury/schedules.yaml` and restore on restart. Responses route back to the channel (CLI or Telegram) where the task was created.
\ No newline at end of file
+Tasks persist to `~/.mercury/schedules.yaml` and restore on restart. Responses route back to the channel (CLI or Telegram) where the task was created.
+## Time zones and overlap
+
+Use a timezone for recurring work, for example: “Every weekday at 9am in
+America/New_York, brief me on my calendar.” The `schedule_task` tool accepts an
+IANA `timezone`; the Schedules editor exposes the same setting. Existing schedules
+without one continue using the host timezone. Cron follows that zone's local clock,
+including daylight-saving changes.
+
+A schedule never queues another main-agent run while its previous run is still
+queued or running. Overlapping ticks are skipped, rather than accumulated for later.
+Recurring ticks missed while Mercury was offline are not replayed.
+
+## Offline and interrupted work
+
+One-shot tasks have an explicit `missed_run_policy`:
+
+- `skip` (default, including older schedules): keep a skipped record when Mercury
+ starts after the due time.
+- `run_once`: start the overdue task once after restart, provided no run had begun.
+
+This policy does **not** retry failed or interrupted work. Mercury records a run
+before invoking it. If the process stops before the outcome is confirmed, the run
+becomes **interrupted** and the schedule pauses. Failed runs also pause, because an
+external action may have succeeded before its response was lost.
+
+Check the prior run's effects before repeating it. In **Schedules → Edit**, choose
+**Schedule again** to explicitly resume. For a one-shot, the delay starts again
+from when you save. Editing only a description or policy does not repeat the work.
+The API equivalent is `PUT /api/schedules/:id` with `{"resume": true}` and, for a
+one-shot, the desired `delaySeconds`. Running tasks cannot be edited.
+
+## Run outcomes
+
+The Schedules page and `list_scheduled_tasks` show pending, running, completed,
+failed, interrupted, skipped, or dispatched status. **Completed** means the main
+agent finished its turn; inspect its response for the actual task result. Bot
+schedules show **dispatched** after their separate queue accepts them; inspect the
+bot's run history for completion. Scheduled bot jobs whose run or retry was
+interrupted go to the dead-letter queue (DLQ) instead of replaying automatically;
+future routines pause. Check their effects, then use `/bots start <id>` to resume
+future routines or `/bots replay <id> <jobId>` to explicitly repeat that job.
+A fresh queued bot job that never started can still resume. Manual/chat bot jobs
+keep their existing recovery behavior.
+
+Run records are stored with the schedule in `~/.mercury/schedules.yaml`, using an
+atomic replacement. The latest run is kept for each recurring task. The most recent
+100 resolved one-shot records are retained; active work and unresolved failures are
+kept until you remove them. These safeguards prevent automatic scheduler replay;
+they cannot guarantee exactly-once behavior from external services.