Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
feat(scheduler): retain run outcomes and require explicit interrupted-work replay
Seth Wilson113290d8ead5
diff --git a/src/bots/bot-manager.test.ts b/src/bots/bot-manager.test.tsindex abbe4fda..85f9a3b1 100644--- a/src/bots/bot-manager.test.ts+++ b/src/bots/bot-manager.test.ts@@ -218,6 +218,51 @@ describe('BotManager queue + turn lifecycle', () => {}});+ it('does not automatically retry a failed scheduled bot run and pauses its routine', async () => {+ mockedGenerateText.mockRejectedValue(new Error('HTTP 429: too many requests'));+ seedBot(store, 'scheduled');+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] });+ const settle = async () => { for (let i = 0; i < 20; i++) await new Promise(r => setImmediate(r)); };+ try {+ const { jobId } = manager.enqueue('scheduled', { trigger: 'cron', prompt: 'Send the report', routineId: 'bot:scheduled:report' });+ await settle();+ expect(manager.getJournal('scheduled').map((r) => r.state)).toEqual(['failed']);+ expect(manager.queue.pendingJobs('scheduled')).toEqual([]);+ expect(manager.queue.listDlq('scheduled').map((j) => j.id)).toEqual([jobId]);+ expect(store.readRoutineState('scheduled').paused['bot:scheduled:report']).toBeDefined();+ await vi.advanceTimersByTimeAsync(30_000);+ await settle();+ expect(manager.getJournal('scheduled')).toHaveLength(1);+ expect(manager.enqueue('scheduled', { trigger: 'cron', prompt: 'Send the report again', routineId: 'bot:scheduled:report' })).toMatchObject({ accepted: false, reasonCode: 'routine_paused' });+ } finally {+ vi.useRealTimers();+ }+ });++ it('pauses future routines on quick restart and retains the DLQ until explicit replay is accepted', async () => {+ seedBot(store, 'crashed');+ manager.queue.enqueue({ id: 'previous-run', botId: 'crashed', trigger: 'cron', prompt: 'Send once', attempts: 0, createdAt: Date.now(), idempotencyKey: 'previous-schedule' });+ manager.queue.claim('previous-run', 3600);+ manager.dispose();+ const restarted = makeManager(root);+ expect(restarted.queue.listDlq('crashed').map((j) => j.id)).toEqual(['previous-run']);+ expect(store.readRoutineState('crashed').paused['*']).toBeDefined();+ expect(restarted.enqueue('crashed', { trigger: 'cron', prompt: 'Send once', routineId: 'bot:crashed:any-name' })).toMatchObject({ accepted: false, reasonCode: 'routine_paused' });+ expect(mockedGenerateText).not.toHaveBeenCalled();+ restarted.setEnabled('crashed', false);+ expect(restarted.replayDlq('crashed', 'previous-run')).toMatchObject({ accepted: false, reasonCode: 'target_disabled' });+ expect(restarted.queue.listDlq('crashed')).toHaveLength(1);+ restarted.setEnabled('crashed', true);+ mockedGenerateText.mockResolvedValue({ text: 'done', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as never);+ expect(restarted.replayDlq('crashed', 'previous-run')).toMatchObject({ accepted: true });+ expect(restarted.queue.listDlq('crashed')).toEqual([]);+ await vi.waitFor(() => expect(restarted.getJournal('crashed')).toHaveLength(1));+ // Explicit replay repeats just this job, not all future ticks.+ expect(store.readRoutineState('crashed').paused['*']).toBeDefined();+ restarted.start('crashed');+ expect(store.readRoutineState('crashed').paused).toEqual({});+ });+it('a retried job never runs twice when the due-sweep and the backoff timer both fire (P0.7)', async () => {let calls = 0;let releaseSecond: () => void = () => {};diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.tsindex 02a28444..8dcf81a6 100644--- a/src/bots/bot-manager.ts+++ b/src/bots/bot-manager.ts@@ -273,6 +273,14 @@ export class BotManager {this.queue = deps.queue ?? new BotQueue(this.store.botsRoot, this.config.bots?.retention?.dlqCap);this.tasks = new BotTaskStore(this.store.botsRoot);this.inbox = new BotInbox(this.store.botsRoot);+ this.queue.onInterruptedSchedule = (job) => {+ if (!this.store.exists(job.botId)) return;+ const routines = this.store.readRoutineState(job.botId);+ routines.paused['*'] = { since: new Date().toISOString(), reason: 'A scheduled run was interrupted. Inspect its effects before /bots start or manual DLQ replay.' };+ this.store.writeRoutineState(job.botId, routines);+ this.needsYou.add(job.botId);+ this.inbox.add({ botId: job.botId, botName: this.store.get(job.botId)?.name ?? job.botId, runId: job.id, files: [], kind: 'needs-you', title: 'Scheduled run interrupted', summary: routines.paused['*'].reason, reasonCode: 'scheduled_interrupted' });+ };// A stop survives a restart (it used to be in memory only, so a restart// quietly undid /bots stop).for (const id of this.store.readHeld()) if (this.store.exists(id)) this.held.add(id);@@ -316,7 +324,7 @@ export class BotManager {for (const m of this.store.list()) {const drained = this.queue.drainMail(m.id);if (drained.length > 0) this.mailboxes.set(m.id, drained);- if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou) this.needsYou.add(m.id);+ if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou || this.store.readRoutineState(m.id).paused['*']) this.needsYou.add(m.id);}// A profile relocation (re-parent or fleet-layout migration) invalidates// the cached journal handle (it is bound to the old dir) and the bot's@@ -435,7 +443,7 @@ export class BotManager {return this.journalFor(botId);}- enqueue(botId: string, job: { trigger: BotTrigger; prompt: string; fromBot?: string; source?: { channelType: string; channelId: string }; attempts?: number; routineId?: string; attachments?: ChannelAttachment[] }): { jobId: string; accepted: boolean; reasonCode?: string; duplicated?: boolean } {+ enqueue(botId: string, job: { trigger: BotTrigger; prompt: string; fromBot?: string; source?: { channelType: string; channelId: string }; attempts?: number; routineId?: string; attachments?: ChannelAttachment[]; manualReplay?: boolean }): { jobId: string; accepted: boolean; reasonCode?: string; duplicated?: boolean } {const manifest = this.store.get(botId);if (!manifest) return { jobId: '', accepted: false, reasonCode: 'target_unknown' };if (!manifest.enabled || this.disabled.has(botId)) return { jobId: '', accepted: false, reasonCode: 'target_disabled' };@@ -447,12 +455,12 @@ export class BotManager {// It is skipped — never stacked — when the bot is still busy, when the// routine ran too recently, or when the routine was paused for// producing nothing. The next tick tries again.- if (job.trigger === 'cron') {+ if (job.trigger === 'cron' && !job.manualReplay) {// A stopped bot's routines don't queue up behind the stop.if (this.held.has(botId)) return { jobId: '', accepted: false, reasonCode: 'stopped' };const key = job.routineId ?? `bot:${botId}:cron`;const routineState = this.store.readRoutineState(botId);- const pausedEntry = routineState.paused[key];+ const pausedEntry = routineState.paused[key] ?? routineState.paused['*'];if (pausedEntry) {const until = pausedEntry.until ? Date.parse(pausedEntry.until) : Number.POSITIVE_INFINITY;if (Date.now() < until) return { jobId: '', accepted: false, reasonCode: 'routine_paused' };@@ -953,7 +961,7 @@ export class BotManager {this.providers.markAuthFailure?.(usedProvider);authSwitch = resolveProvider(this.providers, manifest, this.config).name !== usedProvider;}- const willRetry = output.status === 'failed' && !!output.reasonCode+ const willRetry = job.trigger !== 'cron' && output.status === 'failed' && !!output.reasonCode&& (isTransientFailure(output.reasonCode) || authSwitch) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS;const turnLimited = output.status === 'failed' && isTurnLimitFailure(output.reasonCode);const routineKey = job.trigger === 'cron' ? (job.routineId ?? `bot:${botId}:cron`) : undefined;@@ -1008,6 +1016,9 @@ export class BotManager {} else if (output.status === 'completed' && state.failStreak?.[routineKey]) {delete state.failStreak[routineKey];}+ if (output.status !== 'completed') {+ state.paused[routineKey] = { since: new Date().toISOString(), reason: 'Scheduled run did not finish. Inspect its effects before /bots start or manual DLQ replay.' };+ }this.store.writeRoutineState(botId, state);}@@ -1015,7 +1026,7 @@ export class BotManager {// three runs in a row is paused — the owner is told once, and// /bots start resumes it. This is what stops a bot that has// talked itself into a loop of records about records.- if (routineKey && this.expectationFor(manifest, job) === 'work' && output.status !== 'halted') {+ if (routineKey && this.expectationFor(manifest, job) === 'work' && output.status === 'completed') {const state = this.store.readRoutineState(botId);if (output.outcome === 'none') {const streak = (state.noOutcomeStreak[routineKey] ?? 0) + 1;@@ -1099,6 +1110,10 @@ export class BotManager {if (routineFailStreak <= 1) {await this.alertOwner(botId, `❌ **${manifest.name}** failed permanently [reason: ${output.reasonCode}] — replay with \`/bots replay ${botId} ${job.id}\``);}+ } else if (output.status === 'paused' && job.trigger === 'cron') {+ this.queue.settle(job.id, 'dead', 'scheduled_interrupted');+ this.needsYou.add(botId);+ this.inbox.add({ botId, botName: manifest.name, runId: job.id, files: output.deliverables, kind: 'needs-you', title: 'Scheduled run paused', summary: 'Inspect the existing effects before /bots start or manually replaying this run from the DLQ.', reasonCode: 'scheduled_interrupted' });} else if (output.status === 'paused') {// Step-budget pause: work continues next turn — same job requeues in// place (durable), no attempts bump.@@ -1232,6 +1247,11 @@ export class BotManager {}} catch (err: any) {logger.error({ botId, jobId: job.id, err: err?.message }, 'Bot turn crashed');+ if (job.trigger === 'cron') {+ const state = this.store.readRoutineState(botId);+ state.paused[job.routineId ?? `bot:${botId}:cron`] = { since: new Date().toISOString(), reason: 'Scheduled run crashed. Inspect its effects before /bots start or manual DLQ replay.' };+ this.store.writeRoutineState(botId, state);+ }this.queue.settle(job.id, 'dead', 'unknown_error');this.needsYou.add(botId);void this.alertOwner(botId, `💥 **${botId}** run ${job.id} crashed: ${String(err?.message ?? err).slice(0, 150)} — see journal; replay from DLQ.`);@@ -2086,14 +2106,20 @@ export class BotManager {// then reported not_found).const entry = this.queue.peekDlq(jobId);if (!entry || entry.botId !== botId) return { accepted: false, reasonCode: 'not_found' };- this.queue.removeFromDlq(jobId);- return this.enqueue(botId, {+ const result = this.enqueue(botId, {trigger: entry.trigger,+ // The owner explicitly chose this job. Bypass routine timing/pause gates+ // for this admission only; future ticks stay paused and a new crash still+ // has cron recovery semantics.+ manualReplay: true,+ attachments: entry.attachments,prompt: entry.prompt,fromBot: entry.fromBot,source: entry.source,attempts: 0,});+ if (result.accepted) this.queue.removeFromDlq(jobId);+ return result;}getStorage(): Array<{ id: string; bytes: number; journalBytes: number }> {diff --git a/src/bots/queue.test.ts b/src/bots/queue.test.tsindex 5c30448f..deae18d4 100644--- a/src/bots/queue.test.ts+++ b/src/bots/queue.test.ts@@ -46,6 +46,7 @@ mockedStreamText.mockImplementation(((opts: any) => {}) as any);import { BotQueue, JsonFileQueueBackend, SqliteQueueBackend, idempotencyKeyFor, LEASE_SECONDS } from './queue.js';import type { DurableBotJob } from './queue.js';+import { openSqlite } from '../utils/sqlite-driver.js';function job(id: string, overrides: Partial<DurableBotJob> = {}): Omit<DurableBotJob, 'state'> {return {@@ -181,6 +182,62 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {expect(backend.counts().claimed).toBe(1);});+ it('quarantines interrupted scheduled jobs while preserving safe first attempts and chat recovery', () => {+ const backend = mkBackend(name, root);+ const queue = Object.create(BotQueue.prototype) as BotQueue;+ const interrupted = vi.fn();+ Object.assign(queue, { backend, onInterruptedSchedule: interrupted });+ backend.enqueue(job('scheduled-claimed', { trigger: 'cron' }));+ backend.claim('scheduled-claimed', 0);+ backend.enqueue(job('scheduled-fresh', { trigger: 'cron' }));+ backend.enqueue(job('scheduled-retry', { trigger: 'cron' }));+ backend.claim('scheduled-retry', 0);+ backend.retry('scheduled-retry', 1, Date.now() - 1);+ backend.enqueue(job('scheduled-continuation', { trigger: 'cron' }));+ backend.claim('scheduled-continuation', 0);+ backend.retry('scheduled-continuation', 0, Date.now() - 1);+ backend.enqueue(job('chat-claimed'));+ backend.claim('chat-claimed', 0);+ expect(queue.resumeJobs().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);+ expect(interrupted).toHaveBeenCalledTimes(3);+ expect(backend.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-claimed', 'scheduled-continuation', 'scheduled-retry']);+ expect(backend.listDlq().every((j) => j.reasonCode === 'scheduled_interrupted')).toBe(true);+ const restarted = mkBackend(name, root);+ expect(restarted.rehydratable().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);+ });++ it('quarantines a scheduled run on quick restart before its lease expires', () => {+ const backend = mkBackend(name, root);+ backend.enqueue(job('scheduled-live-lease', { trigger: 'cron' }));+ backend.claim('scheduled-live-lease', 3600);+ backend.enqueue(job('chat-live-lease'));+ backend.claim('chat-live-lease', 3600);+ backend.enqueue(job('scheduled-future-retry', { trigger: 'cron' }));+ backend.retry('scheduled-future-retry', 1, Date.now() + 3600_000);+ const restarted = mkBackend(name, root);+ const queue = Object.create(BotQueue.prototype) as BotQueue;+ const pause = vi.fn();+ Object.assign(queue, { backend: restarted, onInterruptedSchedule: pause });+ expect(queue.resumeJobs()).toEqual([]);+ expect(pause).toHaveBeenCalledTimes(2);+ expect(restarted.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-future-retry', 'scheduled-live-lease']);+ expect(restarted.counts()).toMatchObject({ claimed: 1 });+ });++ it('does not discard the interrupted job when saving its routine pause fails', () => {+ const backend = mkBackend(name, root);+ const queue = Object.create(BotQueue.prototype) as BotQueue;+ Object.assign(queue, { backend, onInterruptedSchedule: () => { throw new Error('Disk full'); } });+ backend.enqueue(job('interrupted', { trigger: 'cron' }));+ backend.claim('interrupted', 0);+ expect(() => queue.resumeJobs()).toThrow('Disk full');+ expect(backend.rehydratable().map((j) => j.id)).toEqual(['interrupted']);+ expect(backend.listDlq()).toEqual([]);+ queue.onInterruptedSchedule = vi.fn();+ expect(queue.resumeJobs()).toEqual([]);+ expect(backend.listDlq()[0].reasonCode).toBe('scheduled_interrupted');+ });+it('DLQ is capped with oldest-first eviction', () => {const backend = mkBackend(name, root, 3);for (let i = 0; i < 5; i++) {@@ -214,6 +271,22 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});}+ it('rolls back SQLite quarantine if the DLQ insert fails after removing the active row', () => {+ const db = openSqlite(join(root, 'queue.db'))!;+ const backend = new SqliteQueueBackend(root, 100, db);+ openBackends.push(backend);+ backend.enqueue(job('quarantine', { trigger: 'cron' }));+ backend.claim('quarantine', 3600);+ db.exec(`CREATE TRIGGER fail_dlq BEFORE INSERT ON bot_dlq BEGIN SELECT RAISE(ABORT, 'injected disk error'); END`);+ expect(() => backend.settle('quarantine', 'dead', 'scheduled_interrupted')).toThrow('injected disk error');+ expect(backend.interruptedSchedules().map((j) => j.id)).toEqual(['quarantine']);+ expect(backend.listDlq()).toEqual([]);+ db.exec('DROP TRIGGER fail_dlq');+ backend.settle('quarantine', 'dead', 'scheduled_interrupted');+ expect(backend.interruptedSchedules()).toEqual([]);+ expect(backend.listDlq()[0]).toMatchObject({ id: 'quarantine', prompt: 'task quarantine', reasonCode: 'scheduled_interrupted' });+ });+it('BotQueue facade resumes expired leases and pending jobs', () => {const dir = join(root, 'bots');const queue = new BotQueue(dir, 100);diff --git a/src/bots/queue.ts b/src/bots/queue.tsindex 6cbfcb20..d88d8e8c 100644--- a/src/bots/queue.ts+++ b/src/bots/queue.ts@@ -64,6 +64,8 @@ export interface BotQueueBackend {retry(jobId: string, attempts: number, runAfterMs?: number): void;/** Jobs safe to resume: pending (due), or claimed whose lease has expired. */rehydratable(): DurableBotJob[];+ /** Startup only: claimed scheduled jobs and old scheduled retry entries, regardless of lease/backoff. */+ interruptedSchedules(): DurableBotJob[];/** Pending jobs whose run-after time has arrived (retry backoff elapsed). */dueJobs(): DurableBotJob[];/** All pending jobs for one bot, in creation order — explicit resume (/bots start), ignores retry backoff. */@@ -147,6 +149,7 @@ function normalizeJob(raw: any): DurableBotJob {idempotencyKey: raw.idempotencyKey ?? '',reasonCode: raw.reasonCode ?? raw.reason_code ?? undefined,settledAt: raw.settledAt ?? raw.settled_at ?? undefined,+ runAfter: raw.runAfter ?? raw.run_after ?? undefined,};}@@ -160,6 +163,8 @@ function normalizeJob(raw: any): DurableBotJob {*/export class BotQueue {readonly backend: BotQueueBackend;+ /** The manager pauses future routines when a prior scheduled run is uncertain. */+ onInterruptedSchedule?: (job: DurableBotJob) => void;/** Post-close guard: in-flight turns' queue access degrades to no-ops. */private closed = false;@@ -201,15 +206,29 @@ export class BotQueue {/** Jobs to resume on startup: due pending, plus expired-lease claimed jobs. */resumeJobs(): DurableBotJob[] {+ this.excludeInterruptedSchedules(this.backend.interruptedSchedules());const expired = this.backend.requeueExpiredLeases();if (expired > 0) logger.info({ expired }, 'Requeued expired bot job leases');- return this.backend.rehydratable();+ return this.excludeInterruptedSchedules(this.backend.rehydratable());+ }++ private excludeInterruptedSchedules(jobs: DurableBotJob[]): DurableBotJob[] {+ return jobs.filter((job) => {+ // A pending first attempt has not acted. Claimed work and old retry/+ // continuation entries may already have effects and require manual replay.+ if (job.trigger !== 'cron' || (job.state === 'pending' && job.attempts === 0 && job.runAfter === undefined)) return true;+ // Persist the routine pause first. If that fails, leave the job eligible+ // for quarantine again rather than losing the opportunity to stop future ticks.+ this.onInterruptedSchedule?.(job);+ this.backend.settle(job.id, 'dead', 'scheduled_interrupted');+ return false;+ });}/** Pending jobs whose retry backoff has elapsed (swept periodically). */dueJobs(): DurableBotJob[] {if (this.closed) return [];- return this.backend.dueJobs();+ return this.excludeInterruptedSchedules(this.backend.dueJobs());}/** All pending jobs for one bot — explicit resume via /bots start. */@@ -365,27 +384,30 @@ export class SqliteQueueBackend implements BotQueueBackend {}settle(jobId: string, outcome: 'done' | 'dead', reasonCode?: string): void {- const row = this.db.prepare(`SELECT * FROM bot_jobs WHERE id = ?`).get(jobId) as any;- if (!row) return;- if (outcome === 'done') {+ if (this.closed) return;+ this.db.transaction(() => {+ const row = this.db.prepare(`SELECT * FROM bot_jobs WHERE id = ?`).get(jobId) as any;+ if (!row) return;+ if (outcome === 'done') {+ this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);+ return;+ }this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);- return;- }- this.db.prepare(`DELETE FROM bot_jobs WHERE id = ?`).run(jobId);- this.db.prepare(- `INSERT INTO bot_dlq (id, bot_id, trigger, prompt, from_bot, source, attempts, created_at, idempotency_key, reason_code, dead_at)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,- ).run(- row.id, row.bot_id, row.trigger, row.prompt, row.from_bot, row.source,- row.attempts + 1, row.created_at, row.idempotency_key, reasonCode ?? row.reason_code ?? null, Date.now(),- );- // Cap the DLQ — evict oldest, bounded by design (§2.10).- const overflow = this.db.prepare(- `SELECT id FROM bot_dlq ORDER BY dead_at DESC, created_at DESC, rowid DESC LIMIT -1 OFFSET ?`,- ).all(this.dlqCap) as any[];- for (const r of overflow) {- this.db.prepare(`DELETE FROM bot_dlq WHERE id = ?`).run(r.id);- }+ this.db.prepare(+ `INSERT INTO bot_dlq (id, bot_id, trigger, prompt, from_bot, source, attempts, created_at, idempotency_key, reason_code, dead_at)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,+ ).run(+ row.id, row.bot_id, row.trigger, row.prompt, row.from_bot, row.source,+ row.attempts + 1, row.created_at, row.idempotency_key, reasonCode ?? row.reason_code ?? null, Date.now(),+ );+ // Cap the DLQ — evict oldest, bounded by design (§2.10).+ const overflow = this.db.prepare(+ `SELECT id FROM bot_dlq ORDER BY dead_at DESC, created_at DESC, rowid DESC LIMIT -1 OFFSET ?`,+ ).all(this.dlqCap) as any[];+ for (const r of overflow) {+ this.db.prepare(`DELETE FROM bot_dlq WHERE id = ?`).run(r.id);+ }+ });}requeueExpiredLeases(): number {@@ -402,6 +424,14 @@ export class SqliteQueueBackend implements BotQueueBackend {).run(attempts, runAfterMs ?? null, jobId);}+ interruptedSchedules(): DurableBotJob[] {+ if (this.closed) return [];+ const rows = this.db.prepare(+ `SELECT * FROM bot_jobs WHERE trigger = 'cron' AND (state = 'claimed' OR attempts > 0 OR run_after IS NOT NULL) ORDER BY created_at`,+ ).all() as unknown[];+ return rows.map(normalizeJob);+ }+rehydratable(): DurableBotJob[] {if (this.closed) return [];// Due pending jobs + expired-lease claimed jobs — jobs still in their@@ -618,6 +648,11 @@ export class JsonFileQueueBackend implements BotQueueBackend {}}+ interruptedSchedules(): DurableBotJob[] {+ return this.data.jobs.filter((job) => job.trigger === 'cron'+ && (job.state === 'claimed' || job.attempts > 0 || job.runAfter !== undefined));+ }+rehydratable(): DurableBotJob[] {// Due pending + expired-lease claimed (INCLUSIVE — a heartbeat to an// instant-expiry lease must be sweepable in the same millisecond;diff --git a/src/bots/tools/bot-schedule.ts b/src/bots/tools/bot-schedule.tsindex eaa64b54..64797bac 100644--- a/src/bots/tools/bot-schedule.ts+++ b/src/bots/tools/bot-schedule.ts@@ -23,7 +23,7 @@ export interface BotScheduler {}): void;persistSchedules(): void;/** All live manifests, used for the per-bot self-schedule cap. */- getManifests(): Array<{ id: string; botId?: string }>;+ getManifests(): Array<{ id: string; botId?: string; lastRun?: { status: string } }>;}const SELF_PREFIX = 'self-';@@ -107,7 +107,7 @@ export function createBotScheduleTool(scheduler: BotScheduler, botId: string) {}// One-shot with a pending cap — this is the runaway-loop guard.- const pending = scheduler.getManifests().filter(m => m.botId === botId && m.id.includes(`:${SELF_PREFIX}`)).length;+ const pending = scheduler.getManifests().filter(m => m.botId === botId && m.id.includes(`:${SELF_PREFIX}`) && (!m.lastRun || m.lastRun.status === 'running')).length;if (pending >= MAX_PENDING_SELF_SCHEDULES) {return `Error: you already have ${MAX_PENDING_SELF_SCHEDULES} pending self-scheduled runs — complete or let them fire before adding more.`;}diff --git a/src/capabilities/scheduler/list-tasks.ts b/src/capabilities/scheduler/list-tasks.tsindex 4a158f9e..78b7197c 100644--- a/src/capabilities/scheduler/list-tasks.ts+++ b/src/capabilities/scheduler/list-tasks.ts@@ -4,7 +4,7 @@ import type { Scheduler, ScheduledTaskManifest } from '../../core/scheduler.js';export function createListTasksTool(scheduler: Scheduler) {return tool({- description: 'List all scheduled tasks with their cron expressions and descriptions.',+ description: 'List scheduled tasks, run outcomes, failures, timezone and missed-run policy. Failed or interrupted tasks remain paused until explicitly rescheduled.',inputSchema: zodSchema(z.object({})),execute: async () => {const manifests = scheduler.getManifests();@@ -13,7 +13,9 @@ export function createListTasksTool(scheduler: Scheduler) {}return manifests.map((m: ScheduledTaskManifest) => {const trigger = m.skillName ? `skill: ${m.skillName}` : m.prompt ? `prompt: "${m.prompt.slice(0, 40)}"` : 'no action';- return `${m.id} | ${m.cron} | ${m.description} | ${trigger}`;+ const timing = m.cron ? `${m.cron} (${m.timezone ?? 'host timezone'})` : `${m.executeAt ?? 'unknown time'} (missed: ${m.missedRunPolicy ?? 'skip'})`;+ const state = m.lastRun ? `${m.lastRun.status}${m.paused ? ', paused' : ''}${m.lastRun.error ? `: ${m.lastRun.error}` : ''}` : 'pending';+ return `${m.id} | ${timing} | ${state} | ${m.description} | ${trigger}`;}).join('\n');},});diff --git a/src/capabilities/scheduler/schedule-task.test.ts b/src/capabilities/scheduler/schedule-task.test.tsindex e090b8c8..ad31b8c3 100644--- a/src/capabilities/scheduler/schedule-task.test.ts+++ b/src/capabilities/scheduler/schedule-task.test.ts@@ -64,3 +64,32 @@ describe('turn grants', () => {expect(await pm.approvePluginTool('whatsapp__call_owner', 'x')).toBe(false);});});++describe('schedule timing policy', () => {+ it('includes catch-up policy in the approval and saves it', async () => {+ const { tool, added, asked } = setup();+ await tool.execute({ delay_seconds: 60, description: 'Brief me', prompt: 'Read the news', missed_run_policy: 'run_once' });+ expect(asked[0]).toContain('if offline, run once after restart');+ expect(added[0].missedRunPolicy).toBe('run_once');+ });++ it('saves and confirms the cron timezone', async () => {+ const { tool, added, asked } = setup();+ const result = await tool.execute({ cron: '0 9 * * *', description: 'Brief me', prompt: 'Read the news', timezone: 'America/New_York' });+ expect(asked[0]).toContain('America/New_York');+ expect(added[0].timezone).toBe('America/New_York');+ expect(result).toContain('America/New_York');+ });++ it.each([+ { delay_seconds: -1 }, { delay_seconds: Infinity },+ { cron: 'invalid' }, { cron: '0 9 * * *', timezone: 'Not/A_Zone' },+ { delay_seconds: 60, missed_run_policy: 'replay_all' },+ { delay_seconds: 60, timezone: 'UTC' },+ ])('rejects invalid timing before asking for approval: %o', async (timing) => {+ const { tool, added, asked } = setup();+ await tool.execute({ ...timing, description: 'Brief me', prompt: 'Read the news' });+ expect(asked).toEqual([]);+ expect(added).toEqual([]);+ });+});diff --git a/src/capabilities/scheduler/schedule-task.ts b/src/capabilities/scheduler/schedule-task.tsindex 05e514a5..11cd444b 100644--- a/src/capabilities/scheduler/schedule-task.ts+++ b/src/capabilities/scheduler/schedule-task.ts@@ -1,6 +1,8 @@import { tool, zodSchema } from 'ai';import { z } from 'zod';import cron from 'node-cron';+import { randomUUID } from 'node:crypto';+import { isValidScheduleTimezone } from '../../core/scheduler.js';import { isPluginToolName } from '../../plugins/tool-bridge.js';import type { Scheduler, ScheduledTaskManifest } from '../../core/scheduler.js';import type { PermissionManager } from '../permissions.js';@@ -10,7 +12,9 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisdescription: 'Schedule a task. Use "cron" for recurring tasks (e.g. "0 9 * * *" for daily at 9am) or "delay_seconds" for one-shot delayed tasks (e.g. 15 for "remind me in 15 seconds"). Provide exactly one of cron or delay_seconds.',inputSchema: zodSchema(z.object({cron: z.string().optional().describe('Cron expression for recurring tasks (e.g. "0 9 * * *" for daily at 9am)'),- delay_seconds: z.number().optional().describe('Delay in seconds for one-shot tasks (e.g. 15 for "remind me in 15 seconds")'),+ delay_seconds: z.number().finite().positive().optional().describe('Delay in seconds for one-shot tasks (e.g. 15 for "remind me in 15 seconds")'),+ timezone: z.string().optional().describe('IANA timezone for recurring cron, e.g. America/New_York. Omit for host local time.'),+ missed_run_policy: z.enum(['skip', 'run_once']).optional().describe('One-shot behavior if Mercury was offline when due: skip (default) or run_once on restart. Interrupted runs are never retried automatically.'),description: z.string().describe('Human-readable description of what this task does'),prompt: z.string().optional().describe('Prompt to send to the agent when the task fires'),skill_name: z.string().optional().describe('Name of a skill to invoke when the task fires'),@@ -18,7 +22,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permis'Plugin tools the task needs when it fires, approved now so it can run unattended, e.g. ["whatsapp__call_owner"] for "call me in 10 minutes" or "call me when the research is done". Only list tools the task actually needs.',),})),- execute: async ({ cron: cronExpr, delay_seconds, description, prompt, skill_name, allow_tools }) => {+ execute: async ({ cron: cronExpr, delay_seconds, description, prompt, skill_name, allow_tools, timezone, missed_run_policy }) => {if (!cronExpr && !delay_seconds) {return 'Either cron or delay_seconds must be provided.';}@@ -26,6 +30,10 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisreturn 'Provide either cron or delay_seconds, not both.';}+ if (delay_seconds !== undefined && (!Number.isFinite(delay_seconds) || delay_seconds <= 0)) return 'delay_seconds must be a positive finite number.';+ if (cronExpr && !cron.validate(cronExpr)) return `Invalid cron expression: "${cronExpr}".`;+ if (timezone !== undefined && (!cronExpr || !isValidScheduleTimezone(timezone))) return 'timezone requires a cron schedule and a valid IANA timezone.';+ if (missed_run_policy !== undefined && (!delay_seconds || !['skip', 'run_once'].includes(missed_run_policy))) return 'missed_run_policy requires a one-shot schedule and skip or run_once.';if (!prompt && !skill_name) {return 'Either prompt or skill_name must be provided so the task has something to do.';}@@ -42,11 +50,11 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisreturn 'Only the owner can schedule tasks that act without asking (allow_tools).';}const approved = await permissions.requestApproval(- `Create scheduled task: ${description}${allowTools.length ? ` (when it runs, it may use ${allowTools.join(', ')} without asking again)` : ''}`,+ `Create scheduled task: ${description}${timezone ? ` (${timezone})` : ''}${missed_run_policy === 'run_once' ? ' (if offline, run once after restart)' : ''}${allowTools.length ? ` (when it runs, it may use ${allowTools.join(', ')} without asking again)` : ''}`,);if (!approved) return 'Permission denied: scheduled task creation was not approved.';- const id = `task-${Date.now().toString(36)}`;+ const id = `task-${randomUUID()}`;const ctx = getContext();if (delay_seconds) {@@ -56,6 +64,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisprompt,skillName: skill_name,delaySeconds: delay_seconds,+ missedRunPolicy: missed_run_policy ?? 'skip',executeAt: new Date(Date.now() + delay_seconds * 1000).toISOString(),createdAt: new Date().toISOString(),sourceChannelId: ctx.channelId,@@ -77,6 +86,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisconst manifest: ScheduledTaskManifest = {id,cron: cronExpr,+ timezone,description,prompt,skillName: skill_name,@@ -90,7 +100,7 @@ export function createScheduleTaskTool(scheduler: Scheduler, permissions: Permisscheduler.persistSchedules();const triggerType = skill_name ? `skill: ${skill_name}` : `prompt: "${prompt!.slice(0, 60)}"`;- return `Task "${id}" scheduled. Cron: ${cronExpr}. Will execute ${triggerType}. Description: ${description}`;+ return `Task "${id}" scheduled. Cron: ${cronExpr} (${timezone ?? 'host timezone'}). Will execute ${triggerType}. Description: ${description}`;},});}diff --git a/src/core/agent.ts b/src/core/agent.tsindex 9c802c2b..8bf26973 100644--- a/src/core/agent.ts+++ b/src/core/agent.ts@@ -12,7 +12,7 @@ import type { ChannelRegistry } from '../channels/registry.js';import type { MercuryConfig, ProviderName } from '../utils/config.js';import type { TokenBudget } from '../utils/tokens.js';import type { CapabilityRegistry } from '../capabilities/registry.js';-import type { ScheduledTaskManifest } from './scheduler.js';+import type { ScheduledDispatch, ScheduledTaskManifest } from './scheduler.js';import { DeepSeekProvider } from '../providers/deepseek.js';import { ArtifactCollector } from '../artifacts/collector.js';import { artifactStore, type Artifact } from '../artifacts/store.js';@@ -671,7 +671,7 @@ export class Agent {this.notifyBackgroundTaskComplete(task);});- this.scheduler.setOnScheduledTask(async (manifest) => this.handleScheduledTask(manifest));+ this.scheduler.setOnScheduledTask(async (manifest, runId) => this.handleScheduledTask(manifest, runId));this.channels.onIncomingMessage((msg) => this.enqueueMessage(msg));this.wireOwnerApprovals();@@ -2165,6 +2165,18 @@ export class Agent {}logger.error({ err, msg: msg.content.slice(0, 50) }, 'Failed to handle message');} finally {+ const taskId = msg.metadata?.scheduledTaskId;+ const runId = msg.metadata?.scheduledRunId;+ if (typeof taskId === 'string' && typeof runId === 'string') {+ const entry = item.workKey ? this.workLedger.get(item.workKey) : undefined;+ try {+ this.scheduler.finishRun(taskId, runId, entry?.status === 'completed' ? 'completed' : 'failed',+ entry?.status === 'completed' ? undefined : entry?.error ?? 'Scheduled work ended without confirmed completion');+ } catch (err) {+ // Leave the durable running record uncertain, but always release queue state.+ logger.error({ err, taskId, runId }, 'Could not persist scheduled run outcome');+ }+ }this.currentWorkKey = null;if (!this.lifecycle.is('idle') && this.lifecycle.canTransitionTo('idle')) {this.lifecycle.transition('idle');@@ -4169,7 +4181,7 @@ export class Agent {return;}logger.error({ err: lastError }, errMsg);- if (this.currentWorkKey && (hasCompletedTool || hasStreamedOutput || requiresContinuationApproval)) {+ if (this.currentWorkKey && !msg.metadata?.scheduledTaskId && (hasCompletedTool || hasStreamedOutput || requiresContinuationApproval)) {const continuationAttempt = typeof msg.metadata?.continuationAttempt === 'number' ? msg.metadata.continuationAttempt : 0;const needsApproval = needsContinuationApproval(continuationAttempt, requiresContinuationApproval);if (needsApproval) {@@ -4203,7 +4215,7 @@ export class Agent {return;}}- if (this.currentWorkKey && !hasCompletedTool && !hasStreamedOutput && this.isRetryableProviderError(lastError)) {+ if (this.currentWorkKey && !msg.metadata?.scheduledTaskId && !hasCompletedTool && !hasStreamedOutput && this.isRetryableProviderError(lastError)) {const attempts = this.workLedger.get(this.currentWorkKey)?.attempts ?? 1;const shouldContinue = !needsRetryApproval(attempts) || (channel && msg.channelType !== 'internal'@@ -4969,28 +4981,36 @@ export class Agent {});}- async processInternalPrompt(prompt: string, channelId?: string, channelType?: string, opts: { allowTools?: string[]; scheduledTaskId?: string } = {}): Promise<void> {+ async processInternalPrompt(prompt: string, channelId?: string, channelType?: string, opts: { allowTools?: string[]; scheduledTaskId?: string; scheduledRunId?: string } = {}): Promise<void> {const syntheticMsg: ChannelMessage = {- id: `internal-${Date.now().toString(36)}`,+ id: opts.scheduledRunId ? `scheduled-${opts.scheduledRunId}` : `internal-${Date.now().toString(36)}`,channelId: channelId || 'internal',channelType: (channelType || 'internal') as ChannelType,senderId: 'system',content: prompt,timestamp: Date.now(),...(opts.allowTools?.length || opts.scheduledTaskId- ? { metadata: { ...(opts.allowTools?.length ? { allowTools: opts.allowTools } : {}), ...(opts.scheduledTaskId ? { scheduledTaskId: opts.scheduledTaskId } : {}) } }+ ? { metadata: { ...(opts.allowTools?.length ? { allowTools: opts.allowTools } : {}), ...(opts.scheduledTaskId ? { scheduledTaskId: opts.scheduledTaskId, scheduledRunId: opts.scheduledRunId } : {}) } }: {}),};- this.enqueueMessage(syntheticMsg);+ if (opts.scheduledTaskId) {+ // A scheduled prompt is work, never an interactive slash command/dialog answer.+ const accepted = this.workLedger.accept(syntheticMsg);+ if (accepted.accepted) {+ this.queueMessage(syntheticMsg, accepted.entry.key);+ void this.processQueue();+ }+ } else {+ this.enqueueMessage(syntheticMsg);+ }}- private async handleScheduledTask(manifest: ScheduledTaskManifest): Promise<void> {+ private async handleScheduledTask(manifest: ScheduledTaskManifest, runId: string): Promise<ScheduledDispatch> {// Bot routines run on the bot lane, never through the main agent loop// (BOTS-ARCHITECTURE.md §2.3 — cron lane).if (manifest.botId) {if (!this.botManager) {- logger.warn({ task: manifest.id, botId: manifest.botId }, 'Bot routine fired but BotManager is not wired');- return;+ throw new Error('Bot routine fired but BotManager is not wired');}logger.info({ task: manifest.id, botId: manifest.botId }, 'Bot routine firing');const result = this.botManager.enqueue(manifest.botId, {@@ -5002,8 +5022,10 @@ export class Agent {// Gate skips (busy, too soon, paused) are the governor working, not faults.const level = result.reasonCode === 'busy' || result.reasonCode === 'too_soon' || result.reasonCode === 'routine_paused' ? 'info' : 'warn';logger[level]({ botId: manifest.botId, task: manifest.id, reasonCode: result.reasonCode }, 'Bot routine tick skipped');+ if (['busy', 'too_soon', 'routine_paused', 'stopped', 'target_disabled'].includes(result.reasonCode ?? '')) return 'skipped';+ throw new Error(`Bot schedule was not accepted: ${result.reasonCode ?? 'unknown error'}`);}- return;+ return 'dispatched';}logger.info({ task: manifest.id, channel: manifest.sourceChannelType }, 'Processing scheduled task');try {@@ -5014,7 +5036,7 @@ export class Agent {if (channel && manifest.sourceChannelType !== 'internal') {const skillInfo = manifest.skillName ? ` (${manifest.skillName})` : '';await channel.send(- ` Scheduled task started${skillInfo}: ${manifest.description}\nAll actions auto-approved for this run.`,+ ` Scheduled task started${skillInfo}: ${manifest.description}\nExisting tool permissions apply to this run.`,manifest.sourceChannelId,).catch((e) => logger.warn({ e }, 'channel send failed'));}@@ -5027,9 +5049,11 @@ export class Agent {if (!prompt) {prompt = `Execute scheduled task: ${manifest.description}`;}- await this.processInternalPrompt(prompt, manifest.sourceChannelId, manifest.sourceChannelType, { allowTools: manifest.allowTools, scheduledTaskId: manifest.id });+ await this.processInternalPrompt(prompt, manifest.sourceChannelId, manifest.sourceChannelType, { allowTools: manifest.allowTools, scheduledTaskId: manifest.id, scheduledRunId: runId });+ return 'queued';} catch (err) {logger.error({ err, task: manifest.id }, 'Scheduled task execution failed');+ throw err;}}diff --git a/src/core/scheduled-dispatch.test.ts b/src/core/scheduled-dispatch.test.tsnew file mode 100644index 00000000..3eb73d52--- /dev/null+++ b/src/core/scheduled-dispatch.test.ts@@ -0,0 +1,66 @@+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { Agent } from './agent.js';+import { WorkLedger } from './work-ledger.js';+import { Scheduler, type ScheduledTaskManifest } from './scheduler.js';+import type { MercuryConfig } from '../utils/config.js';++const dirs: string[] = [];+afterEach(() => {+ vi.useRealTimers();+ vi.unstubAllEnvs();+ dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }));+});++/** Exercise the real scheduler -> internal prompt -> durable queue -> finalization path. */+function harness(fail = false) {+ const dir = mkdtempSync(join(tmpdir(), 'scheduled-dispatch-'));+ dirs.push(dir);+ vi.stubEnv('MERCURY_HOME', dir);+ const workLedger = new WorkLedger({ filePath: join(dir, 'work-ledger.json') });+ const agent = Object.create(Agent.prototype) as Agent;+ const internal = agent as unknown as {+ currentWorkKey: string;+ handleScheduledTask(manifest: ScheduledTaskManifest, runId: string): Promise<'queued'>;+ processQueue(): Promise<void>;+ };+ const scheduler = new Scheduler({ heartbeat: { intervalMinutes: 30 } } as MercuryConfig,+ (manifest, runId) => internal.handleScheduledTask(manifest, runId));+ const effect = vi.fn();+ Object.assign(agent, {+ scheduler, workLedger, messageQueue: [], queuedWorkKeys: new Set(), processing: false, running: false,+ currentMessage: null, currentWorkKey: null,+ channels: { get: () => undefined, getNotificationChannel: () => undefined },+ lifecycle: { is: () => true },+ announceQueue: () => {},+ withSessionProgramming: (_msg: unknown, run: () => Promise<void>) => run(),+ handleMessage: async () => {+ effect();+ if (fail) throw new Error('Lost connection after performing an effect');+ workLedger.markCompleted(internal.currentWorkKey, 'Done');+ },+ });+ return { agent, internal, scheduler, effect, workLedger };+}++describe('scheduled main-agent dispatch', () => {+ it.each([false, true])('tracks execution rather than enqueue acceptance (failure=%s)', async (fail) => {+ vi.useFakeTimers();+ const { agent, internal, scheduler, effect } = harness(fail);+ scheduler.addDelayedTask({ id: 'scheduled', description: 'Run work', prompt: '/a prompt is not a slash command', delaySeconds: 1, createdAt: new Date().toISOString() });+ await vi.advanceTimersByTimeAsync(1000);+ expect(scheduler.getManifest('scheduled')?.lastRun?.status).toBe('running');+ expect(effect).not.toHaveBeenCalled();+ Object.assign(agent, { running: true });+ await internal.processQueue();+ expect(effect).toHaveBeenCalledOnce();+ expect(scheduler.getManifest('scheduled')?.lastRun?.status).toBe(fail ? 'failed' : 'completed');+ scheduler.stopAll();+ scheduler.restorePersistedTasks();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(effect).toHaveBeenCalledOnce();+ scheduler.stopAll();+ });+});diff --git a/src/core/scheduler.test.ts b/src/core/scheduler.test.tsnew file mode 100644index 00000000..fb40564a--- /dev/null+++ b/src/core/scheduler.test.ts@@ -0,0 +1,156 @@+import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import type { MercuryConfig } from '../utils/config.js';+import { loadSchedules, saveSchedules, Scheduler, type ScheduledTaskManifest } from './scheduler.js';++const config = { heartbeat: { intervalMinutes: 30 } } as MercuryConfig;+const NOW = new Date('2026-11-06T13:59:58Z');+let home: string;+let schedulers: Scheduler[];+function create(handler: ConstructorParameters<typeof Scheduler>[1] = async () => {}) {+ const scheduler = new Scheduler(config, handler);+ schedulers.push(scheduler);+ return scheduler;+}+function task(overrides: Partial<ScheduledTaskManifest> = {}): ScheduledTaskManifest {+ return { id: 'reminder', description: 'Remind me', prompt: 'Notify me', delaySeconds: 1, executeAt: new Date(Date.now() + 1000).toISOString(), createdAt: NOW.toISOString(), ...overrides };+}+beforeEach(() => {+ home = mkdtempSync(join(tmpdir(), 'mercury-scheduler-'));+ vi.stubEnv('MERCURY_HOME', home);+ vi.useFakeTimers();+ vi.setSystemTime(NOW);+ schedulers = [];+});+afterEach(() => {+ schedulers.forEach((scheduler) => scheduler.stopAll());+ vi.useRealTimers();+ vi.unstubAllEnvs();+ rmSync(home, { recursive: true, force: true });+});++describe('durable schedules', () => {+ it('persists running intent before invoking an effect, then retains confirmed completion', async () => {+ const handler = vi.fn(async () => {+ expect(loadSchedules()[0].lastRun?.status).toBe('running');+ });+ const scheduler = create(handler);+ scheduler.addDelayedTask(task());+ scheduler.persistSchedules();+ await vi.advanceTimersByTimeAsync(1000);+ expect(handler).toHaveBeenCalledOnce();+ expect(loadSchedules()[0].lastRun).toMatchObject({ status: 'completed', finishedAt: new Date().toISOString() });+ if (process.platform !== 'win32') expect(statSync(join(home, 'schedules.yaml')).mode & 0o777).toBe(0o600);+ expect(readFileSync(join(home, 'schedules.yaml'), 'utf8')).toContain('completed');+ });++ it('retains a failed one-shot and never repeats it on restart', async () => {+ const handler = vi.fn(async () => { throw new Error('Connection lost after sending'); });+ const scheduler = create(handler);+ scheduler.addDelayedTask(task({ missedRunPolicy: 'run_once' }));+ await vi.advanceTimersByTimeAsync(1000);+ expect(loadSchedules()[0]).toMatchObject({ paused: true, lastRun: { status: 'failed', error: 'Connection lost after sending' } });+ scheduler.stopAll();+ create(handler).restorePersistedTasks();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(handler).toHaveBeenCalledOnce();+ });++ it('marks unconfirmed work interrupted and pauses it even with catch-up enabled', async () => {+ const handler = vi.fn(async () => 'queued' as const);+ const first = create(handler);+ first.addDelayedTask(task({ missedRunPolicy: 'run_once' }));+ await vi.advanceTimersByTimeAsync(1000);+ expect(loadSchedules()[0].lastRun?.status).toBe('running');+ first.stopAll();+ const restarted = create(handler);+ restarted.restorePersistedTasks();+ await vi.advanceTimersByTimeAsync(1000);+ expect(handler).toHaveBeenCalledOnce();+ expect(restarted.getManifest('reminder')).toMatchObject({ paused: true, lastRun: { status: 'interrupted' } });+ // A metadata-only edit must not turn uncertain side effects into a retry.+ restarted.replaceManifest({ ...restarted.getManifest('reminder')!, description: 'renamed' });+ await vi.advanceTimersByTimeAsync(2000);+ expect(handler).toHaveBeenCalledOnce();+ restarted.replaceManifest(restarted.getManifest('reminder')!, true);+ await vi.advanceTimersByTimeAsync(1000);+ expect(handler).toHaveBeenCalledTimes(2);+ });++ it('skips legacy overdue one-shots but catches up explicitly opted-in work exactly once', async () => {+ saveSchedules([+ task({ id: 'legacy', executeAt: new Date(Date.now() - 1000).toISOString() }),+ task({ id: 'catch-up', missedRunPolicy: 'run_once', executeAt: new Date(Date.now() - 1000).toISOString() }),+ ]);+ const handler = vi.fn(async () => {});+ const first = create(handler);+ first.restorePersistedTasks();+ await vi.advanceTimersByTimeAsync(1);+ expect(handler).toHaveBeenCalledOnce();+ expect(handler).toHaveBeenCalledWith(expect.objectContaining({ id: 'catch-up' }), expect.any(String));+ expect(first.getManifest('legacy')?.lastRun?.status).toBe('skipped');+ first.stopAll();+ create(handler).restorePersistedTasks();+ await vi.advanceTimersByTimeAsync(1000);+ expect(handler).toHaveBeenCalledOnce();+ });++ it('does not overlap a main-agent run that has only been queued', async () => {+ const handler = vi.fn(async () => 'queued' as const);+ const scheduler = create(handler);+ scheduler.addPersistedTask(task({ delaySeconds: undefined, executeAt: undefined, cron: '* * * * * *' }));+ await vi.advanceTimersByTimeAsync(4000);+ expect(handler).toHaveBeenCalledOnce();+ const runId = scheduler.getManifest('reminder')!.lastRun!.id;+ scheduler.finishRun('reminder', runId, 'completed');+ await vi.advanceTimersByTimeAsync(1000);+ expect(handler).toHaveBeenCalledTimes(2);+ scheduler.finishRun('reminder', runId, 'failed', 'stale result');+ expect(scheduler.getManifest('reminder')!.lastRun!.status).toBe('running');+ });++ it('fires cron in the selected IANA zone, independent of the host timezone', async () => {+ const handler = vi.fn(async () => {});+ create(handler).addPersistedTask(task({ delaySeconds: undefined, executeAt: undefined, cron: '0 9 * * *', timezone: 'America/New_York' }));+ await vi.advanceTimersByTimeAsync(2000); // 14:00 UTC = 09:00 EST+ expect(handler).toHaveBeenCalledOnce();+ });++ it('chunks delays beyond Node’s timer limit without firing early', async () => {+ const handler = vi.fn(async () => {});+ const delaySeconds = 30 * 24 * 3600;+ create(handler).addDelayedTask(task({ delaySeconds, executeAt: new Date(Date.now() + delaySeconds * 1000).toISOString() }));+ await vi.advanceTimersByTimeAsync(2 ** 31 - 1);+ expect(handler).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(delaySeconds * 1000 - (2 ** 31 - 1));+ expect(handler).toHaveBeenCalledOnce();+ });++ it('bounds resolved one-shot history while retaining active and unresolved work', () => {+ const finished = Array.from({ length: 105 }, (_, i) => task({ id: `done-${i}`, lastRun: { id: String(i), status: 'completed', startedAt: NOW.toISOString(), finishedAt: NOW.toISOString() } }));+ saveSchedules([...finished, task({ id: 'pending' }), task({ id: 'failed', paused: true, lastRun: { id: 'f', status: 'failed', startedAt: NOW.toISOString() } })]);+ const scheduler = create();+ scheduler.restorePersistedTasks();+ expect(scheduler.getManifests()).toHaveLength(102);+ expect(scheduler.getManifest('pending')).not.toBeNull();+ expect(scheduler.getManifest('failed')).not.toBeNull();+ });++ it('leaves malformed persisted schedules untouched instead of overwriting recovery evidence', () => {+ const path = join(home, 'schedules.yaml');+ const invalid = 'tasks: [malformed';+ writeFileSync(path, invalid);+ create().restorePersistedTasks();+ expect(readFileSync(path, 'utf8')).toBe(invalid);+ });++ it('validates before changing an existing schedule', () => {+ const scheduler = create();+ scheduler.addDelayedTask(task());+ expect(() => scheduler.replaceManifest(task({ delaySeconds: Infinity }))).toThrow();+ expect(() => scheduler.replaceManifest(task({ timezone: 'Mars/Olympus' }))).toThrow();+ expect(scheduler.getManifest('reminder')?.delaySeconds).toBe(1);+ });+});diff --git a/src/core/scheduler.ts b/src/core/scheduler.tsindex b14720cc..f667518e 100644--- a/src/core/scheduler.ts+++ b/src/core/scheduler.ts@@ -1,5 +1,6 @@import cron from 'node-cron';-import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';+import { randomUUID } from 'node:crypto';import { join } from 'node:path';import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';import type { MercuryConfig } from '../utils/config.js';@@ -11,8 +12,21 @@ export interface ScheduledTask {cron: string;handler: () => Promise<void>;description: string;+ timezone?: string;}+export type ScheduledRunStatus = 'running' | 'completed' | 'failed' | 'interrupted' | 'skipped' | 'dispatched';+export interface ScheduledRun {+ id: string;+ status: ScheduledRunStatus;+ startedAt: string;+ finishedAt?: string;+ error?: string;+}++/** Queued main-agent work completes via finishRun; bots own a separate work ledger. */+export type ScheduledDispatch = void | 'queued' | 'dispatched' | 'skipped';+export interface ScheduledTaskManifest {id: string;cron?: string;@@ -22,6 +36,12 @@ export interface ScheduledTaskManifest {delaySeconds?: number;executeAt?: string;createdAt: string;+ /** IANA timezone for cron; omitted means the host timezone for legacy schedules. */+ timezone?: string;+ /** Overdue one-shots default to skip, preserving the legacy no-replay behavior. */+ missedRunPolicy?: 'skip' | 'run_once';+ paused?: boolean;+ lastRun?: ScheduledRun;sourceChannelId?: string;sourceChannelType?: string;/** Set for Mercury Bot routines — the run routes to the bot, not the main agent. */@@ -46,7 +66,7 @@ export function loadSchedules(): ScheduledTaskManifest[] {try {const raw = readFileSync(path, 'utf-8');const data = parseYaml(raw) as { tasks?: ScheduledTaskManifest[] };- return data.tasks || [];+ return Array.isArray(data?.tasks) ? data.tasks : [];} catch (err) {logger.warn({ err }, 'Failed to load schedules.yaml');return [];@@ -55,13 +75,37 @@ export function loadSchedules(): ScheduledTaskManifest[] {export function saveSchedules(tasks: ScheduledTaskManifest[]): void {const path = getSchedulesPath();- const dir = getMercuryHome();- if (!existsSync(dir)) {- mkdirSync(dir, { recursive: true });+ mkdirSync(getMercuryHome(), { recursive: true, mode: 0o700 });+ const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;+ let fd: number | undefined;+ try {+ fd = openSync(temp, 'wx', 0o600);+ writeFileSync(fd, stringifyYaml({ tasks }), 'utf-8');+ fsyncSync(fd);+ closeSync(fd);+ fd = undefined;+ renameSync(temp, path);+ // Directory fsync is unavailable on some platforms (notably Windows).+ try {+ fd = openSync(getMercuryHome(), 'r');+ fsyncSync(fd);+ } catch { /* the file itself has already been synced */ }+ } finally {+ if (fd !== undefined) closeSync(fd);+ if (existsSync(temp)) unlinkSync(temp);}- writeFileSync(path, stringifyYaml({ tasks }), 'utf-8');}+export function isValidScheduleTimezone(timezone: string): boolean {+ try {+ new Intl.DateTimeFormat('en', { timeZone: timezone }).format();+ return true;+ } catch { return false; }+}++const MAX_TIMER_MS = 2 ** 31 - 1;+const MAX_FINISHED_ONESHOTS = 100;+export class Scheduler {private tasks: Map<string, cron.ScheduledTask> = new Map();private delayedTasks: Map<string, NodeJS.Timeout> = new Map();@@ -72,12 +116,12 @@ export class Scheduler {constructor(config: MercuryConfig,- private onScheduledTask?: (manifest: ScheduledTaskManifest) => Promise<void>,+ private onScheduledTask?: (manifest: ScheduledTaskManifest, runId: string) => Promise<ScheduledDispatch>,) {this.heartbeatIntervalMinutes = config.heartbeat.intervalMinutes;}- setOnScheduledTask(handler: (manifest: ScheduledTaskManifest) => Promise<void>): void {+ setOnScheduledTask(handler: (manifest: ScheduledTaskManifest, runId: string) => Promise<ScheduledDispatch>): void {this.onScheduledTask = handler;}@@ -108,56 +152,93 @@ export class Scheduler {}addTask(task: ScheduledTask): void {- if (this.tasks.has(task.id)) {- this.removeTask(task.id);- }+ this.tasks.get(task.id)?.stop();+ let running = false;const scheduled = cron.schedule(task.cron, async () => {+ if (running) return; // Cron ticks never build an overlapping backlog.+ running = true;try {await task.handler();} catch (err) {logger.error({ task: task.id, err }, 'Scheduled task error');- }- });+ } finally { running = false; }+ }, { timezone: task.timezone });this.tasks.set(task.id, scheduled);- logger.info({ id: task.id, cron: task.cron, desc: task.description }, 'Task scheduled');+ }++ private validate(manifest: ScheduledTaskManifest): void {+ if (manifest.timezone !== undefined && !isValidScheduleTimezone(manifest.timezone)) {+ throw new Error('Invalid schedule timezone');+ }+ if (manifest.missedRunPolicy !== undefined && !['skip', 'run_once'].includes(manifest.missedRunPolicy)) {+ throw new Error('Invalid missed-run policy');+ }+ if (manifest.cron && cron.validate(manifest.cron) && manifest.delaySeconds === undefined) return;+ if (!manifest.cron && Number.isFinite(manifest.delaySeconds) && manifest.delaySeconds! > 0+ && (!manifest.executeAt || Number.isFinite(Date.parse(manifest.executeAt)))) return;+ throw new Error('Manifest requires a valid cron or positive finite delaySeconds');}addPersistedTask(manifest: ScheduledTaskManifest): void {- this.taskManifests.set(manifest.id, manifest);+ this.validate(manifest);+ // Registering a bot's declared routine again must not erase interrupted state.+ const previous = this.taskManifests.get(manifest.id);+ const next = { ...manifest, lastRun: manifest.lastRun ?? previous?.lastRun, paused: manifest.paused ?? previous?.paused };+ this.removeTask(manifest.id);+ this.taskManifests.set(manifest.id, next);+ if (next.paused) return;this.addTask({- id: manifest.id,- cron: manifest.cron!,- description: manifest.description,- handler: async () => {- logger.info({ task: manifest.id }, 'Scheduled task firing');- if (this.onScheduledTask) {- await this.onScheduledTask(manifest);- }- },+ id: next.id, cron: next.cron!, description: next.description, timezone: next.timezone,+ handler: () => this.runTask(next.id),});}addDelayedTask(manifest: ScheduledTaskManifest): void {- this.taskManifests.set(manifest.id, manifest);- const delayMs = (manifest.delaySeconds || 60) * 1000;+ this.validate(manifest);+ const next = { ...manifest, executeAt: manifest.executeAt ?? new Date(Date.now() + manifest.delaySeconds! * 1000).toISOString() };+ this.removeTask(next.id);+ this.taskManifests.set(next.id, next);+ if (!next.paused && !next.lastRun) this.armDelayedTask(next);+ }- const timer = setTimeout(async () => {- try {- logger.info({ task: manifest.id }, 'Delayed task firing');- if (this.onScheduledTask) {- await this.onScheduledTask(manifest);- }- } catch (err) {- logger.error({ task: manifest.id, err }, 'Delayed task error');- } finally {- this.delayedTasks.delete(manifest.id);- this.taskManifests.delete(manifest.id);- this.persistSchedules();+ private armDelayedTask(manifest: ScheduledTaskManifest): void {+ const remaining = Date.parse(manifest.executeAt!) - Date.now();+ const timer = setTimeout(() => {+ this.delayedTasks.delete(manifest.id);+ if (this.taskManifests.get(manifest.id) !== manifest) return;+ if (Date.parse(manifest.executeAt!) > Date.now()) {+ this.armDelayedTask(manifest); // Node timers longer than ~24 days otherwise fire immediately.+ } else {+ void this.runTask(manifest.id).catch((err) => logger.error({ err, task: manifest.id }, 'Could not persist scheduled run'));}- }, delayMs);-+ }, Math.min(MAX_TIMER_MS, Math.max(0, remaining)));this.delayedTasks.set(manifest.id, timer);- logger.info({ id: manifest.id, delaySeconds: manifest.delaySeconds }, 'Delayed task scheduled');+ }++ private async runTask(id: string): Promise<void> {+ const manifest = this.taskManifests.get(id);+ if (!manifest || manifest.paused || manifest.lastRun?.status === 'running') return;+ const runId = randomUUID();+ manifest.lastRun = { id: runId, status: 'running', startedAt: new Date().toISOString() };+ // Fail closed: no handler runs unless the intent is durable first.+ this.persistSchedules();+ try {+ if (!this.onScheduledTask) throw new Error('Scheduled task handler unavailable');+ const outcome = await this.onScheduledTask(manifest, runId);+ if (outcome !== 'queued') this.finishRun(id, runId, outcome ?? 'completed');+ } catch (err) {+ this.finishRun(id, runId, 'failed', err instanceof Error ? err.message : String(err));+ logger.error({ err, task: id }, 'Scheduled task failed; review before resuming');+ }+ }++ /** A run identity prevents a late completion from changing an edited/deleted task. */+ finishRun(id: string, runId: string, status: Exclude<ScheduledRunStatus, 'running'>, error?: string): void {+ const manifest = this.taskManifests.get(id);+ if (!manifest || manifest.lastRun?.id !== runId || manifest.lastRun.status !== 'running') return;+ manifest.lastRun = { ...manifest.lastRun, status, finishedAt: new Date().toISOString(), ...(error ? { error: error.slice(0, 1000) } : {}) };+ if (status === 'failed' || status === 'interrupted') manifest.paused = true;+ this.persistSchedules();}removeTask(id: string): void {@@ -182,45 +263,53 @@ export class Scheduler {return this.taskManifests.get(id) || null;}- replaceManifest(manifest: ScheduledTaskManifest): void {- this.removeTask(manifest.id);- if (manifest.delaySeconds) {- const delay = Math.max(1, manifest.delaySeconds);- manifest.delaySeconds = delay;- manifest.executeAt = new Date(Date.now() + delay * 1000).toISOString();- this.addDelayedTask(manifest);- } else if (manifest.cron) {- this.addPersistedTask(manifest);- } else {- throw new Error('Manifest requires cron or delaySeconds');+ replaceManifest(manifest: ScheduledTaskManifest, resume = false): void {+ this.validate(manifest);+ if (this.getManifest(manifest.id)?.lastRun?.status === 'running') throw new Error('Cannot edit a running schedule');+ const next = { ...manifest };+ if (resume) {+ next.paused = false;+ next.lastRun = undefined;+ if (next.delaySeconds) next.executeAt = new Date(Date.now() + next.delaySeconds * 1000).toISOString();}+ this.removeTask(next.id);+ if (next.cron) this.addPersistedTask(next);+ else this.addDelayedTask(next);}restorePersistedTasks(): void {const persisted = loadSchedules();- for (const manifest of persisted) {- if (manifest.delaySeconds) {- const executeAt = manifest.executeAt ? new Date(manifest.executeAt) : null;- const now = Date.now();- if (executeAt && executeAt.getTime() > now) {- const remainingMs = executeAt.getTime() - now;- manifest.delaySeconds = Math.ceil(remainingMs / 1000);- this.addDelayedTask(manifest);+ for (const stored of persisted) {+ const manifest = { ...stored };+ try {+ this.validate(manifest);+ if (manifest.lastRun?.status === 'running') {+ manifest.lastRun = { ...manifest.lastRun, status: 'interrupted', finishedAt: new Date().toISOString(), error: 'Mercury stopped before this run was confirmed. Check its effects before resuming.' };+ manifest.paused = true;+ }+ if (manifest.cron) {+ this.addPersistedTask(manifest);} else {- logger.info({ id: manifest.id }, 'Delayed task already expired, skipping');+ // Old schedules without executeAt cannot safely be inferred as pending.+ const overdue = !manifest.executeAt || Date.parse(manifest.executeAt) <= Date.now();+ if (!manifest.lastRun && overdue && (manifest.missedRunPolicy ?? 'skip') === 'skip') {+ manifest.lastRun = { id: randomUUID(), status: 'skipped', startedAt: new Date().toISOString(), finishedAt: new Date().toISOString(), error: 'Missed while Mercury was offline (policy: skip).' };+ }+ if (overdue && !manifest.lastRun) manifest.executeAt = new Date().toISOString();+ this.addDelayedTask(manifest);}- } else if (manifest.cron && cron.validate(manifest.cron)) {- this.addPersistedTask(manifest);- } else {- logger.warn({ id: manifest.id, cron: manifest.cron }, 'Skipping invalid task');+ } catch (err) {+ logger.warn({ err, id: manifest.id }, 'Skipping invalid task');}}- if (persisted.length > 0) {- logger.info({ count: persisted.length }, 'Restored persisted scheduled tasks');- }+ if (persisted.length > 0) this.persistSchedules();}persistSchedules(): void {+ // Keep active work and all unresolved failures. Bound successful/skipped one-shot history.+ const finished = this.getManifests().filter((m) => !m.cron && m.lastRun && ['completed', 'skipped', 'dispatched'].includes(m.lastRun.status))+ .sort((a, b) => (b.lastRun!.finishedAt ?? '').localeCompare(a.lastRun!.finishedAt ?? ''));+ for (const manifest of finished.slice(MAX_FINISHED_ONESHOTS)) this.removeTask(manifest.id);saveSchedules(this.getManifests());}diff --git a/src/core/work-ledger.test.ts b/src/core/work-ledger.test.tsindex 3e91052e..cfaa3fb9 100644--- a/src/core/work-ledger.test.ts+++ b/src/core/work-ledger.test.ts@@ -214,3 +214,17 @@ describe('WorkLedger', () => {expect(ledger.get(entry.key)?.status).toBe('cancelled');});});++describe('scheduled work recovery', () => {+ it.each(['queued', 'running', 'paused', 'failed'] as const)('never replays %s scheduled work after restart', (state) => {+ const { ledger, filePath } = setup();+ const entry = ledger.accept(message('schedule', { scheduledTaskId: 'task-1', scheduledRunId: 'run-1' })).entry;+ if (state === 'running') ledger.markRunning(entry.key);+ if (state === 'paused') ledger.markPaused(entry.key, 'Side effects may be partial');+ if (state === 'failed') ledger.markFailed(entry.key, 'interrupted/ambiguous');+ const restarted = new WorkLedger({ filePath });+ expect(restarted.recoverInterrupted()).toEqual([]);+ expect(restarted.get(entry.key)).toMatchObject({ status: 'failed', error: expect.stringContaining('explicitly reschedule') });+ expect(new WorkLedger({ filePath }).recoverInterrupted()).toEqual([]);+ });+});diff --git a/src/core/work-ledger.ts b/src/core/work-ledger.tsindex 192e6dd5..02571e5e 100644--- a/src/core/work-ledger.ts+++ b/src/core/work-ledger.ts@@ -239,6 +239,12 @@ export class WorkLedger {const recoverableFailure = entry.status === 'failed' && /interrupted\/ambiguous|side effects may be partial|after one or more tools completed|partial output/i.test(entry.error || '');const isPaused = entry.status === 'paused';if (entry.status !== 'queued' && entry.status !== 'running' && !recoverableFailure && !isPaused) continue;+ if (typeof entry.message.metadata?.scheduledTaskId === 'string') {+ // The scheduler owns recovery. Never replay a turn whose external effects+ // may have happened before the process stopped, even if it was queued.+ this.markFailed(entry.key, 'Scheduled run interrupted; inspect its effects and explicitly reschedule it.');+ continue;+ }const interrupted = entry.status === 'running' || isPaused;entry.status = 'queued';entry.updatedAt = this.now();diff --git a/src/eval/scenarios/scheduled-work.test.ts b/src/eval/scenarios/scheduled-work.test.tsnew file mode 100644index 00000000..94772c08--- /dev/null+++ b/src/eval/scenarios/scheduled-work.test.ts@@ -0,0 +1,111 @@+import { afterAll, describe, expect, it, vi } from 'vitest';+// Import the harness before production modules to establish its scratch home.+import { EvalHarness } from '../runner.js';+import { readFileSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { performance } from 'node:perf_hooks';+import { loadSchedules } from '../../core/scheduler.js';+import { WorkLedger } from '../../core/work-ledger.js';+import { call, textStep, toolStep } from '../fixture.js';++afterAll(async () => (await import('../env.js')).removeEvalRoot());++async function settleUntil(done: () => boolean): Promise<void> {+ const deadline = performance.now() + 5_000;+ while (!done()) {+ if (performance.now() > deadline) throw new Error('Scheduled agent turn did not settle');+ await new Promise<void>((resolve) => setImmediate(resolve));+ vi.advanceTimersByTime(0);+ }+}++describe.each([false, true])('eval · scheduled real work (streaming=%s)', (streaming) => {+ it.each([false, true])('persists the actual agent outcome and never replays the one-shot (provider fails=%s)', async (fails) => {+ const harness = await EvalHarness.create({+ name: `scheduled-real-work-${streaming}-${fails}`,+ channel: { streaming },+ providers: [{ name: 'primary' }, { name: 'backup' }],+ turns: [],+ });+ const target = join(harness.workDir, 'scheduled-receipt.txt');+ const observedAfterWrite: Array<{ status: string | undefined; content: string }> = [];+ const scheduler = harness.agent.scheduler;+ try {+ harness.player.load({+ message: 'Create scheduled-receipt.txt containing accepted.',+ steps: [toolStep([call('create_file', { path: 'scheduled-receipt.txt', content: 'accepted' })])],+ onExhausted: () => {+ // This is the next real SDK model call, after the real filesystem+ // tool has returned. Enqueueing alone must not complete the run.+ observedAfterWrite.push({ status: loadSchedules()[0]?.lastRun?.status, content: readFileSync(target, 'utf8') });+ if (fails) throw new Error('Provider connection lost after scheduled write');+ return textStep('Created scheduled-receipt.txt with accepted.');+ },+ });+ scheduler.addDelayedTask({+ id: 'scheduled-receipt',+ description: 'Write a receipt once',+ prompt: 'Create scheduled-receipt.txt containing accepted.',+ delaySeconds: 1,+ // Due now, so only zero-delay fake timers need to run. No heartbeat,+ // watchdog, or guessed sleep participates in the test.+ executeAt: new Date(Date.now() - 1).toISOString(),+ createdAt: new Date().toISOString(),+ missedRunPolicy: 'run_once',+ sourceChannelId: harness.channelId,+ sourceChannelType: harness.channelType,+ });+ scheduler.persistSchedules();+ vi.advanceTimersByTime(0);+ await settleUntil(() => {+ const status = scheduler.getManifest('scheduled-receipt')?.lastRun?.status;+ return status === 'completed' || status === 'failed';+ });++ expect(observedAfterWrite).toEqual([{ status: 'running', content: 'accepted' }]);+ expect(harness.capabilities.toolLog).toHaveLength(1);+ expect(harness.capabilities.toolLog[0]).toMatchObject({ tool: 'create_file', recorded: false });+ expect(harness.player.calls).toHaveLength(2);+ expect(harness.providers[1].callCount).toBe(0);+ expect(readFileSync(target, 'utf8')).toBe('accepted');++ const saved = loadSchedules()[0];+ expect(saved.lastRun?.status).toBe(fails ? 'failed' : 'completed');+ expect(saved.lastRun?.finishedAt).toBeTruthy();+ if (fails) {+ expect(saved.paused).toBe(true);+ expect(saved.lastRun?.error).toContain('interrupted/ambiguous');+ expect(harness.channel.events.some((event) => event.text.includes('interrupted/ambiguous'))).toBe(true);+ expect(harness.channel.events.some((event) => event.text.includes('Task complete'))).toBe(false);+ } else {+ expect(harness.channel.events.some((event) => event.kind === 'delivered' && event.text.includes('Created scheduled-receipt.txt'))).toBe(true);+ }++ const workKey = `channel:${harness.channelType}:scheduled-${saved.lastRun!.id}`;+ const reloadedLedger = new WorkLedger();+ expect(reloadedLedger.get(workKey)).toMatchObject({+ status: fails ? 'failed' : 'completed',+ attempts: 1,+ message: { metadata: { scheduledTaskId: saved.id, scheduledRunId: saved.lastRun!.id } },+ });+ expect(reloadedLedger.get(workKey)?.nextAttemptAt).toBeUndefined();+ // In particular, the ordinary chat interrupted-work recovery path must+ // not enqueue a failed scheduled turn for a second side effect.+ expect(reloadedLedger.recoverInterrupted()).toEqual([]);++ writeFileSync(target, 'later owner edit');+ scheduler.stopAll();+ scheduler.restorePersistedTasks();+ // This passes every ordinary short retry delay without a live heartbeat.+ // It also catches a restored overdue run_once timer being armed again.+ vi.advanceTimersByTime(60_000);+ await new Promise<void>((resolve) => setImmediate(resolve));+ expect(scheduler.getManifest(saved.id)?.lastRun).toEqual(saved.lastRun);+ expect(harness.player.calls).toHaveLength(2);+ expect(harness.capabilities.toolLog).toHaveLength(1);+ expect(readFileSync(target, 'utf8')).toBe('later owner edit');+ } finally {+ await harness.dispose();+ }+ });+});diff --git a/src/web/api/schedules-route.test.ts b/src/web/api/schedules-route.test.tsindex ca41e01f..f99dcf81 100644--- a/src/web/api/schedules-route.test.ts+++ b/src/web/api/schedules-route.test.ts@@ -1,5 +1,10 @@-import { describe, expect, it, vi } from 'vitest';+import { afterEach, describe, expect, it, vi } from 'vitest';import system, { setScheduler } from './system.js';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { Scheduler } from '../../core/scheduler.js';+import type { MercuryConfig } from '../../utils/config.js';/** The status bar's schedule list deletes through this route. */function fakeScheduler(ids: string[]) {@@ -29,3 +34,42 @@ describe('DELETE /api/schedules/:id', () => {expect((await system.request('http://127.0.0.1/api/schedules/zzz', { method: 'DELETE' })).status).toBe(404);});});++const dirs: string[] = [];+const realSchedulers: Scheduler[] = [];+afterEach(() => {+ realSchedulers.splice(0).forEach((scheduler) => scheduler.stopAll());+ setScheduler(null);+ vi.unstubAllEnvs();+ dirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true }));+});+function realSchedule() {+ const dir = mkdtempSync(join(tmpdir(), 'schedule-route-'));+ dirs.push(dir);+ vi.stubEnv('MERCURY_HOME', dir);+ const scheduler = new Scheduler({ heartbeat: { intervalMinutes: 30 } } as MercuryConfig);+ realSchedulers.push(scheduler);+ scheduler.addDelayedTask({ id: 'failed', description: 'Remind me', prompt: 'Notify', delaySeconds: 60, executeAt: '2026-01-01T00:00:00Z', createdAt: '2025-12-31T00:00:00Z', paused: true, lastRun: { id: 'run', status: 'failed', startedAt: '2026-01-01T00:00:00Z', error: 'Unknown send outcome' } });+ setScheduler(scheduler);+ return scheduler;+}+function update(body: Record<string, unknown>) {+ return system.request('http://127.0.0.1/api/schedules/failed', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });+}+describe('schedule policy editing', () => {+ it('keeps failed work paused on metadata edits; explicit resume rearms it', async () => {+ const scheduler = realSchedule();+ expect((await update({ description: 'Renamed', missedRunPolicy: 'run_once' })).status).toBe(200);+ expect(scheduler.getManifest('failed')).toMatchObject({ paused: true, lastRun: { status: 'failed' }, executeAt: '2026-01-01T00:00:00Z' });+ expect((await update({ resume: true, delaySeconds: 300 })).status).toBe(200);+ expect(scheduler.getManifest('failed')).toMatchObject({ paused: false, delaySeconds: 300 });+ expect(scheduler.getManifest('failed')?.lastRun).toBeUndefined();+ expect(Date.parse(scheduler.getManifest('failed')!.executeAt!)).toBeGreaterThan(Date.now());+ });++ it.each([{ timezone: 'Not/A_Zone' }, { missedRunPolicy: 'replay_all' }, { delaySeconds: -1 }, { delaySeconds: '60' }, { resume: 'yes' }, { cron: '0 9 * * *', delaySeconds: 60 }])('rejects invalid options without changing the task: %o', async (body) => {+ const scheduler = realSchedule();+ expect((await update(body)).status).toBe(400);+ expect(scheduler.getManifest('failed')).toMatchObject({ description: 'Remind me', paused: true, lastRun: { status: 'failed' } });+ });+});diff --git a/src/web/api/system.ts b/src/web/api/system.tsindex 37ba594c..56aface6 100644--- a/src/web/api/system.ts+++ b/src/web/api/system.ts@@ -12,7 +12,7 @@ import { SkillStore } from '../../skills/store.js';import { PermissionManager, type PermissionsManifest } from '../../capabilities/permissions.js';import { getMercuryHome } from '../../utils/config.js';import { getStatus } from './status.js';-import type { Scheduler } from '../../core/scheduler.js';+import { isValidScheduleTimezone, type Scheduler } from '../../core/scheduler.js';import cron from 'node-cron';const system = new Hono();@@ -286,6 +286,18 @@ system.put('/api/schedules/:id', async (c) => {if (!existing) return c.json({ success: false, error: 'Schedule not found' }, 404);const body = await c.req.json();+ if (existing.lastRun?.status === 'running') return c.json({ success: false, error: 'Cannot edit a running schedule' }, 409);+ if (body.resume !== undefined && typeof body.resume !== 'boolean') return c.json({ success: false, error: 'resume must be boolean' }, 400);+ if (body.timezone !== undefined && (typeof body.timezone !== 'string' || (body.timezone !== '' && !isValidScheduleTimezone(body.timezone)))) {+ return c.json({ success: false, error: 'Invalid timezone' }, 400);+ }+ if (body.missedRunPolicy !== undefined && !['skip', 'run_once'].includes(body.missedRunPolicy)) {+ return c.json({ success: false, error: 'Invalid missed-run policy' }, 400);+ }+ if (body.delaySeconds !== undefined && (typeof body.delaySeconds !== 'number' || !Number.isFinite(body.delaySeconds) || body.delaySeconds <= 0)) {+ return c.json({ success: false, error: 'delaySeconds must be a positive finite number' }, 400);+ }+ if (body.cron && body.delaySeconds !== undefined) return c.json({ success: false, error: 'Provide cron or delaySeconds, not both' }, 400);const description = typeof body.description === 'string' && body.description.trim()? body.description.trim(): existing.description;@@ -320,10 +332,15 @@ system.put('/api/schedules/:id', async (c) => {skillName,cron: cronExpr,delaySeconds,- executeAt: delaySeconds ? new Date(Date.now() + delaySeconds * 1000).toISOString() : undefined,+ executeAt: delaySeconds ? existing.executeAt ?? new Date(Date.now() + delaySeconds * 1000).toISOString() : undefined,+ timezone: cronExpr ? (body.timezone === '' ? undefined : body.timezone ?? existing.timezone) : undefined,+ missedRunPolicy: cronExpr ? undefined : body.missedRunPolicy ?? existing.missedRunPolicy,};- scheduler.replaceManifest(manifest);+ // Editing a label/policy never silently repeats failed or completed work.+ // A new delay explicitly rearms a pending one-shot; terminal runs require resume.+ if (body.delaySeconds !== undefined && !existing.lastRun) manifest.executeAt = new Date(Date.now() + delaySeconds! * 1000).toISOString();+ scheduler.replaceManifest(manifest, body.resume === true);scheduler.persistSchedules();return c.json({ success: true, schedule: scheduler.getManifest(id) });});diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.tsindex c4375d5c..394a1743 100644--- a/ui/src/lib/api.ts+++ b/ui/src/lib/api.ts@@ -478,7 +478,7 @@ export const mcp = {export const schedules = {list: () => get<{ schedules: Schedule[]; total: number }>("/api/schedules"),- update: (id: string, data: Partial<Schedule>) =>+ update: (id: string, data: Partial<Schedule> & { resume?: boolean }) =>put<{ success: boolean; schedule: Schedule }>(`/api/schedules/${id}`, data),delete: (id: string) => del<{ success: boolean }>(`/api/schedules/${id}`),};@@ -1036,12 +1036,16 @@ export interface McpServer {export interface Schedule {id: string;+ executeAt?: string;+ timezone?: string;+ missedRunPolicy?: 'skip' | 'run_once';+ paused?: boolean;+ lastRun?: { id: string; status: 'running' | 'completed' | 'failed' | 'interrupted' | 'skipped' | 'dispatched'; startedAt: string; finishedAt?: string; error?: string };description: string;cron?: string;delaySeconds?: number;prompt?: string;skillName?: string;- lastRun?: string;nextRun?: string;}diff --git a/ui/src/pages/Schedules.tsx b/ui/src/pages/Schedules.tsxindex 3d713a64..e6a08359 100644--- a/ui/src/pages/Schedules.tsx+++ b/ui/src/pages/Schedules.tsx@@ -79,6 +79,9 @@ interface EditForm {delaySeconds: string;prompt: string;skillName: string;+ timezone: string;+ missedRunPolicy: 'skip' | 'run_once';+ resume: boolean;}export function SchedulesPage() {@@ -96,6 +99,9 @@ export function SchedulesPage() {delaySeconds: "",prompt: "",skillName: "",+ timezone: "",+ missedRunPolicy: 'skip',+ resume: false,});const [saving, setSaving] = useState(false);@@ -134,6 +140,9 @@ export function SchedulesPage() {delaySeconds: schedule.delaySeconds?.toString() ?? "",prompt: schedule.prompt ?? "",skillName: schedule.skillName ?? "",+ timezone: schedule.timezone ?? "",+ missedRunPolicy: schedule.missedRunPolicy ?? 'skip',+ resume: false,});setEditOpen(true);}@@ -142,11 +151,16 @@ export function SchedulesPage() {if (!editId) return;setSaving(true);try {- const payload: Partial<Schedule> = {+ const payload: Partial<Schedule> & { resume?: boolean } = {description: editForm.description,+ timezone: editForm.timezone.trim(),+ missedRunPolicy: editForm.missedRunPolicy,+ resume: editForm.resume,};if (editForm.cron.trim()) payload.cron = editForm.cron.trim();- if (editForm.delaySeconds.trim())+ const original = schedules.find((s) => s.id === editId);+ if (!editForm.cron.trim() && editForm.delaySeconds.trim()+ && (editForm.resume || original?.cron || Number(editForm.delaySeconds) !== original?.delaySeconds))payload.delaySeconds = Number(editForm.delaySeconds);if (editForm.prompt.trim()) payload.prompt = editForm.prompt.trim();if (editForm.skillName.trim())@@ -286,7 +300,7 @@ export function SchedulesPage() {{schedule.cron && (<Badge variant="secondary" className="gap-1"><Calendar className="h-3 w-3" />- {humanizeCron(schedule.cron)}+ {humanizeCron(schedule.cron)} · {schedule.timezone ?? 'host timezone'}</Badge>)}{schedule.delaySeconds != null && (@@ -295,6 +309,9 @@ export function SchedulesPage() {{humanizeDelay(schedule.delaySeconds)}</Badge>)}+ <Badge variant={schedule.paused ? 'destructive' : 'outline'}>+ {schedule.lastRun?.status ?? 'pending'}{schedule.paused ? ' · paused' : ''}+ </Badge>{schedule.skillName && (<Badge variant="default" className="gap-1"><Sparkles className="h-3 w-3" />@@ -308,6 +325,8 @@ export function SchedulesPage() {variant="ghost"size="icon"onClick={() => openEdit(schedule)}+ disabled={schedule.lastRun?.status === 'running'}+ aria-label="Edit schedule"><Pencil className="h-4 w-4" /></Button>@@ -322,6 +341,9 @@ export function SchedulesPage() {</CardHeader><CardContent><div className="flex flex-col gap-2 text-sm">+ {schedule.lastRun?.error && <p className="text-destructive">{schedule.lastRun.error}</p>}+ {schedule.lastRun?.status === 'dispatched' && <p className="text-muted-foreground">Accepted by the bot queue; see the bot's runs for its outcome.</p>}+ {!schedule.cron && <p className="text-muted-foreground">Due: {schedule.executeAt ? formatDate(schedule.executeAt) : 'unknown'} · If offline: {schedule.missedRunPolicy === 'run_once' ? 'run once on restart' : 'skip'}</p>}{schedule.prompt && (<p className="text-muted-foreground"><span className="font-medium text-foreground">Prompt: </span>@@ -329,11 +351,11 @@ export function SchedulesPage() {</p>)}<div className="flex flex-wrap gap-x-6 gap-y-1 text-xs text-muted-foreground">- {schedule.lastRun && (+ {schedule.lastRun?.startedAt && (<span>Last run:{" "}<span className="font-medium tabular-nums text-foreground">- {formatDate(schedule.lastRun)}+ {formatDate(schedule.lastRun.startedAt)}</span></span>)}@@ -360,7 +382,7 @@ export function SchedulesPage() {<DialogHeader><DialogTitle>Edit Schedule</DialogTitle><DialogDescription>- Update the schedule configuration.+ Update the schedule configuration. Failed or interrupted work stays paused until you choose to schedule it again.</DialogDescription></DialogHeader><div className="flex flex-col gap-4 py-2">@@ -397,6 +419,26 @@ export function SchedulesPage() {/></div></div>+ {editForm.cron.trim() ? (+ <div className="space-y-1.5">+ <label htmlFor="schedule-timezone" className="text-sm font-medium">Timezone</label>+ <Input id="schedule-timezone" value={editForm.timezone} placeholder="Host timezone (e.g. America/New_York)" onChange={(e) => setEditForm((f) => ({ ...f, timezone: e.target.value }))} />+ </div>+ ) : (+ <div className="space-y-1.5">+ <label htmlFor="schedule-missed" className="text-sm font-medium">If Mercury is offline when due</label>+ <select id="schedule-missed" className="w-full rounded border bg-background p-2" value={editForm.missedRunPolicy} onChange={(e) => setEditForm((f) => ({ ...f, missedRunPolicy: e.target.value as 'skip' | 'run_once' }))}>+ <option value="skip">Skip the missed run</option>+ <option value="run_once">Run once after restart</option>+ </select>+ </div>+ )}+ {schedules.find((s) => s.id === editId)?.lastRun && (+ <label className="flex gap-2 text-sm">+ <input type="checkbox" checked={editForm.resume} onChange={(e) => setEditForm((f) => ({ ...f, resume: e.target.checked }))} />+ Schedule again. I checked the prior run's effects; repeating the work is intended.+ </label>+ )}<div className="space-y-1.5"><label className="text-sm font-medium">Prompt</label><Inputdiff --git a/website/docs/reference/scheduling.mdx b/website/docs/reference/scheduling.mdxindex 2acd031d..87a29d8e 100644--- a/website/docs/reference/scheduling.mdx+++ b/website/docs/reference/scheduling.mdx@@ -17,4 +17,53 @@ Tell Mercury in conversation:- "Remind me in 15 minutes to take a break"- "Ping me in 2 hours about the deployment"-Tasks persist to `~/.mercury/schedules.yaml` and restore on restart. Responses route back to the channel (CLI or Telegram) where the task was created.\ No newline at end of file+Tasks persist to `~/.mercury/schedules.yaml` and restore on restart. Responses route back to the channel (CLI or Telegram) where the task was created.+## Time zones and overlap++Use a timezone for recurring work, for example: “Every weekday at 9am in+America/New_York, brief me on my calendar.” The `schedule_task` tool accepts an+IANA `timezone`; the Schedules editor exposes the same setting. Existing schedules+without one continue using the host timezone. Cron follows that zone's local clock,+including daylight-saving changes.++A schedule never queues another main-agent run while its previous run is still+queued or running. Overlapping ticks are skipped, rather than accumulated for later.+Recurring ticks missed while Mercury was offline are not replayed.++## Offline and interrupted work++One-shot tasks have an explicit `missed_run_policy`:++- `skip` (default, including older schedules): keep a skipped record when Mercury+ starts after the due time.+- `run_once`: start the overdue task once after restart, provided no run had begun.++This policy does **not** retry failed or interrupted work. Mercury records a run+before invoking it. If the process stops before the outcome is confirmed, the run+becomes **interrupted** and the schedule pauses. Failed runs also pause, because an+external action may have succeeded before its response was lost.++Check the prior run's effects before repeating it. In **Schedules → Edit**, choose+**Schedule again** to explicitly resume. For a one-shot, the delay starts again+from when you save. Editing only a description or policy does not repeat the work.+The API equivalent is `PUT /api/schedules/:id` with `{"resume": true}` and, for a+one-shot, the desired `delaySeconds`. Running tasks cannot be edited.++## Run outcomes++The Schedules page and `list_scheduled_tasks` show pending, running, completed,+failed, interrupted, skipped, or dispatched status. **Completed** means the main+agent finished its turn; inspect its response for the actual task result. Bot+schedules show **dispatched** after their separate queue accepts them; inspect the+bot's run history for completion. Scheduled bot jobs whose run or retry was+interrupted go to the dead-letter queue (DLQ) instead of replaying automatically;+future routines pause. Check their effects, then use `/bots start <id>` to resume+future routines or `/bots replay <id> <jobId>` to explicitly repeat that job.+A fresh queued bot job that never started can still resume. Manual/chat bot jobs+keep their existing recovery behavior.++Run records are stored with the schedule in `~/.mercury/schedules.yaml`, using an+atomic replacement. The latest run is kept for each recurring task. The most recent+100 resolved one-shot records are retained; active work and unresolved failures are+kept until you remove them. These safeguards prevent automatic scheduler replay;+they cannot guarantee exactly-once behavior from external services.