Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
fix(bots): close delivery-contract gaps — durable retries + durable mailboxes
Salman Qureshiadb59b3ace42
diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.tsindex 0a1080a2..3ecdcaa6 100644--- a/src/bots/bot-manager.ts+++ b/src/bots/bot-manager.ts@@ -157,6 +157,34 @@ export class BotManager {this.queue.settle(job.id, 'dead', 'bot_removed');}}+ // Durable mailboxes survive restarts: rehydrate into the in-memory boxes.+ for (const m of this.store.list()) {+ const drained = this.queue.drainMail(m.id);+ if (drained.length > 0) this.mailboxes.set(m.id, drained);+ }+ // Periodic due-sweep: retry-backoff jobs re-enter the in-memory queues+ // when their run_after elapses (also covers crash-restart backoffs).+ const dueTimer = setInterval(() => this.resumeDueJobs(), 30_000);+ dueTimer.unref?.();+ }++ /** Pull due (backoff-elapsed) jobs from the durable queue into memory and run them. */+ private resumeDueJobs(): void {+ const due = this.queue.dueJobs();+ for (const job of due) {+ if (this.disabled.has(job.botId)) continue;+ const running = this.running.get(job.botId)?.size ?? 0;+ const q = this.queues.get(job.botId) ?? [];+ const alreadyQueued = q.some(j => j.id === job.id);+ if (!alreadyQueued && running === 0) {+ q.push({+ id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,+ fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,+ });+ this.queues.set(job.botId, q);+ this.pump(job.botId);+ }+ }}/** Fleet-wide concurrency cap: config override or clamp(2, cpus-1). */@@ -223,6 +251,9 @@ export class BotManager {if (box.length >= MAILBOX_CAPACITY) {return { accepted: false, reasonCode: 'queue_full' };}+ // Durable-before-ack for handoffs too: a bot_send that returns "queued"+ // must survive a crash before the target's next turn drains it.+ this.queue.enqueueMail({ botId: targetBotId, from: fromBot, content, createdAt: Date.now() });box.push({ from: fromBot, content });this.mailboxes.set(targetBotId, box);@@ -240,10 +271,11 @@ export class BotManager {return { accepted: true, jobId };}- /** Poll-and-drain a bot's mailbox (used by turns and by inbox inspection). */+ /** Poll-and-drain a bot's mailbox (turns); durable rows removed too. */drainMailbox(botId: string): BotTurnMail[] {const box = this.mailboxes.get(botId) ?? [];this.mailboxes.set(botId, []);+ this.queue.drainMail(botId);return box;}@@ -355,12 +387,19 @@ export class BotManager {// Transient provider failures retry with backoff, bounded; permanent// failures go to the capped DLQ and stop (never silently re-queued — §2.6).+ // Retries requeue the SAME job in place (durable): no settle-then-+ // reenqueue window where a crash would lose the work.if (output.status === 'failed' && output.reasonCode && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS) {- this.queue.settle(job.id, 'done'); // the retry re-enqueues a fresh attemptconst delay = Math.min(15000, 1000 * 2 ** job.attempts);- logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying');+ this.queue.retry(job.id, job.attempts + 1, Date.now() + delay);+ logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying in place');setTimeout(() => {- this.enqueue(botId, { trigger: job.trigger, prompt: job.prompt, fromBot: job.fromBot, source: job.source, attempts: job.attempts + 1 });+ const q = this.queues.get(botId) ?? [];+ if (!q.some(j => j.id === job.id)) {+ q.push({ ...job, attempts: job.attempts + 1 });+ this.queues.set(botId, q);+ this.pump(botId);+ }}, delay).unref?.();} else if (output.status === 'failed') {this.queue.settle(job.id, 'dead', output.reasonCode);@@ -368,28 +407,39 @@ export class BotManager {logger.warn({ botId, jobId: job.id, reasonCode: output.reasonCode }, 'Bot turn failed permanently — moved to DLQ (replayable via /bots dlq)');await this.alertOwner(botId, `❌ **${manifest.name}** failed permanently [reason: ${output.reasonCode}] — replay with \`/bots replay ${botId} ${job.id}\``);} else if (output.status === 'paused') {- // Step-budget pause: work continues next turn — keep the job pending.- this.queue.settle(job.id, 'done');- this.enqueue(botId, { trigger: job.trigger, prompt: job.prompt, fromBot: job.fromBot, source: job.source, attempts: job.attempts });+ // Step-budget pause: work continues next turn — same job requeues in+ // place (durable), no attempts bump.+ this.queue.retry(job.id, job.attempts, Date.now() + 2000);+ setTimeout(() => {+ const q = this.queues.get(botId) ?? [];+ if (!q.some(j => j.id === job.id)) {+ q.push({ ...job });+ this.queues.set(botId, q);+ this.pump(botId);+ }+ }, 2000).unref?.();} else {this.queue.settle(job.id, 'done');}- // Deliver the outcome (guaranteed delivery target: the bot's own chat- // minimum — never silently dropped, BOTS-ARCHITECTURE §3.1.5). Jobs- // without a source (cron/API) deliver to the bot's own thread, which- // the CLI channel routes via the `bot:<id>` targetId.- if (this.notify && output.status !== 'halted' && job.trigger !== 'mailbox') {+ // Deliver the outcome — including halts (a stopped run must report+ // that it stopped, never vanish silently, §2.6). Jobs without a source+ // (cron/API) deliver to the bot's own thread via the `bot:<id>` targetId.+ if (this.notify && job.trigger !== 'mailbox') {const channelType = job.source?.channelType ?? 'cli';const channelId = job.source?.channelId ?? `bot:${botId}`;- const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : '⏸';- const text = `${icon} **${manifest.name}** (${job.trigger}): ${output.output.slice(0, 800)}`;+ const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : output.status === 'halted' ? '⏹' : '⏸';+ const text = output.status === 'halted'+ ? `⏹ **${manifest.name}** run ${job.id} was stopped by you — no further output. It is recorded in \`/bots journal ${botId}\`.`+ : `${icon} **${manifest.name}** (${job.trigger}): ${output.output.slice(0, 800)}`;await this.notify(channelType, channelId, text).catch((e) =>logger.warn({ e, botId }, 'Bot completion notify failed'));}} catch (err: any) {logger.error({ botId, jobId: job.id, err: err?.message }, 'Bot turn crashed');this.queue.settle(job.id, 'dead', 'unknown_error');+ this.needsYou.add(botId);+ void this.alertOwner(botId, `💥 **${botId}** run ${job.id} crashed: ${String(err?.message ?? err).slice(0, 150)} — see journal; replay from DLQ.`);this.journalFor(botId).append({runId: job.id,botId,diff --git a/src/bots/queue.test.ts b/src/bots/queue.test.tsindex 46896d6c..3551df17 100644--- a/src/bots/queue.test.ts+++ b/src/bots/queue.test.ts@@ -72,11 +72,13 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {expect(backend.rehydratable()).toHaveLength(1);});- it('claim sets a lease; settle done removes the job', () => {+ it('claim sets a lease (in-flight, not resumable); settle done removes the job', () => {const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));backend.enqueue(job('a1'));backend.claim('a1', LEASE_SECONDS);- expect(backend.rehydratable()[0].state).toBe('claimed');+ // Fresh lease = in-flight: not resumable, but visible in counts.+ expect(backend.rehydratable()).toHaveLength(0);+ expect(backend.counts().claimed).toBe(1);backend.settle('a1', 'done');expect(backend.rehydratable()).toHaveLength(0);expect(backend.counts()).toEqual({ pending: 0, claimed: 0, dlq: 0 });@@ -93,7 +95,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {expect(dlq[0].attempts).toBe(1);});- it('expired leases requeue to pending; fresh leases survive', () => {+ it('expired leases requeue to pending; fresh leases survive in-flight', () => {const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));backend.enqueue(job('expired'));backend.enqueue(job('alive'));@@ -101,9 +103,10 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {backend.claim('alive', 3600);const requeued = backend.requeueExpiredLeases();expect(requeued).toBe(1);- const jobs = backend.rehydratable();- expect(jobs.find(j => j.id === 'expired')?.state).toBe('pending');- expect(jobs.find(j => j.id === 'alive')?.state).toBe('claimed');+ expect(backend.rehydratable().find(j => j.id === 'expired')?.state).toBe('pending');+ // 'alive' keeps its fresh lease: in-flight, not resumable+ expect(backend.rehydratable().find(j => j.id === 'alive')).toBeUndefined();+ expect(backend.counts().claimed).toBe(1);});it('DLQ is capped with oldest-first eviction', () => {@@ -149,6 +152,40 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {expect(resumed.map(j => j.id).sort()).toEqual(['r1', 'r2']);expect(resumed.every(j => j.state === 'pending')).toBe(true);});++ it('retry requeues the same job in place — no settle-then-reenqueue window', () => {+ for (const name of ['json', 'sqlite'] as const) {+ const dir = join(root, `retry-${name}`);+ const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ backend.enqueue(job('t1'));+ backend.claim('t1', LEASE_SECONDS);+ // Crash-safe retry: attempts bump + backoff, still durably pending.+ backend.retry('t1', 1, Date.now() + 5000);+ expect(backend.rehydratable()).toHaveLength(0); // backoff not elapsed+ expect(backend.dueJobs()).toHaveLength(0);+ // A restart (fresh instance over the same storage) still sees the job+ const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ expect(fresh.rehydratable().length).toBeGreaterThanOrEqual(0);+ expect(fresh.counts().pending).toBe(1);+ }+ });++ it('durable mail survives a restart and drains once', () => {+ for (const name of ['json', 'sqlite'] as const) {+ const dir = join(root, `mail-${name}`);+ const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ backend.enqueueMail({ botId: 'publisher', from: 'researcher', content: 'findings here', createdAt: Date.now() });+ // Restart: the mail is still there+ const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ const drained = fresh.drainMail('publisher');+ expect(drained).toHaveLength(1);+ expect(drained[0].from).toBe('researcher');+ expect(drained[0].content).toBe('findings here');+ // Draining is destructive — no double delivery+ expect(fresh.drainMail('publisher')).toHaveLength(0);+ expect(fresh.drainMail('other-bot')).toHaveLength(0);+ }+ });});describe('BotManager durable queue integration', () => {diff --git a/src/bots/queue.ts b/src/bots/queue.tsindex 6d76971b..266c207f 100644--- a/src/bots/queue.ts+++ b/src/bots/queue.ts@@ -1,4 +1,4 @@-import { createHash } from 'node:crypto';+import { createHash, randomUUID } from 'node:crypto';import { existsSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs';import { createRequire } from 'node:module';import { join, resolve } from 'node:path';@@ -47,6 +47,16 @@ export interface DurableBotJob {idempotencyKey: string;reasonCode?: string;settledAt?: number;+ /** Earliest time the job may run again (transient-retry backoff). */+ runAfter?: number;+}++export interface DurableMail {+ id: string;+ botId: string;+ from: string;+ content: string;+ createdAt: number;}export interface DlqEntry extends DurableBotJob {@@ -66,13 +76,24 @@ export interface BotQueueBackend {claim(jobId: string, leaseSeconds: number): void;/** done → removed; dead → moved to the capped DLQ. */settle(jobId: string, outcome: 'done' | 'dead', reasonCode?: string): void;- /** Jobs safe to resume: pending, or claimed whose lease has expired. */+ /**+ * Durable in-place retry for a claimed job: attempts+1, back to pending,+ * with an optional run-after timestamp. No settle-then-reenqueue window —+ * a crash at any point still leaves the job in the queue.+ */+ retry(jobId: string, attempts: number, runAfterMs?: number): void;+ /** Jobs safe to resume: pending (due), or claimed whose lease has expired. */rehydratable(): DurableBotJob[];+ /** Pending jobs whose run-after time has arrived (retry backoff elapsed). */+ dueJobs(): DurableBotJob[];/** Expired-lease claimed jobs → pending. Returns the count requeued. */requeueExpiredLeases(): number;heartbeatLease(jobId: string, leaseSeconds: number): void;listDlq(botId?: string): DlqEntry[];removeFromDlq(jobId: string): DurableBotJob | null;+ /** Durable bot-to-bot mailbox (§2.6: a handoff must survive a crash). */+ enqueueMail(mail: Omit<DurableMail, 'id'>): string;+ drainMail(botId: string): DurableMail[];counts(): QueueCounts;}@@ -138,17 +159,31 @@ export class BotQueue {this.backend.settle(jobId, outcome, reasonCode);}- heartbeatLease(jobId: string, leaseSeconds: number = LEASE_SECONDS): void {- this.backend.heartbeatLease(jobId, leaseSeconds);+ /** Durable in-place retry: no settle-then-reenqueue window. */+ retry(jobId: string, attempts: number, runAfterMs?: number): void {+ this.backend.retry(jobId, attempts, runAfterMs);}- /** Jobs to resume on startup: pending, plus expired-lease claimed jobs. */+ /** Jobs to resume on startup: due pending, plus expired-lease claimed jobs. */resumeJobs(): DurableBotJob[] {const expired = this.backend.requeueExpiredLeases();if (expired > 0) logger.info({ expired }, 'Requeued expired bot job leases');return this.backend.rehydratable();}+ /** Pending jobs whose retry backoff has elapsed (swept periodically). */+ dueJobs(): DurableBotJob[] {+ return this.backend.dueJobs();+ }++ enqueueMail(mail: Omit<DurableMail, 'id'>): string {+ return this.backend.enqueueMail(mail);+ }++ drainMail(botId: string): DurableMail[] {+ return this.backend.drainMail(botId);+ }+listDlq(botId?: string): DlqEntry[] {return this.backend.listDlq(botId);}@@ -194,6 +229,14 @@ export class SqliteQueueBackend implements BotQueueBackend {settled_at INTEGER);CREATE INDEX IF NOT EXISTS idx_bot_jobs_state ON bot_jobs (state, created_at);+ CREATE TABLE IF NOT EXISTS bot_mail (+ id TEXT PRIMARY KEY,+ bot_id TEXT NOT NULL,+ from_bot TEXT NOT NULL,+ content TEXT NOT NULL,+ created_at INTEGER NOT NULL+ );+ CREATE INDEX IF NOT EXISTS idx_bot_mail_bot ON bot_mail (bot_id, created_at);CREATE TABLE IF NOT EXISTS bot_dlq (id TEXT PRIMARY KEY,bot_id TEXT NOT NULL,@@ -208,6 +251,8 @@ export class SqliteQueueBackend implements BotQueueBackend {dead_at INTEGER NOT NULL);`);+ // Migration for queues created before run_after / bot_mail existed.+ try { this.db.exec('ALTER TABLE bot_jobs ADD COLUMN run_after INTEGER'); } catch { /* already present */ }this.dlqCap = dlqCap;}@@ -271,11 +316,49 @@ export class SqliteQueueBackend implements BotQueueBackend {return result.changes as number;}+ retry(jobId: string, attempts: number, runAfterMs?: number): void {+ this.db.prepare(+ `UPDATE bot_jobs SET state = 'pending', attempts = ?, lease_expires_at = NULL, run_after = ? WHERE id = ?`,+ ).run(attempts, runAfterMs ?? null, jobId);+ }+rehydratable(): DurableBotJob[] {- const rows = this.db.prepare(`SELECT * FROM bot_jobs WHERE state IN ('pending','claimed') ORDER BY created_at`).all() as any[];+ // Due pending jobs + expired-lease claimed jobs — jobs still in their+ // retry backoff (run_after in the future) wait until due.+ const rows = this.db.prepare(+ `SELECT * FROM bot_jobs WHERE state = 'pending' AND (run_after IS NULL OR run_after <= ?)+ UNION ALL SELECT * FROM bot_jobs WHERE state = 'claimed' AND lease_expires_at <= ? ORDER BY created_at`,+ ).all(Date.now(), Date.now()) as any[];return rows.map(normalizeJob);}+ dueJobs(): DurableBotJob[] {+ const rows = this.db.prepare(+ `SELECT * FROM bot_jobs WHERE state = 'pending' AND (run_after IS NULL OR run_after <= ?) ORDER BY created_at`,+ ).all(Date.now()) as any[];+ return rows.map(normalizeJob);+ }++ enqueueMail(mail: Omit<DurableMail, 'id'>): string {+ const id = randomUUID().slice(0, 12);+ this.db.prepare(+ `INSERT INTO bot_mail (id, bot_id, from_bot, content, created_at) VALUES (?, ?, ?, ?, ?)`,+ ).run(id, mail.botId, mail.from, mail.content, mail.createdAt);+ return id;+ }++ drainMail(botId: string): DurableMail[] {+ const rows = this.db.prepare(+ `SELECT * FROM bot_mail WHERE bot_id = ? ORDER BY created_at`,+ ).all(botId) as any[];+ if (rows.length === 0) return [];+ this.db.prepare(`DELETE FROM bot_mail WHERE bot_id = ?`).run(botId);+ return rows.map(r => ({+ id: String(r.id), botId: String(r.bot_id), from: String(r.from_bot),+ content: String(r.content), createdAt: Number(r.created_at),+ }));+ }+listDlq(botId?: string): DlqEntry[] {const rows = botId? this.db.prepare(`SELECT * FROM bot_dlq WHERE bot_id = ? ORDER BY dead_at DESC`).all(botId) as any[]@@ -324,6 +407,7 @@ const QUEUE_FILE = 'queue.json';interface QueueFile {jobs: DurableBotJob[];dlq: DlqEntry[];+ mails: DurableMail[];}export class JsonFileQueueBackend implements BotQueueBackend {@@ -336,7 +420,7 @@ export class JsonFileQueueBackend implements BotQueueBackend {mkdirSync(dir, { recursive: true });this.file = join(dir, QUEUE_FILE);this.dlqCap = dlqCap;- this.data = existsSync(this.file) ? this.read() : { jobs: [], dlq: [] };+ this.data = existsSync(this.file) ? this.read() : { jobs: [], dlq: [], mails: [] };}private read(): QueueFile {@@ -345,9 +429,10 @@ export class JsonFileQueueBackend implements BotQueueBackend {return {jobs: (parsed.jobs ?? []).map(normalizeJob),dlq: (parsed.dlq ?? []).map((d: any) => ({ ...normalizeJob(d), state: 'dead' as const, deadAt: d.deadAt ?? Date.now() })),+ mails: parsed.mails ?? [],};} catch {- return { jobs: [], dlq: [] };+ return { jobs: [], dlq: [], mails: [] };}}@@ -412,8 +497,44 @@ export class JsonFileQueueBackend implements BotQueueBackend {return n;}+ retry(jobId: string, attempts: number, runAfterMs?: number): void {+ const job = this.data.jobs.find(j => j.id === jobId);+ if (job) {+ job.state = 'pending';+ job.attempts = attempts;+ job.leaseExpiresAt = undefined;+ job.runAfter = runAfterMs;+ this.flush();+ }+ }+rehydratable(): DurableBotJob[] {- return [...this.data.jobs];+ // Due pending + expired-lease claimed — future run_after waits.+ const now = Date.now();+ return this.data.jobs.filter(j =>+ (j.state === 'pending' && (j.runAfter === undefined || j.runAfter <= now))+ || (j.state === 'claimed' && (j.leaseExpiresAt ?? 0) < now));+ }++ dueJobs(): DurableBotJob[] {+ const now = Date.now();+ return this.data.jobs.filter(j => j.state === 'pending' && (j.runAfter === undefined || j.runAfter <= now));+ }++ enqueueMail(mail: Omit<DurableMail, 'id'>): string {+ const id = randomUUID().slice(0, 12);+ this.data.mails.push({ ...mail, id });+ this.flush();+ return id;+ }++ drainMail(botId: string): DurableMail[] {+ const drained = this.data.mails.filter(m => m.botId === botId);+ if (drained.length > 0) {+ this.data.mails = this.data.mails.filter(m => m.botId !== botId);+ this.flush();+ }+ return drained;}listDlq(botId?: string): DlqEntry[] {