Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

fix(bots): close delivery-contract gaps — durable retries + durable mailboxes

Salman Qureshiadb59b3ace42
diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.ts
index 0a1080a2..3ecdcaa6 100644
--- a/src/bots/bot-manager.ts
+++ b/src/bots/bot-manager.ts
@@ -157,6 +157,34 @@ export class BotManager {
this.queue.settle(job.id, 'dead', 'bot_removed');
}
}
+ // Durable mailboxes survive restarts: rehydrate into the in-memory boxes.
+ for (const m of this.store.list()) {
+ const drained = this.queue.drainMail(m.id);
+ if (drained.length > 0) this.mailboxes.set(m.id, drained);
+ }
+ // Periodic due-sweep: retry-backoff jobs re-enter the in-memory queues
+ // when their run_after elapses (also covers crash-restart backoffs).
+ const dueTimer = setInterval(() => this.resumeDueJobs(), 30_000);
+ dueTimer.unref?.();
+ }
+
+ /** Pull due (backoff-elapsed) jobs from the durable queue into memory and run them. */
+ private resumeDueJobs(): void {
+ const due = this.queue.dueJobs();
+ for (const job of due) {
+ if (this.disabled.has(job.botId)) continue;
+ const running = this.running.get(job.botId)?.size ?? 0;
+ const q = this.queues.get(job.botId) ?? [];
+ const alreadyQueued = q.some(j => j.id === job.id);
+ if (!alreadyQueued && running === 0) {
+ q.push({
+ id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,
+ fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,
+ });
+ this.queues.set(job.botId, q);
+ this.pump(job.botId);
+ }
+ }
}
/** Fleet-wide concurrency cap: config override or clamp(2, cpus-1). */
@@ -223,6 +251,9 @@ export class BotManager {
if (box.length >= MAILBOX_CAPACITY) {
return { accepted: false, reasonCode: 'queue_full' };
}
+ // Durable-before-ack for handoffs too: a bot_send that returns "queued"
+ // must survive a crash before the target's next turn drains it.
+ this.queue.enqueueMail({ botId: targetBotId, from: fromBot, content, createdAt: Date.now() });
box.push({ from: fromBot, content });
this.mailboxes.set(targetBotId, box);
@@ -240,10 +271,11 @@ export class BotManager {
return { accepted: true, jobId };
}
- /** Poll-and-drain a bot's mailbox (used by turns and by inbox inspection). */
+ /** Poll-and-drain a bot's mailbox (turns); durable rows removed too. */
drainMailbox(botId: string): BotTurnMail[] {
const box = this.mailboxes.get(botId) ?? [];
this.mailboxes.set(botId, []);
+ this.queue.drainMail(botId);
return box;
}
@@ -355,12 +387,19 @@ export class BotManager {
// Transient provider failures retry with backoff, bounded; permanent
// failures go to the capped DLQ and stop (never silently re-queued — §2.6).
+ // Retries requeue the SAME job in place (durable): no settle-then-
+ // reenqueue window where a crash would lose the work.
if (output.status === 'failed' && output.reasonCode && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS) {
- this.queue.settle(job.id, 'done'); // the retry re-enqueues a fresh attempt
const delay = Math.min(15000, 1000 * 2 ** job.attempts);
- logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying');
+ this.queue.retry(job.id, job.attempts + 1, Date.now() + delay);
+ logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying in place');
setTimeout(() => {
- this.enqueue(botId, { trigger: job.trigger, prompt: job.prompt, fromBot: job.fromBot, source: job.source, attempts: job.attempts + 1 });
+ const q = this.queues.get(botId) ?? [];
+ if (!q.some(j => j.id === job.id)) {
+ q.push({ ...job, attempts: job.attempts + 1 });
+ this.queues.set(botId, q);
+ this.pump(botId);
+ }
}, delay).unref?.();
} else if (output.status === 'failed') {
this.queue.settle(job.id, 'dead', output.reasonCode);
@@ -368,28 +407,39 @@ export class BotManager {
logger.warn({ botId, jobId: job.id, reasonCode: output.reasonCode }, 'Bot turn failed permanently — moved to DLQ (replayable via /bots dlq)');
await this.alertOwner(botId, `❌ **${manifest.name}** failed permanently [reason: ${output.reasonCode}] — replay with \`/bots replay ${botId} ${job.id}\``);
} else if (output.status === 'paused') {
- // Step-budget pause: work continues next turn — keep the job pending.
- this.queue.settle(job.id, 'done');
- this.enqueue(botId, { trigger: job.trigger, prompt: job.prompt, fromBot: job.fromBot, source: job.source, attempts: job.attempts });
+ // Step-budget pause: work continues next turn — same job requeues in
+ // place (durable), no attempts bump.
+ this.queue.retry(job.id, job.attempts, Date.now() + 2000);
+ setTimeout(() => {
+ const q = this.queues.get(botId) ?? [];
+ if (!q.some(j => j.id === job.id)) {
+ q.push({ ...job });
+ this.queues.set(botId, q);
+ this.pump(botId);
+ }
+ }, 2000).unref?.();
} else {
this.queue.settle(job.id, 'done');
}
- // Deliver the outcome (guaranteed delivery target: the bot's own chat
- // minimum — never silently dropped, BOTS-ARCHITECTURE §3.1.5). Jobs
- // without a source (cron/API) deliver to the bot's own thread, which
- // the CLI channel routes via the `bot:<id>` targetId.
- if (this.notify && output.status !== 'halted' && job.trigger !== 'mailbox') {
+ // Deliver the outcome — including halts (a stopped run must report
+ // that it stopped, never vanish silently, §2.6). Jobs without a source
+ // (cron/API) deliver to the bot's own thread via the `bot:<id>` targetId.
+ if (this.notify && job.trigger !== 'mailbox') {
const channelType = job.source?.channelType ?? 'cli';
const channelId = job.source?.channelId ?? `bot:${botId}`;
- const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : '⏸';
- const text = `${icon} **${manifest.name}** (${job.trigger}): ${output.output.slice(0, 800)}`;
+ const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : output.status === 'halted' ? '⏹' : '⏸';
+ const text = output.status === 'halted'
+ ? `⏹ **${manifest.name}** run ${job.id} was stopped by you — no further output. It is recorded in \`/bots journal ${botId}\`.`
+ : `${icon} **${manifest.name}** (${job.trigger}): ${output.output.slice(0, 800)}`;
await this.notify(channelType, channelId, text).catch((e) =>
logger.warn({ e, botId }, 'Bot completion notify failed'));
}
} catch (err: any) {
logger.error({ botId, jobId: job.id, err: err?.message }, 'Bot turn crashed');
this.queue.settle(job.id, 'dead', 'unknown_error');
+ this.needsYou.add(botId);
+ void this.alertOwner(botId, `💥 **${botId}** run ${job.id} crashed: ${String(err?.message ?? err).slice(0, 150)} — see journal; replay from DLQ.`);
this.journalFor(botId).append({
runId: job.id,
botId,
diff --git a/src/bots/queue.test.ts b/src/bots/queue.test.ts
index 46896d6c..3551df17 100644
--- a/src/bots/queue.test.ts
+++ b/src/bots/queue.test.ts
@@ -72,11 +72,13 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
expect(backend.rehydratable()).toHaveLength(1);
});
- it('claim sets a lease; settle done removes the job', () => {
+ it('claim sets a lease (in-flight, not resumable); settle done removes the job', () => {
const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));
backend.enqueue(job('a1'));
backend.claim('a1', LEASE_SECONDS);
- expect(backend.rehydratable()[0].state).toBe('claimed');
+ // Fresh lease = in-flight: not resumable, but visible in counts.
+ expect(backend.rehydratable()).toHaveLength(0);
+ expect(backend.counts().claimed).toBe(1);
backend.settle('a1', 'done');
expect(backend.rehydratable()).toHaveLength(0);
expect(backend.counts()).toEqual({ pending: 0, claimed: 0, dlq: 0 });
@@ -93,7 +95,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
expect(dlq[0].attempts).toBe(1);
});
- it('expired leases requeue to pending; fresh leases survive', () => {
+ it('expired leases requeue to pending; fresh leases survive in-flight', () => {
const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));
backend.enqueue(job('expired'));
backend.enqueue(job('alive'));
@@ -101,9 +103,10 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
backend.claim('alive', 3600);
const requeued = backend.requeueExpiredLeases();
expect(requeued).toBe(1);
- const jobs = backend.rehydratable();
- expect(jobs.find(j => j.id === 'expired')?.state).toBe('pending');
- expect(jobs.find(j => j.id === 'alive')?.state).toBe('claimed');
+ expect(backend.rehydratable().find(j => j.id === 'expired')?.state).toBe('pending');
+ // 'alive' keeps its fresh lease: in-flight, not resumable
+ expect(backend.rehydratable().find(j => j.id === 'alive')).toBeUndefined();
+ expect(backend.counts().claimed).toBe(1);
});
it('DLQ is capped with oldest-first eviction', () => {
@@ -149,6 +152,40 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
expect(resumed.map(j => j.id).sort()).toEqual(['r1', 'r2']);
expect(resumed.every(j => j.state === 'pending')).toBe(true);
});
+
+ it('retry requeues the same job in place — no settle-then-reenqueue window', () => {
+ for (const name of ['json', 'sqlite'] as const) {
+ const dir = join(root, `retry-${name}`);
+ const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));
+ backend.enqueue(job('t1'));
+ backend.claim('t1', LEASE_SECONDS);
+ // Crash-safe retry: attempts bump + backoff, still durably pending.
+ backend.retry('t1', 1, Date.now() + 5000);
+ expect(backend.rehydratable()).toHaveLength(0); // backoff not elapsed
+ expect(backend.dueJobs()).toHaveLength(0);
+ // A restart (fresh instance over the same storage) still sees the job
+ const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));
+ expect(fresh.rehydratable().length).toBeGreaterThanOrEqual(0);
+ expect(fresh.counts().pending).toBe(1);
+ }
+ });
+
+ it('durable mail survives a restart and drains once', () => {
+ for (const name of ['json', 'sqlite'] as const) {
+ const dir = join(root, `mail-${name}`);
+ const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));
+ backend.enqueueMail({ botId: 'publisher', from: 'researcher', content: 'findings here', createdAt: Date.now() });
+ // Restart: the mail is still there
+ const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));
+ const drained = fresh.drainMail('publisher');
+ expect(drained).toHaveLength(1);
+ expect(drained[0].from).toBe('researcher');
+ expect(drained[0].content).toBe('findings here');
+ // Draining is destructive — no double delivery
+ expect(fresh.drainMail('publisher')).toHaveLength(0);
+ expect(fresh.drainMail('other-bot')).toHaveLength(0);
+ }
+ });
});
describe('BotManager durable queue integration', () => {
diff --git a/src/bots/queue.ts b/src/bots/queue.ts
index 6d76971b..266c207f 100644
--- a/src/bots/queue.ts
+++ b/src/bots/queue.ts
@@ -1,4 +1,4 @@
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, resolve } from 'node:path';
@@ -47,6 +47,16 @@ export interface DurableBotJob {
idempotencyKey: string;
reasonCode?: string;
settledAt?: number;
+ /** Earliest time the job may run again (transient-retry backoff). */
+ runAfter?: number;
+}
+
+export interface DurableMail {
+ id: string;
+ botId: string;
+ from: string;
+ content: string;
+ createdAt: number;
}
export interface DlqEntry extends DurableBotJob {
@@ -66,13 +76,24 @@ export interface BotQueueBackend {
claim(jobId: string, leaseSeconds: number): void;
/** done → removed; dead → moved to the capped DLQ. */
settle(jobId: string, outcome: 'done' | 'dead', reasonCode?: string): void;
- /** Jobs safe to resume: pending, or claimed whose lease has expired. */
+ /**
+ * Durable in-place retry for a claimed job: attempts+1, back to pending,
+ * with an optional run-after timestamp. No settle-then-reenqueue window —
+ * a crash at any point still leaves the job in the queue.
+ */
+ retry(jobId: string, attempts: number, runAfterMs?: number): void;
+ /** Jobs safe to resume: pending (due), or claimed whose lease has expired. */
rehydratable(): DurableBotJob[];
+ /** Pending jobs whose run-after time has arrived (retry backoff elapsed). */
+ dueJobs(): DurableBotJob[];
/** Expired-lease claimed jobs → pending. Returns the count requeued. */
requeueExpiredLeases(): number;
heartbeatLease(jobId: string, leaseSeconds: number): void;
listDlq(botId?: string): DlqEntry[];
removeFromDlq(jobId: string): DurableBotJob | null;
+ /** Durable bot-to-bot mailbox (§2.6: a handoff must survive a crash). */
+ enqueueMail(mail: Omit<DurableMail, 'id'>): string;
+ drainMail(botId: string): DurableMail[];
counts(): QueueCounts;
}
@@ -138,17 +159,31 @@ export class BotQueue {
this.backend.settle(jobId, outcome, reasonCode);
}
- heartbeatLease(jobId: string, leaseSeconds: number = LEASE_SECONDS): void {
- this.backend.heartbeatLease(jobId, leaseSeconds);
+ /** Durable in-place retry: no settle-then-reenqueue window. */
+ retry(jobId: string, attempts: number, runAfterMs?: number): void {
+ this.backend.retry(jobId, attempts, runAfterMs);
}
- /** Jobs to resume on startup: pending, plus expired-lease claimed jobs. */
+ /** Jobs to resume on startup: due pending, plus expired-lease claimed jobs. */
resumeJobs(): DurableBotJob[] {
const expired = this.backend.requeueExpiredLeases();
if (expired > 0) logger.info({ expired }, 'Requeued expired bot job leases');
return this.backend.rehydratable();
}
+ /** Pending jobs whose retry backoff has elapsed (swept periodically). */
+ dueJobs(): DurableBotJob[] {
+ return this.backend.dueJobs();
+ }
+
+ enqueueMail(mail: Omit<DurableMail, 'id'>): string {
+ return this.backend.enqueueMail(mail);
+ }
+
+ drainMail(botId: string): DurableMail[] {
+ return this.backend.drainMail(botId);
+ }
+
listDlq(botId?: string): DlqEntry[] {
return this.backend.listDlq(botId);
}
@@ -194,6 +229,14 @@ export class SqliteQueueBackend implements BotQueueBackend {
settled_at INTEGER
);
CREATE INDEX IF NOT EXISTS idx_bot_jobs_state ON bot_jobs (state, created_at);
+ CREATE TABLE IF NOT EXISTS bot_mail (
+ id TEXT PRIMARY KEY,
+ bot_id TEXT NOT NULL,
+ from_bot TEXT NOT NULL,
+ content TEXT NOT NULL,
+ created_at INTEGER NOT NULL
+ );
+ CREATE INDEX IF NOT EXISTS idx_bot_mail_bot ON bot_mail (bot_id, created_at);
CREATE TABLE IF NOT EXISTS bot_dlq (
id TEXT PRIMARY KEY,
bot_id TEXT NOT NULL,
@@ -208,6 +251,8 @@ export class SqliteQueueBackend implements BotQueueBackend {
dead_at INTEGER NOT NULL
);
`);
+ // Migration for queues created before run_after / bot_mail existed.
+ try { this.db.exec('ALTER TABLE bot_jobs ADD COLUMN run_after INTEGER'); } catch { /* already present */ }
this.dlqCap = dlqCap;
}
@@ -271,11 +316,49 @@ export class SqliteQueueBackend implements BotQueueBackend {
return result.changes as number;
}
+ retry(jobId: string, attempts: number, runAfterMs?: number): void {
+ this.db.prepare(
+ `UPDATE bot_jobs SET state = 'pending', attempts = ?, lease_expires_at = NULL, run_after = ? WHERE id = ?`,
+ ).run(attempts, runAfterMs ?? null, jobId);
+ }
+
rehydratable(): DurableBotJob[] {
- const rows = this.db.prepare(`SELECT * FROM bot_jobs WHERE state IN ('pending','claimed') ORDER BY created_at`).all() as any[];
+ // Due pending jobs + expired-lease claimed jobs — jobs still in their
+ // retry backoff (run_after in the future) wait until due.
+ const rows = this.db.prepare(
+ `SELECT * FROM bot_jobs WHERE state = 'pending' AND (run_after IS NULL OR run_after <= ?)
+ UNION ALL SELECT * FROM bot_jobs WHERE state = 'claimed' AND lease_expires_at <= ? ORDER BY created_at`,
+ ).all(Date.now(), Date.now()) as any[];
return rows.map(normalizeJob);
}
+ dueJobs(): DurableBotJob[] {
+ const rows = this.db.prepare(
+ `SELECT * FROM bot_jobs WHERE state = 'pending' AND (run_after IS NULL OR run_after <= ?) ORDER BY created_at`,
+ ).all(Date.now()) as any[];
+ return rows.map(normalizeJob);
+ }
+
+ enqueueMail(mail: Omit<DurableMail, 'id'>): string {
+ const id = randomUUID().slice(0, 12);
+ this.db.prepare(
+ `INSERT INTO bot_mail (id, bot_id, from_bot, content, created_at) VALUES (?, ?, ?, ?, ?)`,
+ ).run(id, mail.botId, mail.from, mail.content, mail.createdAt);
+ return id;
+ }
+
+ drainMail(botId: string): DurableMail[] {
+ const rows = this.db.prepare(
+ `SELECT * FROM bot_mail WHERE bot_id = ? ORDER BY created_at`,
+ ).all(botId) as any[];
+ if (rows.length === 0) return [];
+ this.db.prepare(`DELETE FROM bot_mail WHERE bot_id = ?`).run(botId);
+ return rows.map(r => ({
+ id: String(r.id), botId: String(r.bot_id), from: String(r.from_bot),
+ content: String(r.content), createdAt: Number(r.created_at),
+ }));
+ }
+
listDlq(botId?: string): DlqEntry[] {
const rows = botId
? this.db.prepare(`SELECT * FROM bot_dlq WHERE bot_id = ? ORDER BY dead_at DESC`).all(botId) as any[]
@@ -324,6 +407,7 @@ const QUEUE_FILE = 'queue.json';
interface QueueFile {
jobs: DurableBotJob[];
dlq: DlqEntry[];
+ mails: DurableMail[];
}
export class JsonFileQueueBackend implements BotQueueBackend {
@@ -336,7 +420,7 @@ export class JsonFileQueueBackend implements BotQueueBackend {
mkdirSync(dir, { recursive: true });
this.file = join(dir, QUEUE_FILE);
this.dlqCap = dlqCap;
- this.data = existsSync(this.file) ? this.read() : { jobs: [], dlq: [] };
+ this.data = existsSync(this.file) ? this.read() : { jobs: [], dlq: [], mails: [] };
}
private read(): QueueFile {
@@ -345,9 +429,10 @@ export class JsonFileQueueBackend implements BotQueueBackend {
return {
jobs: (parsed.jobs ?? []).map(normalizeJob),
dlq: (parsed.dlq ?? []).map((d: any) => ({ ...normalizeJob(d), state: 'dead' as const, deadAt: d.deadAt ?? Date.now() })),
+ mails: parsed.mails ?? [],
};
} catch {
- return { jobs: [], dlq: [] };
+ return { jobs: [], dlq: [], mails: [] };
}
}
@@ -412,8 +497,44 @@ export class JsonFileQueueBackend implements BotQueueBackend {
return n;
}
+ retry(jobId: string, attempts: number, runAfterMs?: number): void {
+ const job = this.data.jobs.find(j => j.id === jobId);
+ if (job) {
+ job.state = 'pending';
+ job.attempts = attempts;
+ job.leaseExpiresAt = undefined;
+ job.runAfter = runAfterMs;
+ this.flush();
+ }
+ }
+
rehydratable(): DurableBotJob[] {
- return [...this.data.jobs];
+ // Due pending + expired-lease claimed — future run_after waits.
+ const now = Date.now();
+ return this.data.jobs.filter(j =>
+ (j.state === 'pending' && (j.runAfter === undefined || j.runAfter <= now))
+ || (j.state === 'claimed' && (j.leaseExpiresAt ?? 0) < now));
+ }
+
+ dueJobs(): DurableBotJob[] {
+ const now = Date.now();
+ return this.data.jobs.filter(j => j.state === 'pending' && (j.runAfter === undefined || j.runAfter <= now));
+ }
+
+ enqueueMail(mail: Omit<DurableMail, 'id'>): string {
+ const id = randomUUID().slice(0, 12);
+ this.data.mails.push({ ...mail, id });
+ this.flush();
+ return id;
+ }
+
+ drainMail(botId: string): DurableMail[] {
+ const drained = this.data.mails.filter(m => m.botId === botId);
+ if (drained.length > 0) {
+ this.data.mails = this.data.mails.filter(m => m.botId !== botId);
+ this.flush();
+ }
+ return drained;
}
listDlq(botId?: string): DlqEntry[] {