Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
fix(bots): single re-entry path for retried jobs; deliver lead synthesis; durable needs-you
Salman Qureshi73378e9d89ef
diff --git a/src/bots/bot-manager.test.ts b/src/bots/bot-manager.test.tsindex d1b804c4..dd257732 100644--- a/src/bots/bot-manager.test.ts+++ b/src/bots/bot-manager.test.ts@@ -213,6 +213,79 @@ describe('BotManager queue + turn lifecycle', () => {}});+ it('a retried job never runs twice when the due-sweep and the backoff timer both fire (P0.7)', async () => {+ let calls = 0;+ let releaseSecond: () => void = () => {};+ const secondGate = new Promise<void>(r => { releaseSecond = r; });+ mockedGenerateText.mockImplementation(async () => {+ calls++;+ if (calls === 1) throw new Error('HTTP 429: too many requests');+ await secondGate; // hold the retry turn so the timer fires while it RUNS+ return { text: 'recovered', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;+ });+ seedBot(store, 'racer');+ // setImmediate stays real: the mocked turn is promise-driven, so a few+ // real macrotask yields settle it deterministically without waitFor.+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] });+ const settle = async () => { for (let i = 0; i < 20; i++) await new Promise(r => setImmediate(r)); };+ try {+ const { jobId } = manager.enqueue('racer', { trigger: 'chat', prompt: 'go' });+ await settle();+ // Attempt 1 failed transiently → durable in-place retry, 1s backoff timer.+ expect(manager.getJournal('racer').map(r => r.state)).toEqual(['failed']);+ expect(manager.queue.pendingJobs('racer').map(j => j.id)).toEqual([jobId]);++ // The periodic sweep wins the race: the clock passes run_after WITHOUT+ // the backoff timer firing, and the sweep moves the job into `running`.+ vi.setSystemTime(Date.now() + 1000);+ (manager as any).resumeDueJobs();+ await settle();+ expect(calls).toBe(2);+ expect((manager as any).running.get('racer')?.has(jobId)).toBe(true);++ // Now the backoff timer fires while that retry is still running. The+ // old timer pushed its own copy (the queue was empty) → a second run.+ await vi.advanceTimersByTimeAsync(1000);+ expect(manager.getQueuedCount('racer')).toBe(0);+ expect(calls).toBe(2);++ releaseSecond();+ await settle();+ // A later sweep tick finds nothing due either: the job settled 'done'.+ await vi.advanceTimersByTimeAsync(30_000);+ await settle();+ expect(calls).toBe(2);+ expect(manager.getJournal('racer').map(r => r.state)).toEqual(['failed', 'completed']);+ expect(manager.queue.counts()).toMatchObject({ pending: 0, claimed: 0, dlq: 0 });+ } finally {+ vi.useRealTimers();+ }+ });++ it('needs-you survives a manager restart (journal row) and clears on the next run (P0.7)', async () => {+ mockedGenerateText.mockRejectedValue(new Error('permission denied: no permission for that path'));+ seedBot(store, 'stuck');+ manager.enqueue('stuck', { trigger: 'chat', prompt: 'go' });+ await vi.waitFor(() => {+ expect(manager.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(true);+ });+ expect(manager.getJournal('stuck')[0]).toMatchObject({ state: 'failed', reasonCode: 'permission_denied', needsYou: true });++ // Restart: a new manager on the same root must still show the escalation.+ manager.dispose();+ const restarted = makeManager(root);+ expect(restarted.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(true);++ // The next run clears it — in memory AND durably.+ mockedGenerateText.mockResolvedValue({ text: 'fine now', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);+ restarted.enqueue('stuck', { trigger: 'chat', prompt: 'again' });+ await vi.waitFor(() => expect(restarted.getJournal('stuck')).toHaveLength(2));+ expect(restarted.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(false);+ restarted.dispose();+ const again = makeManager(root);+ expect(again.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(false);+ });+it('stop holds queued jobs (durable, nothing lost) and start resumes them', async () => {let release!: () => void;const gate = new Promise<void>(res => { release = res; });@@ -851,6 +924,38 @@ describe('Bot fleets (lead + crew)', () => {expect(manager.peekMailbox('ceo')).toHaveLength(0);});+ it('a lead\'s mailbox wake delivers its synthesis to the bot thread; crew task results stay internal (P0.7)', async () => {+ setupFleet();+ const delivered: Array<{ target: string; message: string }> = [];+ manager['notify'] = async (_t, target, message) => { delivered.push({ target, message }); };+ mockedGenerateText.mockImplementation(async (opts: any) => {+ const text = JSON.stringify(opts?.messages ?? '');+ // Lead wake (mailbox trigger) after the crew result landed: synthesize.+ if (text.includes('Task complete')) {+ return { text: 'SYNTHESIS: combined market report', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;+ }+ // Crew task turn (mailbox trigger with a reply target = the lead).+ if (text.includes('Study the market')) {+ return { text: 'CREW RESULT: market is up', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;+ }+ // Lead's first (chat) turn: delegate.+ manager.dispatchTask('researcher', 'ceo', 'Study the market');+ return { text: 'dispatched', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;+ });++ manager.enqueue('ceo', { trigger: 'chat', prompt: 'delegate the market work' });+ // The lead's combined report reaches the lead's own thread, tagged as+ // the mailbox turn it came from (it used to be journaled only).+ await vi.waitFor(() => {+ expect(delivered.some(d => d.target === 'bot:ceo' && d.message.includes('(mailbox): SYNTHESIS'))).toBe(true);+ });+ // The crew's task result went to the lead's MAILBOX only — the internal+ // crew→lead mail is never shown to the user, on any surface.+ expect(delivered.some(d => d.message.includes('CREW RESULT'))).toBe(false);+ expect(delivered.some(d => d.message.includes('Task complete (job'))).toBe(false);+ expect(delivered.some(d => d.target === 'bot:researcher')).toBe(false);+ });+it('leads get fleet tools; bot_spawn/bot_retire manage the crew within the cap', async () => {mockedGenerateText.mockResolvedValue({ text: 'A meticulous QA reviewer persona.', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);setupFleet();diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.tsindex ee4ce82d..b19a2725 100644--- a/src/bots/bot-manager.ts+++ b/src/bots/bot-manager.ts@@ -204,9 +204,14 @@ export class BotManager {}}// Durable mailboxes survive restarts: rehydrate into the in-memory boxes.+ // The needs-you badge too: it lives on the newest journal row (set when a+ // run is DLQ'd or crashes, cleared by the next run's row), so the+ // in-memory set is only a cache of that — a restart must not clear an+ // escalation the owner has not seen.for (const m of this.store.list()) {const drained = this.queue.drainMail(m.id);if (drained.length > 0) this.mailboxes.set(m.id, drained);+ if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou) this.needsYou.add(m.id);}// A profile relocation (re-parent or fleet-layout migration) invalidates// the cached journal handle (it is bound to the old dir) and the bot's@@ -257,25 +262,39 @@ export class BotManager {}}- /** Pull due (backoff-elapsed) jobs from the durable queue into memory and run them. */+ /**+ * Pull due (backoff-elapsed) jobs from the durable queue into memory and+ * run them. This is the ONLY re-entry path for a retried/paused job: the+ * precise backoff timers call it too, instead of pushing a captured copy+ * of the job themselves. A timer that pushed its own copy raced the 30s+ * sweep — the sweep moved the job from the queue into `running`, the+ * timer saw an empty queue and pushed it again, and the job ran twice+ * (the second settle a silent no-op). Reading durable state here makes+ * re-entry idempotent: a running job is `claimed` (not due), a finished+ * one is gone, a queued one is skipped by id.+ */private resumeDueJobs(): void {const due = this.queue.dueJobs();for (const job of due) {if (this.disabled.has(job.botId) || this.held.has(job.botId)) continue;- const running = this.running.get(job.botId)?.size ?? 0;- const q = this.queues.get(job.botId) ?? [];- const alreadyQueued = q.some(j => j.id === job.id);- if (!alreadyQueued && running === 0) {- q.push({- id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,- fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,- });- this.queues.set(job.botId, q);- this.pump(job.botId);- }+ this.reenterJob(job);}}+ /** Re-enter a durable job into its bot's in-memory lane unless that id is+ * already queued OR running — a job id is never in both, and never twice. */+ private reenterJob(job: DurableBotJob): boolean {+ const q = this.queues.get(job.botId) ?? [];+ if (q.some(j => j.id === job.id) || this.running.get(job.botId)?.has(job.id)) return false;+ q.push({+ id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,+ fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,+ });+ this.queues.set(job.botId, q);+ this.pump(job.botId);+ return true;+ }+/*** Fleet-wide concurrency cap: config override or a fixed default of 8.* NOT cpu-derived: a bot turn is a network-bound LLM call plus small tool@@ -474,6 +493,12 @@ export class BotManager {});turn.cleanup();+ // Decided up front so the journal row can carry the needs-you flag:+ // a failure that will retry is not an escalation; one headed for the+ // DLQ is (the flag is durable via the row — the constructor rehydrates+ // it from the newest journal row on restart).+ const willRetry = output.status === 'failed' && !!output.reasonCode+ && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS;const record: BotRunRecord = {runId: job.id,botId,@@ -486,6 +511,7 @@ export class BotManager {summary: output.output.slice(0, 300),error: output.error,reasonCode: output.reasonCode,+ ...(output.status === 'failed' && !willRetry ? { needsYou: true } : {}),};this.journalFor(botId).append(record);this.lastRun.set(botId, { at: Date.now(), state: record.state });@@ -518,18 +544,14 @@ export class BotManager {// failures go to the capped DLQ and stop (never silently re-queued — §2.6).// Retries requeue the SAME job in place (durable): no settle-then-// reenqueue window where a crash would lose the work.- if (output.status === 'failed' && output.reasonCode && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS) {+ // Re-entry rides the due-sweep (resumeDueJobs) — the timer only makes+ // it precise; it never pushes a copy of the job itself (duplicate-run+ // race with the periodic sweep).+ if (willRetry) {const delay = Math.min(15000, 1000 * 2 ** job.attempts);this.queue.retry(job.id, job.attempts + 1, Date.now() + delay);logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying in place');- setTimeout(() => {- const q = this.queues.get(botId) ?? [];- if (!q.some(j => j.id === job.id)) {- q.push({ ...job, attempts: job.attempts + 1 });- this.queues.set(botId, q);- this.pump(botId);- }- }, delay).unref?.();+ setTimeout(() => this.resumeDueJobs(), delay).unref?.();} else if (output.status === 'failed') {this.queue.settle(job.id, 'dead', output.reasonCode);this.needsYou.add(botId);@@ -539,14 +561,7 @@ export class BotManager {// Step-budget pause: work continues next turn — same job requeues in// place (durable), no attempts bump.this.queue.retry(job.id, job.attempts, Date.now() + 2000);- setTimeout(() => {- const q = this.queues.get(botId) ?? [];- if (!q.some(j => j.id === job.id)) {- q.push({ ...job });- this.queues.set(botId, q);- this.pump(botId);- }- }, 2000).unref?.();+ setTimeout(() => this.resumeDueJobs(), 2000).unref?.();} else {this.queue.settle(job.id, 'done');}@@ -560,7 +575,15 @@ export class BotManager {// channels (Telegram/web) stay the exception: their user cannot open// bot threads, so the full result is delivered in that chat. Halts// report too (a stopped run must announce it stopped, §2.6).- if (this.notify && job.trigger !== 'mailbox') {+ //+ // Mailbox turns deliver too UNLESS the result is addressed to another+ // bot (a delegated task — replyTargetFor routes it to the sender's+ // mailbox below, and the user must not see the internal crew→lead mail+ // twice). A lead's wake after crew results IS a mailbox trigger+ // (sendToBot), and its synthesis is the user-facing outcome of the+ // whole delegation — skipping every mailbox turn left it journaled+ // but never shown.+ if (this.notify && !replyTargetFor(job)) {const botThread = `bot:${botId}`;const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : output.status === 'halted' ? '⏹' : '⏸';// Full result, matched to the channel's per-message cap (64KB) — the@@ -618,6 +641,7 @@ export class BotManager {tokensOut: 0,error: err?.message,reasonCode: 'unknown_error',+ needsYou: true,});this.needsYou.add(botId);} finally {diff --git a/src/bots/types.ts b/src/bots/types.tsindex 2b173a8c..794a5627 100644--- a/src/bots/types.ts+++ b/src/bots/types.ts@@ -166,6 +166,9 @@ export interface BotRunRecord {error?: string;/** Typed failure code surfaced to senders/channels (e.g. provider_rate_limit). */reasonCode?: string;+ /** Escalation marker: this run needs the owner (DLQ'd failure / crash).+ * The newest row's flag is the durable "needs you" state across restarts. */+ needsYou?: boolean;}/** Live bot state surfaced by /bots, the status bar, and the API. */