Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

fix(bots): single re-entry path for retried jobs; deliver lead synthesis; durable needs-you

Salman Qureshi73378e9d89ef
diff --git a/src/bots/bot-manager.test.ts b/src/bots/bot-manager.test.ts
index d1b804c4..dd257732 100644
--- a/src/bots/bot-manager.test.ts
+++ b/src/bots/bot-manager.test.ts
@@ -213,6 +213,79 @@ describe('BotManager queue + turn lifecycle', () => {
}
});
+ it('a retried job never runs twice when the due-sweep and the backoff timer both fire (P0.7)', async () => {
+ let calls = 0;
+ let releaseSecond: () => void = () => {};
+ const secondGate = new Promise<void>(r => { releaseSecond = r; });
+ mockedGenerateText.mockImplementation(async () => {
+ calls++;
+ if (calls === 1) throw new Error('HTTP 429: too many requests');
+ await secondGate; // hold the retry turn so the timer fires while it RUNS
+ return { text: 'recovered', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;
+ });
+ seedBot(store, 'racer');
+ // setImmediate stays real: the mocked turn is promise-driven, so a few
+ // real macrotask yields settle it deterministically without waitFor.
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] });
+ const settle = async () => { for (let i = 0; i < 20; i++) await new Promise(r => setImmediate(r)); };
+ try {
+ const { jobId } = manager.enqueue('racer', { trigger: 'chat', prompt: 'go' });
+ await settle();
+ // Attempt 1 failed transiently → durable in-place retry, 1s backoff timer.
+ expect(manager.getJournal('racer').map(r => r.state)).toEqual(['failed']);
+ expect(manager.queue.pendingJobs('racer').map(j => j.id)).toEqual([jobId]);
+
+ // The periodic sweep wins the race: the clock passes run_after WITHOUT
+ // the backoff timer firing, and the sweep moves the job into `running`.
+ vi.setSystemTime(Date.now() + 1000);
+ (manager as any).resumeDueJobs();
+ await settle();
+ expect(calls).toBe(2);
+ expect((manager as any).running.get('racer')?.has(jobId)).toBe(true);
+
+ // Now the backoff timer fires while that retry is still running. The
+ // old timer pushed its own copy (the queue was empty) → a second run.
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(manager.getQueuedCount('racer')).toBe(0);
+ expect(calls).toBe(2);
+
+ releaseSecond();
+ await settle();
+ // A later sweep tick finds nothing due either: the job settled 'done'.
+ await vi.advanceTimersByTimeAsync(30_000);
+ await settle();
+ expect(calls).toBe(2);
+ expect(manager.getJournal('racer').map(r => r.state)).toEqual(['failed', 'completed']);
+ expect(manager.queue.counts()).toMatchObject({ pending: 0, claimed: 0, dlq: 0 });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it('needs-you survives a manager restart (journal row) and clears on the next run (P0.7)', async () => {
+ mockedGenerateText.mockRejectedValue(new Error('permission denied: no permission for that path'));
+ seedBot(store, 'stuck');
+ manager.enqueue('stuck', { trigger: 'chat', prompt: 'go' });
+ await vi.waitFor(() => {
+ expect(manager.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(true);
+ });
+ expect(manager.getJournal('stuck')[0]).toMatchObject({ state: 'failed', reasonCode: 'permission_denied', needsYou: true });
+
+ // Restart: a new manager on the same root must still show the escalation.
+ manager.dispose();
+ const restarted = makeManager(root);
+ expect(restarted.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(true);
+
+ // The next run clears it — in memory AND durably.
+ mockedGenerateText.mockResolvedValue({ text: 'fine now', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);
+ restarted.enqueue('stuck', { trigger: 'chat', prompt: 'again' });
+ await vi.waitFor(() => expect(restarted.getJournal('stuck')).toHaveLength(2));
+ expect(restarted.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(false);
+ restarted.dispose();
+ const again = makeManager(root);
+ expect(again.getStatusSummaries().find(s => s.id === 'stuck')?.needsYou).toBe(false);
+ });
+
it('stop holds queued jobs (durable, nothing lost) and start resumes them', async () => {
let release!: () => void;
const gate = new Promise<void>(res => { release = res; });
@@ -851,6 +924,38 @@ describe('Bot fleets (lead + crew)', () => {
expect(manager.peekMailbox('ceo')).toHaveLength(0);
});
+ it('a lead\'s mailbox wake delivers its synthesis to the bot thread; crew task results stay internal (P0.7)', async () => {
+ setupFleet();
+ const delivered: Array<{ target: string; message: string }> = [];
+ manager['notify'] = async (_t, target, message) => { delivered.push({ target, message }); };
+ mockedGenerateText.mockImplementation(async (opts: any) => {
+ const text = JSON.stringify(opts?.messages ?? '');
+ // Lead wake (mailbox trigger) after the crew result landed: synthesize.
+ if (text.includes('Task complete')) {
+ return { text: 'SYNTHESIS: combined market report', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;
+ }
+ // Crew task turn (mailbox trigger with a reply target = the lead).
+ if (text.includes('Study the market')) {
+ return { text: 'CREW RESULT: market is up', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;
+ }
+ // Lead's first (chat) turn: delegate.
+ manager.dispatchTask('researcher', 'ceo', 'Study the market');
+ return { text: 'dispatched', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any;
+ });
+
+ manager.enqueue('ceo', { trigger: 'chat', prompt: 'delegate the market work' });
+ // The lead's combined report reaches the lead's own thread, tagged as
+ // the mailbox turn it came from (it used to be journaled only).
+ await vi.waitFor(() => {
+ expect(delivered.some(d => d.target === 'bot:ceo' && d.message.includes('(mailbox): SYNTHESIS'))).toBe(true);
+ });
+ // The crew's task result went to the lead's MAILBOX only — the internal
+ // crew→lead mail is never shown to the user, on any surface.
+ expect(delivered.some(d => d.message.includes('CREW RESULT'))).toBe(false);
+ expect(delivered.some(d => d.message.includes('Task complete (job'))).toBe(false);
+ expect(delivered.some(d => d.target === 'bot:researcher')).toBe(false);
+ });
+
it('leads get fleet tools; bot_spawn/bot_retire manage the crew within the cap', async () => {
mockedGenerateText.mockResolvedValue({ text: 'A meticulous QA reviewer persona.', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);
setupFleet();
diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.ts
index ee4ce82d..b19a2725 100644
--- a/src/bots/bot-manager.ts
+++ b/src/bots/bot-manager.ts
@@ -204,9 +204,14 @@ export class BotManager {
}
}
// Durable mailboxes survive restarts: rehydrate into the in-memory boxes.
+ // The needs-you badge too: it lives on the newest journal row (set when a
+ // run is DLQ'd or crashes, cleared by the next run's row), so the
+ // in-memory set is only a cache of that — a restart must not clear an
+ // escalation the owner has not seen.
for (const m of this.store.list()) {
const drained = this.queue.drainMail(m.id);
if (drained.length > 0) this.mailboxes.set(m.id, drained);
+ if (this.journalFor(m.id).read(m.id, 1)[0]?.needsYou) this.needsYou.add(m.id);
}
// A profile relocation (re-parent or fleet-layout migration) invalidates
// the cached journal handle (it is bound to the old dir) and the bot's
@@ -257,25 +262,39 @@ export class BotManager {
}
}
- /** Pull due (backoff-elapsed) jobs from the durable queue into memory and run them. */
+ /**
+ * Pull due (backoff-elapsed) jobs from the durable queue into memory and
+ * run them. This is the ONLY re-entry path for a retried/paused job: the
+ * precise backoff timers call it too, instead of pushing a captured copy
+ * of the job themselves. A timer that pushed its own copy raced the 30s
+ * sweep — the sweep moved the job from the queue into `running`, the
+ * timer saw an empty queue and pushed it again, and the job ran twice
+ * (the second settle a silent no-op). Reading durable state here makes
+ * re-entry idempotent: a running job is `claimed` (not due), a finished
+ * one is gone, a queued one is skipped by id.
+ */
private resumeDueJobs(): void {
const due = this.queue.dueJobs();
for (const job of due) {
if (this.disabled.has(job.botId) || this.held.has(job.botId)) continue;
- const running = this.running.get(job.botId)?.size ?? 0;
- const q = this.queues.get(job.botId) ?? [];
- const alreadyQueued = q.some(j => j.id === job.id);
- if (!alreadyQueued && running === 0) {
- q.push({
- id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,
- fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,
- });
- this.queues.set(job.botId, q);
- this.pump(job.botId);
- }
+ this.reenterJob(job);
}
}
+ /** Re-enter a durable job into its bot's in-memory lane unless that id is
+ * already queued OR running — a job id is never in both, and never twice. */
+ private reenterJob(job: DurableBotJob): boolean {
+ const q = this.queues.get(job.botId) ?? [];
+ if (q.some(j => j.id === job.id) || this.running.get(job.botId)?.has(job.id)) return false;
+ q.push({
+ id: job.id, botId: job.botId, trigger: job.trigger, prompt: job.prompt,
+ fromBot: job.fromBot, source: job.source, createdAt: job.createdAt, attempts: job.attempts,
+ });
+ this.queues.set(job.botId, q);
+ this.pump(job.botId);
+ return true;
+ }
+
/**
* Fleet-wide concurrency cap: config override or a fixed default of 8.
* NOT cpu-derived: a bot turn is a network-bound LLM call plus small tool
@@ -474,6 +493,12 @@ export class BotManager {
});
turn.cleanup();
+ // Decided up front so the journal row can carry the needs-you flag:
+ // a failure that will retry is not an escalation; one headed for the
+ // DLQ is (the flag is durable via the row — the constructor rehydrates
+ // it from the newest journal row on restart).
+ const willRetry = output.status === 'failed' && !!output.reasonCode
+ && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS;
const record: BotRunRecord = {
runId: job.id,
botId,
@@ -486,6 +511,7 @@ export class BotManager {
summary: output.output.slice(0, 300),
error: output.error,
reasonCode: output.reasonCode,
+ ...(output.status === 'failed' && !willRetry ? { needsYou: true } : {}),
};
this.journalFor(botId).append(record);
this.lastRun.set(botId, { at: Date.now(), state: record.state });
@@ -518,18 +544,14 @@ export class BotManager {
// failures go to the capped DLQ and stop (never silently re-queued — §2.6).
// Retries requeue the SAME job in place (durable): no settle-then-
// reenqueue window where a crash would lose the work.
- if (output.status === 'failed' && output.reasonCode && isTransientFailure(output.reasonCode) && job.attempts + 1 < MAX_TRANSIENT_ATTEMPTS) {
+ // Re-entry rides the due-sweep (resumeDueJobs) — the timer only makes
+ // it precise; it never pushes a copy of the job itself (duplicate-run
+ // race with the periodic sweep).
+ if (willRetry) {
const delay = Math.min(15000, 1000 * 2 ** job.attempts);
this.queue.retry(job.id, job.attempts + 1, Date.now() + delay);
logger.info({ botId, jobId: job.id, reasonCode: output.reasonCode, retryIn: delay }, 'Bot turn failed transiently — retrying in place');
- setTimeout(() => {
- const q = this.queues.get(botId) ?? [];
- if (!q.some(j => j.id === job.id)) {
- q.push({ ...job, attempts: job.attempts + 1 });
- this.queues.set(botId, q);
- this.pump(botId);
- }
- }, delay).unref?.();
+ setTimeout(() => this.resumeDueJobs(), delay).unref?.();
} else if (output.status === 'failed') {
this.queue.settle(job.id, 'dead', output.reasonCode);
this.needsYou.add(botId);
@@ -539,14 +561,7 @@ export class BotManager {
// Step-budget pause: work continues next turn — same job requeues in
// place (durable), no attempts bump.
this.queue.retry(job.id, job.attempts, Date.now() + 2000);
- setTimeout(() => {
- const q = this.queues.get(botId) ?? [];
- if (!q.some(j => j.id === job.id)) {
- q.push({ ...job });
- this.queues.set(botId, q);
- this.pump(botId);
- }
- }, 2000).unref?.();
+ setTimeout(() => this.resumeDueJobs(), 2000).unref?.();
} else {
this.queue.settle(job.id, 'done');
}
@@ -560,7 +575,15 @@ export class BotManager {
// channels (Telegram/web) stay the exception: their user cannot open
// bot threads, so the full result is delivered in that chat. Halts
// report too (a stopped run must announce it stopped, §2.6).
- if (this.notify && job.trigger !== 'mailbox') {
+ //
+ // Mailbox turns deliver too UNLESS the result is addressed to another
+ // bot (a delegated task — replyTargetFor routes it to the sender's
+ // mailbox below, and the user must not see the internal crew→lead mail
+ // twice). A lead's wake after crew results IS a mailbox trigger
+ // (sendToBot), and its synthesis is the user-facing outcome of the
+ // whole delegation — skipping every mailbox turn left it journaled
+ // but never shown.
+ if (this.notify && !replyTargetFor(job)) {
const botThread = `bot:${botId}`;
const icon = output.status === 'completed' ? '🤖' : output.status === 'failed' ? '❌' : output.status === 'halted' ? '⏹' : '⏸';
// Full result, matched to the channel's per-message cap (64KB) — the
@@ -618,6 +641,7 @@ export class BotManager {
tokensOut: 0,
error: err?.message,
reasonCode: 'unknown_error',
+ needsYou: true,
});
this.needsYou.add(botId);
} finally {
diff --git a/src/bots/types.ts b/src/bots/types.ts
index 2b173a8c..794a5627 100644
--- a/src/bots/types.ts
+++ b/src/bots/types.ts
@@ -166,6 +166,9 @@ export interface BotRunRecord {
error?: string;
/** Typed failure code surfaced to senders/channels (e.g. provider_rate_limit). */
reasonCode?: string;
+ /** Escalation marker: this run needs the owner (DLQ'd failure / crash).
+ * The newest row's flag is the durable "needs you" state across restarts. */
+ needsYou?: boolean;
}
/** Live bot state surfaced by /bots, the status bar, and the API. */