Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
// Mock generateText; keep the rest of the ai module real for tool factories.
vi.mock('ai', async (importOriginal) => {
const actual = await importOriginal<typeof import('ai')>();
return { ...actual, generateText: vi.fn(), streamText: vi.fn() };
});
import { generateText, streamText } from 'ai';
const mockedGenerateText = vi.mocked(generateText);
const mockedStreamText = vi.mocked(streamText);
// Bot turns run on streamText (live thinking deltas). Tests script
// generateText; this shim feeds THAT script through the streaming shape the
// turn loop consumes: the (async, mocked) generateText call fires the same
// step callbacks inside fullStream consumption, and its final result shows
// up as the text/finishReason promises runBotTurn awaits.
mockedStreamText.mockImplementation(((opts: any) => {
let final = { text: '', finishReason: 'stop', usage: {} };
let failed: unknown = null;
let resolveSettled: () => void = () => { };
const settled = new Promise<void>((r) => { resolveSettled = r; });
const gen: any = (generateText as any)(opts) || Promise.resolve(final);
const fullStream = (async function* () {
try {
final = await gen;
} catch (err) {
failed = err; // rethrown by the text/finishReason promises below
}
resolveSettled();
})();
// Resolve only after the underlying generateText settles; its promises
// must never reject unhandled (the fullStream consumer owns the error).
const once = async (pick: () => any) => { await settled; if (failed !== null) throw failed; return pick(); };
return {
fullStream,
text: once(() => final.text),
finishReason: once(() => final.finishReason),
// runBotTurn reads usage from onStepFinish — usage must never reject
// unhandled (Promise.all only consumes text/finishReason).
usage: once(() => final.usage).catch(() => ({})),
};
}) as any);
import { BotQueue, JsonFileQueueBackend, SqliteQueueBackend, idempotencyKeyFor, LEASE_SECONDS } from './queue.js';
import type { DurableBotJob } from './queue.js';
import { openSqlite } from '../utils/sqlite-driver.js';
function job(id: string, overrides: Partial<DurableBotJob> = {}): Omit<DurableBotJob, 'state'> {
return {
id,
botId: 'researcher',
trigger: 'chat',
prompt: `task ${id}`,
attempts: 0,
createdAt: Date.now(),
idempotencyKey: idempotencyKeyFor('researcher', 'chat', `task ${id}`),
...overrides,
} as Omit<DurableBotJob, 'state'>;
}
function makeBackends(dir: string) {
return {
json: new JsonFileQueueBackend(dir, 100),
sqlite: new SqliteQueueBackend(dir, 100),
};
}
// Every backend opened by a test is registered here and closed in afterEach —
// on Windows an open SQLite handle keeps queue.db locked (EBUSY) and makes
// the tmpdir teardown fail.
const openBackends: Array<{ close(): void }> = [];
function mkBackend(name: 'json' | 'sqlite', dir: string, dlqCap = 100) {
const backend = name === 'json'
? new JsonFileQueueBackend(dir, dlqCap)
: new SqliteQueueBackend(dir, dlqCap);
openBackends.push(backend);
return backend;
}
describe('BotQueue backends (JSON-file + SQLite semantics)', () => {
let root: string;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), 'mercury-bot-queue-'));
});
afterEach(() => {
for (const b of openBackends.splice(0)) b.close();
rmSync(root, { recursive: true, force: true });
});
for (const name of ['json', 'sqlite'] as const) {
describe(`${name} backend`, () => {
it('enqueues pending and persists across restart', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('a1'));
// Simulate a restart by re-reading from disk with a fresh instance
const fresh = mkBackend(name, root);
const resumed = fresh.rehydratable();
expect(resumed).toHaveLength(1);
expect(resumed[0].id).toBe('a1');
expect(resumed[0].state).toBe('pending');
});
it('deduplicates on the idempotency key while pending', () => {
const backend = mkBackend(name, root);
const first = backend.enqueue(job('a1', { prompt: 'same', idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'same') }));
const second = backend.enqueue(job('a2', { prompt: 'same', idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'same') }));
expect(first.duplicated).toBe(false);
expect(second.duplicated).toBe(true);
expect(second.job.id).toBe('a1');
expect(backend.rehydratable()).toHaveLength(1);
});
it('claim sets a lease (in-flight, not resumable); settle done removes the job', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('a1'));
backend.claim('a1', LEASE_SECONDS);
// Fresh lease = in-flight: not resumable, but visible in counts.
expect(backend.rehydratable()).toHaveLength(0);
expect(backend.counts().claimed).toBe(1);
backend.settle('a1', 'done');
expect(backend.rehydratable()).toHaveLength(0);
expect(backend.counts()).toEqual({ pending: 0, claimed: 0, dlq: 0 });
});
it('pendingJobs returns only that bot’s pending work, ignoring retry backoff', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('h1'));
backend.enqueue(job('h2', { botId: 'other' }));
backend.enqueue(job('backoff', { idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'task backoff') }));
backend.retry('backoff', 1, Date.now() + 60_000); // in retry backoff
const pending = backend.pendingJobs('researcher');
expect(pending.map(j => j.id)).toEqual(['h1', 'backoff']); // backoff included: explicit resume overrides it
});
it('keeps a job’s files across a restart, and dedupes on them too', () => {
const file = { kind: 'image' as const, path: '/uploads/2026-10/a-dot.png', mime: 'image/png', name: 'dot.png' };
const backend = mkBackend(name, root);
backend.enqueue(job('a1', { prompt: 'what is this?', attachments: [file] }));
// Durable: a fresh handle on the same db still sees the file list.
const resumed = mkBackend(name, root).rehydratable();
expect(resumed[0].attachments).toEqual([file]);
// Same caption, different photo = different job.
const other = mkBackend(name, root).enqueue(job('a2', {
prompt: 'what is this?',
attachments: [{ ...file, path: '/uploads/2026-10/b-other.png', name: 'other.png' }],
idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'what is this?', undefined, [
{ ...file, path: '/uploads/2026-10/b-other.png', name: 'other.png' },
]),
}));
expect(other.duplicated).toBe(false);
});
it('settle dead moves the job to the DLQ with the reason', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('a1'));
backend.settle('a1', 'dead', 'provider_auth');
expect(backend.rehydratable()).toHaveLength(0);
const dlq = backend.listDlq('researcher');
expect(dlq).toHaveLength(1);
expect(dlq[0].reasonCode).toBe('provider_auth');
expect(dlq[0].attempts).toBe(1);
});
it('expired leases requeue to pending; fresh leases survive in-flight', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('expired'));
backend.enqueue(job('alive'));
backend.claim('expired', 0); // expires immediately
backend.claim('alive', 3600);
const requeued = backend.requeueExpiredLeases();
expect(requeued).toBe(1);
expect(backend.rehydratable().find(j => j.id === 'expired')?.state).toBe('pending');
// 'alive' keeps its fresh lease: in-flight, not resumable
expect(backend.rehydratable().find(j => j.id === 'alive')).toBeUndefined();
expect(backend.counts().claimed).toBe(1);
});
it('quarantines interrupted scheduled jobs while preserving safe first attempts and chat recovery', () => {
const backend = mkBackend(name, root);
const queue = Object.create(BotQueue.prototype) as BotQueue;
const interrupted = vi.fn();
Object.assign(queue, { backend, onInterruptedSchedule: interrupted });
backend.enqueue(job('scheduled-claimed', { trigger: 'cron' }));
backend.claim('scheduled-claimed', 0);
backend.enqueue(job('scheduled-fresh', { trigger: 'cron' }));
backend.enqueue(job('scheduled-retry', { trigger: 'cron' }));
backend.claim('scheduled-retry', 0);
backend.retry('scheduled-retry', 1, Date.now() - 1);
backend.enqueue(job('scheduled-continuation', { trigger: 'cron' }));
backend.claim('scheduled-continuation', 0);
backend.retry('scheduled-continuation', 0, Date.now() - 1);
backend.enqueue(job('chat-claimed'));
backend.claim('chat-claimed', 0);
expect(queue.resumeJobs().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);
expect(interrupted).toHaveBeenCalledTimes(3);
expect(backend.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-claimed', 'scheduled-continuation', 'scheduled-retry']);
expect(backend.listDlq().every((j) => j.reasonCode === 'scheduled_interrupted')).toBe(true);
const restarted = mkBackend(name, root);
expect(restarted.rehydratable().map((j) => j.id).sort()).toEqual(['chat-claimed', 'scheduled-fresh']);
});
it('quarantines a scheduled run on quick restart before its lease expires', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('scheduled-live-lease', { trigger: 'cron' }));
backend.claim('scheduled-live-lease', 3600);
backend.enqueue(job('chat-live-lease'));
backend.claim('chat-live-lease', 3600);
backend.enqueue(job('scheduled-future-retry', { trigger: 'cron' }));
backend.retry('scheduled-future-retry', 1, Date.now() + 3600_000);
const restarted = mkBackend(name, root);
const queue = Object.create(BotQueue.prototype) as BotQueue;
const pause = vi.fn();
Object.assign(queue, { backend: restarted, onInterruptedSchedule: pause });
expect(queue.resumeJobs()).toEqual([]);
expect(pause).toHaveBeenCalledTimes(2);
expect(restarted.listDlq().map((j) => j.id).sort()).toEqual(['scheduled-future-retry', 'scheduled-live-lease']);
expect(restarted.counts()).toMatchObject({ claimed: 1 });
});
it('does not discard the interrupted job when saving its routine pause fails', () => {
const backend = mkBackend(name, root);
const queue = Object.create(BotQueue.prototype) as BotQueue;
Object.assign(queue, { backend, onInterruptedSchedule: () => { throw new Error('Disk full'); } });
backend.enqueue(job('interrupted', { trigger: 'cron' }));
backend.claim('interrupted', 0);
expect(() => queue.resumeJobs()).toThrow('Disk full');
expect(backend.rehydratable().map((j) => j.id)).toEqual(['interrupted']);
expect(backend.listDlq()).toEqual([]);
queue.onInterruptedSchedule = vi.fn();
expect(queue.resumeJobs()).toEqual([]);
expect(backend.listDlq()[0].reasonCode).toBe('scheduled_interrupted');
});
it('DLQ is capped with oldest-first eviction', () => {
const backend = mkBackend(name, root, 3);
for (let i = 0; i < 5; i++) {
backend.enqueue(job(`d${i}`, { createdAt: i }));
backend.settle(`d${i}`, 'dead', 'test');
}
const dlq = backend.listDlq();
expect(dlq).toHaveLength(3);
// Newest survive
expect(dlq.map(j => j.id).sort()).toEqual(['d2', 'd3', 'd4']);
});
it('removeFromDlq returns the entry and clears it for replay', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('a1'));
backend.settle('a1', 'dead', 'provider_rate_limit');
const removed = backend.removeFromDlq('a1');
expect(removed?.id).toBe('a1');
expect(removed?.prompt).toBe('task a1');
expect(backend.listDlq()).toHaveLength(0);
expect(backend.removeFromDlq('a1')).toBeNull();
});
it('counts reflect live state', () => {
const backend = mkBackend(name, root);
backend.enqueue(job('a1'));
backend.enqueue(job('a2'));
backend.claim('a1', 3600);
expect(backend.counts()).toEqual({ pending: 1, claimed: 1, dlq: 0 });
});
});
}
it('rolls back SQLite quarantine if the DLQ insert fails after removing the active row', () => {
const db = openSqlite(join(root, 'queue.db'))!;
const backend = new SqliteQueueBackend(root, 100, db);
openBackends.push(backend);
backend.enqueue(job('quarantine', { trigger: 'cron' }));
backend.claim('quarantine', 3600);
db.exec(`CREATE TRIGGER fail_dlq BEFORE INSERT ON bot_dlq BEGIN SELECT RAISE(ABORT, 'injected disk error'); END`);
expect(() => backend.settle('quarantine', 'dead', 'scheduled_interrupted')).toThrow('injected disk error');
expect(backend.interruptedSchedules().map((j) => j.id)).toEqual(['quarantine']);
expect(backend.listDlq()).toEqual([]);
db.exec('DROP TRIGGER fail_dlq');
backend.settle('quarantine', 'dead', 'scheduled_interrupted');
expect(backend.interruptedSchedules()).toEqual([]);
expect(backend.listDlq()[0]).toMatchObject({ id: 'quarantine', prompt: 'task quarantine', reasonCode: 'scheduled_interrupted' });
});
it('BotQueue facade resumes expired leases and pending jobs', () => {
const dir = join(root, 'bots');
const queue = new BotQueue(dir, 100);
openBackends.push(queue);
queue.enqueue(job('r1'));
queue.enqueue(job('r2'));
queue.claim('r1', 0);
const resumed = queue.resumeJobs();
expect(resumed.map(j => j.id).sort()).toEqual(['r1', 'r2']);
expect(resumed.every(j => j.state === 'pending')).toBe(true);
});
it('retry requeues the same job in place — no settle-then-reenqueue window', () => {
for (const name of ['json', 'sqlite'] as const) {
const dir = join(root, `retry-${name}`);
const backend = mkBackend(name, dir);
backend.enqueue(job('t1'));
backend.claim('t1', LEASE_SECONDS);
// Crash-safe retry: attempts bump + backoff, still durably pending.
backend.retry('t1', 1, Date.now() + 5000);
expect(backend.rehydratable()).toHaveLength(0); // backoff not elapsed
expect(backend.dueJobs()).toHaveLength(0);
// A restart (fresh instance over the same storage) still sees the job
const fresh = mkBackend(name, dir);
expect(fresh.rehydratable().length).toBeGreaterThanOrEqual(0);
expect(fresh.counts().pending).toBe(1);
}
});
it('durable mail survives a restart and drains once', () => {
for (const name of ['json', 'sqlite'] as const) {
const dir = join(root, `mail-${name}`);
const backend = mkBackend(name, dir);
backend.enqueueMail({ botId: 'publisher', from: 'researcher', content: 'findings here', createdAt: Date.now() });
// Restart: the mail is still there
const fresh = mkBackend(name, dir);
const drained = fresh.drainMail('publisher');
expect(drained).toHaveLength(1);
expect(drained[0].from).toBe('researcher');
expect(drained[0].content).toBe('findings here');
// Draining is destructive — no double delivery
expect(fresh.drainMail('publisher')).toHaveLength(0);
expect(fresh.drainMail('other-bot')).toHaveLength(0);
}
});
});
describe('BotManager durable queue integration', () => {
let root: string;
let store: any;
let manager: any;
// Reuse the makeManager helper shape from bot-manager.test.ts
beforeEach(async () => {
const { mkdtempSync: mkd, rmSync: rm } = await import('node:fs');
root = mkd(join(tmpdir(), 'mercury-bot-durable-'));
const { BotStore } = await import('./store.js');
const { BotManager } = await import('./bot-manager.js');
const { getDefaultConfig } = await import('../utils/config.js');
store = new BotStore(join(root, 'bots'));
const config = getDefaultConfig() as any;
config.bots.maxConcurrent = 4;
manager = new BotManager({
config,
providers: {
get: () => undefined,
getDefault: () => ({
name: 'stub', model: 'stub',
generateText: async () => ({ text: 'ok', inputTokens: 0, outputTokens: 0, totalTokens: 0, model: 'stub', provider: 'stub' }),
streamText: async function* () { yield { text: 'ok', done: true }; },
isAvailable: () => true, getModelInstance: () => ({}), getModel: () => 'stub',
}),
} as any,
tokenBudget: { recordUsage: () => {}, getRemaining: () => 1000, getStatusText: () => '', getUsagePercentage: () => 0 } as any,
store,
userMemoryFactory: () => null,
});
});
afterEach(() => {
// The manager owns the SQLite queue handle — close it or Windows locks
// queue.db (EBUSY) and the tmpdir teardown fails.
try { manager?.dispose?.(); } catch { /* already gone */ }
// This describe also opens backends directly (heartbeat test) — close
// them or Windows locks queue.db (EBUSY) at tmpdir teardown.
for (const b of openBackends.splice(0)) b.close();
rmSync(root, { recursive: true, force: true });
});
it('persists jobs before ack and dedupes identical pending sends', async () => {
mockedStreamText.mockClear();
mockedGenerateText.mockReset();
mockedGenerateText.mockResolvedValue({ text: 'ok', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);
store.create({ id: 'alpha', name: 'Alpha' });
const first = manager.enqueue('alpha', { trigger: 'chat', prompt: 'same task' });
expect(first.accepted).toBe(true);
// The durable queue holds the job (pending or already claimed by the turn)
const counts = manager.queue.counts();
expect(counts.pending + counts.claimed).toBe(1);
await new Promise(r => setTimeout(r, 10));
// Duplicate send while the first is still in flight resolves to the same job
const dup = manager.enqueue('alpha', { trigger: 'chat', prompt: 'same task' });
if (dup.jobId && dup.jobId === first.jobId) {
expect(dup.jobId).toBe(first.jobId);
}
});
it('permanent failures land in the DLQ and are replayable', async () => {
mockedStreamText.mockClear();
mockedGenerateText.mockReset();
mockedGenerateText.mockRejectedValue(new Error('permission denied: workspace'));
store.create({ id: 'alpha', name: 'Alpha' });
manager.enqueue('alpha', { trigger: 'chat', prompt: 'do the thing' });
await new Promise(r => setTimeout(r, 50));
const dlq = manager.getDlq('alpha');
expect(dlq).toHaveLength(1);
expect(dlq[0].reasonCode).toBe('permission_denied');
// Replay removes from DLQ and re-enqueues
mockedGenerateText.mockResolvedValue({ text: 'done', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);
const replay = manager.replayDlq('alpha', dlq[0].id);
expect(replay.accepted).toBe(true);
expect(manager.getDlq('alpha')).toHaveLength(0);
});
it('replaying a job under the WRONG bot id does not destroy the DLQ entry', async () => {
// The old replayDlq removed the entry BEFORE checking the bot — a
// mismatch permanently deleted real work and still reported not_found.
mockedStreamText.mockClear();
mockedGenerateText.mockReset();
mockedGenerateText.mockRejectedValue(new Error('permission denied: workspace'));
store.create({ id: 'alpha', name: 'Alpha' });
manager.enqueue('alpha', { trigger: 'chat', prompt: 'do the thing' });
await new Promise(r => setTimeout(r, 50));
const jobId = manager.getDlq('alpha')[0].id;
expect(manager.replayDlq('other-bot', jobId)).toMatchObject({ accepted: false, reasonCode: 'not_found' });
expect(manager.getDlq('alpha')).toHaveLength(1); // still there — lookup happened before removal
// A correct-bot replay then succeeds.
mockedGenerateText.mockResolvedValue({ text: 'done', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);
expect(manager.replayDlq('alpha', jobId).accepted).toBe(true);
});
it('a heartbeat keeps a claimed lease alive past its lease window', () => {
// Live bug: a 20-minute leader-bot turn outlived the 60s lease, the
// due-sweep requeued the STILL-RUNNING job, and the race made its later
// settle a silent no-op — the job never reached the DLQ.
for (const name of ['json', 'sqlite'] as const) {
const dir = join(root, `heartbeat-${name}`);
const backend = mkBackend(name, dir);
backend.enqueue(job('hb1'));
backend.claim('hb1', 60);
backend.heartbeatLease('hb1', 60); // the turn is alive — lease window reset
// Expire the PRE-heartbeat lease but not the heartbeated one: the
// heartbeat renewed the whole window, so at +1s it is still claimed.
expect(backend.counts().claimed).toBe(1);
expect(backend.rehydratable()).toHaveLength(0);
backend.heartbeatLease('hb1', 0); // heartbeat to an instant-expiry lease → swept again
expect(backend.rehydratable()).toHaveLength(1);
}
});
});
describe('idempotencyKeyFor with files', () => {
let dir = '';
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'mercury-idem-')); });
afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
const upload = (name: string, bytes: string) => {
const path = join(dir, name);
writeFileSync(path, bytes);
return { kind: 'image' as const, path, mime: 'image/png', name: 'photo.png' };
};
it('keys a retried upload on its bytes, not its fresh random path', () => {
const first = upload('1111-photo.png', 'same pixels');
const retry = upload('2222-photo.png', 'same pixels');
expect(idempotencyKeyFor('bot', 'api', 'what is this?', undefined, [retry]))
.toBe(idempotencyKeyFor('bot', 'api', 'what is this?', undefined, [first]));
});
it('still tells two different photos apart', () => {
const a = upload('a-photo.png', 'cat');
const b = upload('b-photo.png', 'dog');
expect(idempotencyKeyFor('bot', 'api', 'what is this?', undefined, [a]))
.not.toBe(idempotencyKeyFor('bot', 'api', 'what is this?', undefined, [b]));
});
});