Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.
fix(bots): release the SQLite queue handle on teardown (Windows EBUSY)
Salman Qureshi99da0ea8e2bc
diff --git a/src/bots/bot-manager.test.ts b/src/bots/bot-manager.test.tsindex 204ada08..dc7daadb 100644--- a/src/bots/bot-manager.test.ts+++ b/src/bots/bot-manager.test.ts@@ -51,13 +51,15 @@ function makeManager(root: string, overrides: Partial<MercuryConfig> = {}): BotMconst config = getDefaultConfig() as MercuryConfig;config.bots.maxConcurrent = 4;Object.assign(config, overrides);- return new BotManager({+ const manager = new BotManager({config,providers: providersRegistry,tokenBudget,store: new BotStore(join(root, 'bots')),userMemoryFactory: () => null, // no SQLite dependency in unit tests});+ activeManagers.push(manager);+ return manager;}function seedBot(store: BotStore, id: string, manifestOverrides: Partial<BotManifest> = {}) {@@ -68,6 +70,10 @@ beforeEach(() => {mockedGenerateText.mockReset();});+// Windows EBUSY guard: every BotManager owns an open SQLite queue handle;+// afterEach disposes all of them before the tmpdir is deleted.+const activeManagers: Array<{ dispose: () => void }> = [];+describe('BotManager queue + turn lifecycle', () => {let root: string;let store: BotStore;@@ -80,6 +86,7 @@ describe('BotManager queue + turn lifecycle', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -245,6 +252,7 @@ describe('BotManager queue + turn lifecycle', () => {store: new BotStore(join(root, 'bots')),userMemoryFactory: () => { throw new Error('better-sqlite3 is not available'); },});+ activeManagers.push(manager);mockedGenerateText.mockResolvedValue({ text: 'stateless ok', finishReason: 'stop', usage: { inputTokens: 1, outputTokens: 1 } } as any);seedBot(store, 'nodb');manager.enqueue('nodb', { trigger: 'chat', prompt: 'hello' });@@ -268,6 +276,7 @@ describe('Main-agent bots awareness (system prompt section)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -329,6 +338,7 @@ describe('BotManager mailboxes (bot-to-bot comms)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -375,6 +385,7 @@ describe('Per-bot permission isolation (fail-closed)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -673,6 +684,7 @@ describe('bot_send tool scoping', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -710,6 +722,7 @@ describe('Bot skill access (native + own library)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -750,6 +763,7 @@ describe('Bot fleets (lead + crew)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/bot-manager.ts b/src/bots/bot-manager.tsindex 53fd6145..b76a3c50 100644--- a/src/bots/bot-manager.ts+++ b/src/bots/bot-manager.ts@@ -160,6 +160,8 @@ export class BotManager {/** Per-bot daily token usage: botId → { day (UTC yyyy-mm-dd), tokens }. */private dailyTokens: Map<string, { day: string; tokens: number }> = new Map();private pausedForBudget = new Set<string>();+ /** Periodic due-sweep interval — cleared by dispose() (tests, shutdown). */+ private dueTimer?: ReturnType<typeof setInterval>;constructor(deps: BotManagerDeps) {this.config = deps.config;@@ -212,8 +214,8 @@ export class BotManager {};// Periodic due-sweep: retry-backoff jobs re-enter the in-memory queues// when their run_after elapses (also covers crash-restart backoffs).- const dueTimer = setInterval(() => this.resumeDueJobs(), 30_000);- dueTimer.unref?.();+ this.dueTimer = setInterval(() => this.resumeDueJobs(), 30_000);+ this.dueTimer.unref?.();// Real-time activity consumer registration happens via onBotActivity();// executeTurn feeds every registered listener AND the live `activity`// map (rendered by the /bots roster and fleet_status).@@ -1220,6 +1222,20 @@ export class BotManager {- The dispatch_bot tool lets you hand a task to a bot mid-conversation and continue talking; the result is delivered when the bot finishes.`);return lines.join('\n');}++ /**+ * Release timers and native queue handles. Idempotent — safe to call twice.+ * Teardown MUST run this before deleting the bots root: on Windows an open+ * SQLite handle keeps queue.db locked and its directory impossible to+ * remove (EBUSY). Wired into the process shutdown and every test teardown.+ */+ dispose(): void {+ if (this.dueTimer) {+ clearInterval(this.dueTimer);+ this.dueTimer = undefined;+ }+ this.queue.close();+ }}function resolveProvider(providers: ProviderRegistry, manifest: BotManifest) {diff --git a/src/bots/fleet-delegation.test.ts b/src/bots/fleet-delegation.test.tsindex cdcaef90..baeb9740 100644--- a/src/bots/fleet-delegation.test.ts+++ b/src/bots/fleet-delegation.test.ts@@ -20,7 +20,7 @@ const mockedGenerateText = vi.mocked(generateText);function makeManager(root: string): BotManager {const config = getDefaultConfig() as MercuryConfig;config.bots.maxConcurrent = 4;- return new BotManager({+ const manager = new BotManager({config,providers: {get: () => undefined,@@ -35,12 +35,18 @@ function makeManager(root: string): BotManager {store: new BotStore(join(root, 'bots')),userMemoryFactory: () => null,});+ activeManagers.push(manager);+ return manager;}beforeEach(() => {mockedGenerateText.mockReset();});+// Windows EBUSY guard: every BotManager owns an open SQLite queue handle;+// afterEach disposes all of them before the tmpdir is deleted.+const activeManagers: Array<{ dispose: () => void }> = [];+describe('fleet delegation lights up crew status', () => {let root: string;let manager: BotManager;@@ -53,6 +59,7 @@ describe('fleet delegation lights up crew status', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/fleet-layout.test.ts b/src/bots/fleet-layout.test.tsindex 37c32f66..70264ede 100644--- a/src/bots/fleet-layout.test.ts+++ b/src/bots/fleet-layout.test.ts@@ -15,7 +15,7 @@ vi.mock('ai', async (importOriginal) => {function makeManager(root: string): BotManager {const config = getDefaultConfig() as MercuryConfig;config.bots.maxConcurrent = 4;- return new BotManager({+ const manager = new BotManager({config,providers: {get: () => undefined,@@ -30,8 +30,14 @@ function makeManager(root: string): BotManager {store: new Store(join(root, 'bots')),userMemoryFactory: () => null,});+ activeManagers.push(manager);+ return manager;}+// Windows EBUSY guard: every BotManager owns an open SQLite queue handle;+// afterEach disposes all of them before the tmpdir is deleted.+const activeManagers: Array<{ dispose: () => void }> = [];+describe('fleet physical layout (crew nests under the lead)', () => {let root: string;let store: BotStore;@@ -42,6 +48,7 @@ describe('fleet physical layout (crew nests under the lead)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -190,6 +197,7 @@ describe('fleet cascade delete (manager lifecycle)', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/fleet-stop-start.test.ts b/src/bots/fleet-stop-start.test.tsindex c7aa63b4..2d87c651 100644--- a/src/bots/fleet-stop-start.test.ts+++ b/src/bots/fleet-stop-start.test.ts@@ -19,7 +19,7 @@ const mockedGenerateText = vi.mocked(generateText);function makeManager(root: string): BotManager {const config = getDefaultConfig() as MercuryConfig;config.bots.maxConcurrent = 4;- return new BotManager({+ const manager = new BotManager({config,providers: {get: () => undefined,@@ -34,12 +34,18 @@ function makeManager(root: string): BotManager {store: new BotStore(join(root, 'bots')),userMemoryFactory: () => null,});+ activeManagers.push(manager);+ return manager;}beforeEach(() => {mockedGenerateText.mockReset();});+// Windows EBUSY guard: every BotManager owns an open SQLite queue handle;+// afterEach disposes all of them before the tmpdir is deleted.+const activeManagers: Array<{ dispose: () => void }> = [];+describe('fleet stop/start cascade', () => {let root: string;let manager: BotManager;@@ -57,6 +63,7 @@ describe('fleet stop/start cascade', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/permissions-single-source.test.ts b/src/bots/permissions-single-source.test.tsindex 340a9969..8c77816d 100644--- a/src/bots/permissions-single-source.test.ts+++ b/src/bots/permissions-single-source.test.ts@@ -19,7 +19,7 @@ import { getDefaultConfig, type MercuryConfig } from '../utils/config.js';function makeManager(root: string): BotManager {const config = getDefaultConfig() as MercuryConfig;config.bots.maxConcurrent = 4;- return new BotManager({+ const manager = new BotManager({config,providers: {get: () => undefined,@@ -34,8 +34,14 @@ function makeManager(root: string): BotManager {store: new BotStore(join(root, 'bots')),userMemoryFactory: () => null,});+ activeManagers.push(manager);+ return manager;}+// Windows EBUSY guard: every BotManager owns an open SQLite queue handle;+// afterEach disposes all of them before the tmpdir is deleted.+const activeManagers: Array<{ dispose: () => void }> = [];+describe('permissions.yaml — single source of truth', () => {let root: string;let store: BotStore;@@ -48,6 +54,7 @@ describe('permissions.yaml — single source of truth', () => {});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});@@ -121,6 +128,7 @@ describe('fleet permission inheritance (crew runs on the lead\'s permissions)',});afterEach(() => {+ for (const m of activeManagers.splice(0)) m.dispose();rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/queue.test.ts b/src/bots/queue.test.tsindex 7e888bed..6f1e2896 100644--- a/src/bots/queue.test.ts+++ b/src/bots/queue.test.ts@@ -34,6 +34,18 @@ function makeBackends(dir: string) {};}+// Every backend opened by a test is registered here and closed in afterEach —+// on Windows an open SQLite handle keeps queue.db locked (EBUSY) and makes+// the tmpdir teardown fail.+const openBackends: Array<{ close(): void }> = [];+function mkBackend(name: 'json' | 'sqlite', dir: string, dlqCap = 100) {+ const backend = name === 'json'+ ? new JsonFileQueueBackend(dir, dlqCap)+ : new SqliteQueueBackend(dir, dlqCap, require('better-sqlite3'));+ openBackends.push(backend);+ return backend;+}+// `require` equivalent for the test (ESM)import { createRequire } from 'node:module';const require = createRequire(import.meta.url);@@ -46,16 +58,17 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});afterEach(() => {+ for (const b of openBackends.splice(0)) b.close();rmSync(root, { recursive: true, force: true });});for (const name of ['json', 'sqlite'] as const) {describe(`${name} backend`, () => {it('enqueues pending and persists across restart', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('a1'));// Simulate a restart by re-reading from disk with a fresh instance- const fresh = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const fresh = mkBackend(name, root);const resumed = fresh.rehydratable();expect(resumed).toHaveLength(1);expect(resumed[0].id).toBe('a1');@@ -63,7 +76,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('deduplicates on the idempotency key while pending', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);const first = backend.enqueue(job('a1', { prompt: 'same', idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'same') }));const second = backend.enqueue(job('a2', { prompt: 'same', idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'same') }));expect(first.duplicated).toBe(false);@@ -73,7 +86,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('claim sets a lease (in-flight, not resumable); settle done removes the job', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('a1'));backend.claim('a1', LEASE_SECONDS);// Fresh lease = in-flight: not resumable, but visible in counts.@@ -85,7 +98,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('pendingJobs returns only that bot’s pending work, ignoring retry backoff', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('h1'));backend.enqueue(job('h2', { botId: 'other' }));backend.enqueue(job('backoff', { idempotencyKey: idempotencyKeyFor('researcher', 'chat', 'task backoff') }));@@ -95,7 +108,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('settle dead moves the job to the DLQ with the reason', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('a1'));backend.settle('a1', 'dead', 'provider_auth');expect(backend.rehydratable()).toHaveLength(0);@@ -106,7 +119,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('expired leases requeue to pending; fresh leases survive in-flight', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('expired'));backend.enqueue(job('alive'));backend.claim('expired', 0); // expires immediately@@ -120,7 +133,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('DLQ is capped with oldest-first eviction', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 3) : new SqliteQueueBackend(root, 3, require('better-sqlite3'));+ const backend = mkBackend(name, root, 3);for (let i = 0; i < 5; i++) {backend.enqueue(job(`d${i}`, { createdAt: i }));backend.settle(`d${i}`, 'dead', 'test');@@ -132,7 +145,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('removeFromDlq returns the entry and clears it for replay', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('a1'));backend.settle('a1', 'dead', 'provider_rate_limit');const removed = backend.removeFromDlq('a1');@@ -143,7 +156,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {});it('counts reflect live state', () => {- const backend = name === 'json' ? new JsonFileQueueBackend(root, 100) : new SqliteQueueBackend(root, 100, require('better-sqlite3'));+ const backend = mkBackend(name, root);backend.enqueue(job('a1'));backend.enqueue(job('a2'));backend.claim('a1', 3600);@@ -155,6 +168,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {it('BotQueue facade resumes expired leases and pending jobs', () => {const dir = join(root, 'bots');const queue = new BotQueue(dir, 100);+ openBackends.push(queue);queue.enqueue(job('r1'));queue.enqueue(job('r2'));queue.claim('r1', 0);@@ -166,7 +180,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {it('retry requeues the same job in place — no settle-then-reenqueue window', () => {for (const name of ['json', 'sqlite'] as const) {const dir = join(root, `retry-${name}`);- const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ const backend = mkBackend(name, dir);backend.enqueue(job('t1'));backend.claim('t1', LEASE_SECONDS);// Crash-safe retry: attempts bump + backoff, still durably pending.@@ -174,7 +188,7 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {expect(backend.rehydratable()).toHaveLength(0); // backoff not elapsedexpect(backend.dueJobs()).toHaveLength(0);// A restart (fresh instance over the same storage) still sees the job- const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ const fresh = mkBackend(name, dir);expect(fresh.rehydratable().length).toBeGreaterThanOrEqual(0);expect(fresh.counts().pending).toBe(1);}@@ -183,10 +197,10 @@ describe('BotQueue backends (JSON-file + SQLite semantics)', () => {it('durable mail survives a restart and drains once', () => {for (const name of ['json', 'sqlite'] as const) {const dir = join(root, `mail-${name}`);- const backend = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ const backend = mkBackend(name, dir);backend.enqueueMail({ botId: 'publisher', from: 'researcher', content: 'findings here', createdAt: Date.now() });// Restart: the mail is still there- const fresh = name === 'json' ? new JsonFileQueueBackend(dir, 100) : new SqliteQueueBackend(dir, 100, require('better-sqlite3'));+ const fresh = mkBackend(name, dir);const drained = fresh.drainMail('publisher');expect(drained).toHaveLength(1);expect(drained[0].from).toBe('researcher');@@ -231,6 +245,9 @@ describe('BotManager durable queue integration', () => {});afterEach(() => {+ // The manager owns the SQLite queue handle — close it or Windows locks+ // queue.db (EBUSY) and the tmpdir teardown fails.+ try { manager?.dispose?.(); } catch { /* already gone */ }rmSync(root, { recursive: true, force: true });});diff --git a/src/bots/queue.ts b/src/bots/queue.tsindex abfbedb0..f7d05464 100644--- a/src/bots/queue.ts+++ b/src/bots/queue.ts@@ -99,6 +99,13 @@ export interface BotQueueBackend {/** Remove every trace of a bot: jobs, mail, DLQ rows (delete lifecycle). */purgeBot(botId: string): void;counts(): QueueCounts;+ /**+ * Release native handles (the SQLite connection). Idempotent — a second+ * close is a no-op. On Windows an open handle keeps the file locked+ * (EBUSY on delete): teardown, fleet-delete cleanup, and shutdown must+ * close before any directory removal.+ */+ close(): void;}export const LEASE_SECONDS = 60;@@ -151,6 +158,11 @@ export class BotQueue {logger.info({ backend: this.backend.name, path }, 'Bot queue initialized');}+ /** Release native handles (the SQLite connection). Idempotent. */+ close(): void {+ this.backend.close();+ }+enqueue(job: Omit<DurableBotJob, 'state'>): { job: DurableBotJob; duplicated: boolean } {return this.backend.enqueue(job);}@@ -409,6 +421,15 @@ export class SqliteQueueBackend implements BotQueueBackend {const dlq = (this.db.prepare(`SELECT COUNT(*) c FROM bot_dlq`).get() as any).c as number;return { pending, claimed, dlq };}++ /** WAL checkpoints on close, releasing the -wal/-shm sidecars too. Idempotent. */+ close(): void {+ try {+ this.db.close();+ } catch {+ // Already closed (double teardown must never throw).+ }+ }}// Minimal structural type for the better-sqlite3 API we use (avoids pulling@@ -442,6 +463,9 @@ export class JsonFileQueueBackend implements BotQueueBackend {private data: QueueFile;private dlqCap: number;+ /** Writes are fd-opened-and-closed per op (write-to-rename + fsync) — nothing to release. */+ close(): void { /* no native handles */ }+constructor(dir: string, dlqCap: number) {mkdirSync(dir, { recursive: true });this.file = join(dir, QUEUE_FILE);diff --git a/src/index.ts b/src/index.tsindex 14df370d..13b4becf 100644--- a/src/index.ts+++ b/src/index.ts@@ -2543,8 +2543,9 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {}// Mercury Bots: fleet manager runs outside the main message queue.+ let botManager: BotManager | undefined;if (config.bots?.enabled) {- const botManager = new BotManager({+ const bot = new BotManager({config,providers,tokenBudget,@@ -2556,21 +2557,22 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {return new UserMemoryStore(config, `bot:${manifest.id}`);},});- botManager.setScheduler(scheduler);+ botManager = bot;+ bot.setScheduler(scheduler);// Startup migrations must run before routines register: fleet layout// (crew nesting + orphan cascade) and permission consolidation to the// single permissions.yaml source.- void botManager.migrateFleetLayout()- .then(() => botManager.migratePermissions())- .then(() => botManager.registerRoutines(scheduler));- agent.setBotManager(botManager);- setWebBotManager(botManager);+ void bot.migrateFleetLayout()+ .then(() => bot.migratePermissions())+ .then(() => bot.registerRoutines(scheduler));+ agent.setBotManager(bot);+ setWebBotManager(bot);// Real-time bot activity bus → surfaces: the CLI bot-thread live region// (what the bot is doing RIGHT NOW while you watch its thread) and the// web SSE feed (dashboards / third-party backends). The roster's activity// label is updated inside the manager itself.const botCliChannel = channels.get('cli');- botManager.onBotActivity((ev) => {+ bot.onBotActivity((ev) => {if (botCliChannel instanceof CLIChannel) botCliChannel.setBotLiveActivity(ev.botId, ev);webChannel.broadcastBotActivity(ev);});@@ -2579,9 +2581,9 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {}// Main-agent awareness: bots section in the system prompt + dispatch tool.capabilities.setBotDispatchHandler((botIdOrName, message, ctx) => {- const botId = botManager.resolveBotId(botIdOrName);+ const botId = bot.resolveBotId(botIdOrName);if (!botId) return { accepted: false, reasonCode: 'target_unknown' };- const result = botManager.enqueue(botId, {+ const result = bot.enqueue(botId, {trigger: 'chat',prompt: message,source: { channelType: ctx.channelType, channelId: ctx.channelId },@@ -3609,6 +3611,9 @@ async function runAgent(isDaemon: boolean = false): Promise<void> {} catch {}}await stopWebServer();+ // Release the bots' SQLite queue handle — on Windows an open handle+ // keeps queue.db locked against any later uninstall/replace.+ try { botManager?.dispose(); } catch { /* best effort */ }await agent.shutdown();clearCloudRuntimeOnline();releaseRuntimeProcess(runtimeMode);