Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): bound db worker progress instead of write residence
Alfredo Lopezdc1a299be92b
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex edd790f5e..5ae4f745d 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.574",+ "version": "0.1.575","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.574",+ "version": "0.1.575","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 6fd48fd24..7c1cbc29d 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.574",+ "version": "0.1.575","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.ts b/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.tsnew file mode 100644index 000000000..ed205b15c--- /dev/null+++ b/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.ts@@ -0,0 +1,26 @@+import { describe, expect, it } from 'vitest'+import { parseDbWorkerStallTimeoutMs } from '../config'++describe('CLERUM_DB_WORKER_STALL_TIMEOUT_MS', () => {+ it('defaults to 30000 when unset or blank', () => {+ expect(parseDbWorkerStallTimeoutMs(undefined, 2000)).toBe(30_000)+ expect(parseDbWorkerStallTimeoutMs(' ', 2000)).toBe(30_000)+ })++ it('accepts an integer of at least 1000 that exceeds the sync timeout', () => {+ expect(parseDbWorkerStallTimeoutMs('15000', 2000)).toBe(15_000)+ expect(parseDbWorkerStallTimeoutMs('1000', 999)).toBe(1000)+ })++ it.each(['abc', '1.5', '-1', '0', '999', '30s'])('rejects %s', raw => {+ expect(() => parseDbWorkerStallTimeoutMs(raw, 500)).toThrow(+ `CLERUM_DB_WORKER_STALL_TIMEOUT_MS='${raw}' must be an integer of at least 1000`+ )+ })++ it('rejects a value that does not exceed the sync timeout', () => {+ expect(() => parseDbWorkerStallTimeoutMs('2000', 2000)).toThrow(+ "CLERUM_DB_WORKER_STALL_TIMEOUT_MS='2000' must exceed CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS (2000)"+ )+ })+})diff --git a/mcp-host/src/config.ts b/mcp-host/src/config.tsindex 07ed07891..25133690f 100644--- a/mcp-host/src/config.ts+++ b/mcp-host/src/config.ts@@ -233,6 +233,7 @@ export interface Config {dbWorkerHeartbeatMs: numberdbPersistSyncTimeoutMs: numberdbPersistAsyncTimeoutMs: number+ dbWorkerStallTimeoutMs: numberdbCheckpointEveryWrites: numberdbWalSizeAlarmBytes: number@@ -473,6 +474,31 @@ export function parseStatelessHeartbeatIntervalMs(raw: string | undefined): numbreturn parsed}+/**+ * Parse CLERUM_DB_WORKER_STALL_TIMEOUT_MS, the longest time the db worker may+ * go without completing an operation while it has work in flight. It must+ * exceed the sync write timeout, so a sync write times out on its own budget+ * before the worker is declared stalled. An explicit invalid value fails at+ * boot.+ */+export function parseDbWorkerStallTimeoutMs(+ raw: string | undefined,+ syncTimeoutMs: number+): number {+ if (raw === undefined || raw.trim() === '') return 30_000+ const name = 'CLERUM_DB_WORKER_STALL_TIMEOUT_MS'+ const value = /^\d+$/.test(raw) ? Number(raw) : NaN+ if (!Number.isSafeInteger(value) || value < 1000) {+ throw new Error(`${name}='${raw}' must be an integer of at least 1000`)+ }+ if (value <= syncTimeoutMs) {+ throw new Error(+ `${name}='${raw}' must exceed CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS (${syncTimeoutMs})`+ )+ }+ return value+}+/*** Parse an MCP status-heartbeat duration. These values participate in the* liveness scheduler, so an explicit typo must fail at boot rather than turn@@ -1069,6 +1095,10 @@ export const config: Config = {dbWorkerHeartbeatMs: parseInt(getEnv('CLERUM_DB_WORKER_HEARTBEAT_MS', '5000')!, 10),dbPersistSyncTimeoutMs: parseInt(getEnv('CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS', '2000')!, 10),dbPersistAsyncTimeoutMs: parseInt(getEnv('CLERUM_DB_PERSIST_ASYNC_TIMEOUT_MS', '5000')!, 10),+ dbWorkerStallTimeoutMs: parseDbWorkerStallTimeoutMs(+ getEnv('CLERUM_DB_WORKER_STALL_TIMEOUT_MS', ''),+ parseInt(getEnv('CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS', '2000')!, 10)+ ),dbCheckpointEveryWrites: parseInt(getEnv('CLERUM_DB_CHECKPOINT_EVERY_WRITES', '100')!, 10),dbWalSizeAlarmBytes: parseInt(getEnv('CLERUM_DB_WAL_SIZE_ALARM_BYTES', '52428800')!, 10),diff --git a/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.tsindex a133b5388..ba00dc84a 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts@@ -26,6 +26,7 @@ function setup() {const queue = new PersistQueue(worker, {syncTimeoutMs: 2000,asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,onFailure,})const manager = new ConversationManager(new SqliteConversationStore(queue, { cacheSize: 8 }))diff --git a/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.tsindex df236a599..74ecde176 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts@@ -47,6 +47,7 @@ describe('createConversationStore persist failure', () => {cacheSize: 10,syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -86,6 +87,7 @@ describe('createConversationStore persist failure', () => {cacheSize: 10,syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -128,6 +130,7 @@ describe('createConversationStore store identity', () => {cacheSize: 10,syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -173,6 +176,7 @@ describe('createConversationStore store identity', () => {cacheSize: 10,syncTimeoutMs: 5000,asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -232,6 +236,7 @@ describe('createConversationStore store identity', () => {cacheSize: 10,syncTimeoutMs: 100,asyncTimeoutMs: 100,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -267,6 +272,7 @@ describe('createConversationStore store identity', () => {cacheSize: 10,syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,workerScriptPath: script,@@ -300,6 +306,7 @@ describe('createConversationStore store identity', () => {cacheSize: 10,syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,checkpointEveryWrites: 100,heartbeatMs: 60_000,})diff --git a/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.tsindex afb4002f8..140643e74 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts@@ -241,7 +241,11 @@ describe('D3 durability barrier', () => {it('a worker failure applies NEITHER the assistant message NOR the active_task_id clear', async () => {const raw = createInProcessWorker(tmpDbPath())const faulty = new FaultInjectingWorker(raw.worker)- const queue = new PersistQueue(faulty, { syncTimeoutMs: 2000, asyncTimeoutMs: 5000 })+ const queue = new PersistQueue(faulty, {+ syncTimeoutMs: 2000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,+ })const store = new SqliteConversationStore(queue, { cacheSize: 8 })const manager = new ConversationManager(store)@@ -263,7 +267,11 @@ describe('D3 durability barrier', () => {it('a worker failure on turn-start applies NOTHING and rolls back the in-RAM turn', async () => {const raw = createInProcessWorker(tmpDbPath())const faulty = new FaultInjectingWorker(raw.worker)- const queue = new PersistQueue(faulty, { syncTimeoutMs: 2000, asyncTimeoutMs: 5000 })+ const queue = new PersistQueue(faulty, {+ syncTimeoutMs: 2000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,+ })const store = new SqliteConversationStore(queue, { cacheSize: 8 })const manager = new ConversationManager(store)@@ -382,7 +390,11 @@ describe('D3 durability barrier', () => {const dbPath = tmpDbPath()const raw = createInProcessWorker(dbPath, { barrierMode: true })const faulty = new FaultInjectingWorker(raw.worker)- const queue = new PersistQueue(faulty, { syncTimeoutMs: 500, asyncTimeoutMs: 5000 })+ const queue = new PersistQueue(faulty, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,+ })const store = new SqliteConversationStore(queue, { cacheSize: 8 })const manager = new ConversationManager(store)diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.tsindex 100ebab20..7258b1032 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts@@ -3,12 +3,13 @@ import { spawnSync } from 'node:child_process'import * as fs from 'node:fs'import * as os from 'node:os'import * as path from 'node:path'-import { exitOnPersistFailure } from '../persistFailureExit'+import { exitOnPersistFailure, storeShutdownExitCode } from '../persistFailureExit'// Runs the real factory, PersistQueue and exitOnPersistFailure in a child// Node process and reads that process's exit code. The worker thread is a-// JS stand-in: it acknowledges writes by appending them to a file, fails the-// `fail` op and exits on `crash`.+// JS stand-in: it answers the open ping, acknowledges writes by appending them+// to a file (`slow` after 30 ms), fails the `fail` op, never answers `hang`+// and exits on `crash`.const SRC = path.resolve(__dirname, '..', '..', '..', '..')const HOOK = path.join(__dirname, 'fixtures', 'tsResolveHook.cjs')@@ -23,6 +24,13 @@ const fs = require('node:fs')const { parentPort, workerData } = require('node:worker_threads')parentPort.on('message', msg => {const kind = msg.op.kind+ if (kind === 'ping') parentPort.postMessage({ id: msg.id, ok: true, result: null })+ if (kind === 'slow') {+ setTimeout(() => {+ fs.appendFileSync(workerData.dbPath, msg.op.value + '\\n')+ parentPort.postMessage({ id: msg.id, ok: true, result: null })+ }, 30)+ }if (kind === 'write') {fs.appendFileSync(workerData.dbPath, msg.op.value + '\\n')parentPort.postMessage({ id: msg.id, ok: true, result: null })@@ -44,17 +52,25 @@ const path = require('node:path')const [srcRoot, workerPath, dataPath, scenario] = process.argv.slice(2)const persistence = path.join(srcRoot, 'core/conversation/persistence')const { createConversationStore } = require(path.join(persistence, 'conversationStoreFactory.ts'))-const { exitOnPersistFailure } = require(path.join(persistence, 'persistFailureExit.ts'))+const { exitOnPersistFailure, storeShutdownExitCode } = require(path.join(persistence, 'persistFailureExit.ts'))+// Mirrors main.ts: the graceful path marks the Host as shutting down, closes+// the store and exits with the code storeShutdownExitCode returns.+let shuttingDown = false+async function gracefulExit() {+ shuttingDown = true+ process.exit(await storeShutdownExitCode(() => handle.shutdown()))+}const handle = createConversationStore({mode: 'sqlite',dbPath: dataPath,cacheSize: 10,syncTimeoutMs: 5000,- asyncTimeoutMs: 5000,+ asyncTimeoutMs: 50,+ stallTimeoutMs: 300,checkpointEveryWrites: 100,heartbeatMs: 60000,workerScriptPath: workerPath,- onPersistFailure: exitOnPersistFailure(() => false, {+ onPersistFailure: exitOnPersistFailure(() => shuttingDown, {beforeExit: async () => console.log('PRE_EXIT_FLUSH'),beforeExitTimeoutMs: 1000,}),@@ -69,6 +85,24 @@ const queue = handle.persistQueueconsole.log('SHUTDOWN_CLEAN')return}+ if (scenario === 'shutdown-slow-backlog') {+ for (let i = 0; i < 20; i++) queue.enqueueAsync('s1', { kind: 'slow', value: 'slow' + i })+ await gracefulExit()+ return+ }+ if (scenario === 'shutdown-wedged') {+ queue.enqueueAsync('s1', { kind: 'hang' })+ await gracefulExit()+ return+ }+ if (scenario === 'wedged') {+ queue.enqueueAsync('s1', { kind: 'hang' })+ setTimeout(() => {+ console.log('STILL_RUNNING')+ process.exit(0)+ }, 5000)+ return+ }queue.enqueueAsync('s1', { kind: scenario === 'async-write-failure' ? 'fail' : 'crash' })setTimeout(() => {console.log('STILL_RUNNING')@@ -77,7 +111,15 @@ const queue = handle.persistQueue})()`-function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {+type Scenario =+ | 'alive'+ | 'async-write-failure'+ | 'worker-exit'+ | 'wedged'+ | 'shutdown-slow-backlog'+ | 'shutdown-wedged'++function runChild(scenario: Scenario) {const root = fs.mkdtempSync(path.join(os.tmpdir(), 'persist-failure-exit-'))roots.push(root)const worker = path.join(root, 'worker.cjs')@@ -85,6 +127,7 @@ function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {const data = path.join(root, 'writes.log')fs.writeFileSync(worker, WORKER)fs.writeFileSync(child, CHILD)+ const started = Date.now()const result = spawnSync(process.execPath,[@@ -100,10 +143,11 @@ function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {],{ encoding: 'utf8', timeout: 20_000 })+ const elapsedMs = Date.now() - startedconst writes = fs.existsSync(data)? fs.readFileSync(data, 'utf8').split('\n').filter(Boolean): []- return { status: result.status, out: `${result.stdout}${result.stderr}`, writes }+ return { status: result.status, out: `${result.stdout}${result.stderr}`, writes, elapsedMs }}describe('exitOnPersistFailure in a Host process', () => {@@ -132,6 +176,51 @@ describe('exitOnPersistFailure in a Host process', () => {expect(run.out).not.toContain('STILL_RUNNING')expect(run.status).toBe(1)})++ it('exits 1, bounded, when the worker stays alive but stops answering', () => {+ const run = runChild('wedged')+ expect(run.out).toContain('FIRST_WRITE_ACKED')+ expect(run.out).toContain('db worker made no progress')+ expect(run.out).toContain('"stallTimeoutMs":300')+ expect(run.out).toContain('PRE_EXIT_FLUSH')+ expect(run.out).not.toContain('STILL_RUNNING')+ expect(run.status).toBe(1)+ // 300 ms bound + 1 s flush deadline + process start-up.+ expect(run.elapsedMs).toBeLessThan(4000)+ })+})++describe('Host process exit on a graceful shutdown', () => {+ it('drains a slow backlog longer than asyncTimeoutMs and exits 0', () => {+ const run = runChild('shutdown-slow-backlog')+ expect(run.out).toContain('FIRST_WRITE_ACKED')+ // Liveness witness: every queued write reached the store.+ expect(run.writes).toEqual(['first', ...Array.from({ length: 20 }, (_, i) => `slow${i}`)])+ expect(run.out).not.toContain('ConversationStore shutdown failed')+ expect(run.status).toBe(0)+ })++ it('exits 1 when the store cannot drain because the worker is wedged', () => {+ const run = runChild('shutdown-wedged')+ expect(run.out).toContain('FIRST_WRITE_ACKED')+ expect(run.out).toContain('ConversationStore shutdown failed; exiting with code 1')+ expect(run.out).toContain('db worker made no progress')+ expect(run.out).toContain('"stallTimeoutMs":300')+ expect(run.status).toBe(1)+ expect(run.elapsedMs).toBeLessThan(4000)+ })+})++describe('storeShutdownExitCode', () => {+ it('returns 0 when the store closes and 1 when closing rejects', async () => {+ // Liveness witness: a clean close is 0, so the 1 below comes from the rejection.+ await expect(storeShutdownExitCode(async () => undefined)).resolves.toBe(0)+ await expect(+ storeShutdownExitCode(async () => {+ throw new Error('async write failed (sessionKey=s1): disk I/O error')+ })+ ).resolves.toBe(1)+ })})describe('exitOnPersistFailure during shutdown', () => {diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.tsindex 80365eebe..7211bd0c7 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts@@ -1,6 +1,7 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import { EventEmitter } from 'node:events'-import type { WorkerMessage, WorkerReply } from '../../../../db/worker/protocol'+import { Worker } from 'node:worker_threads'+import type { WorkerMessage, WorkerOp, WorkerReply } from '../../../../db/worker/protocol'import { PersistQueue, type WorkerLike } from '../persistQueue'class FakeWorker extends EventEmitter implements WorkerLike {@@ -23,7 +24,11 @@ class FakeWorker extends EventEmitter implements WorkerLike {describe('PersistQueue', () => {it('resolves enqueueSync when worker replies ok', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 200,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })const promise = q.enqueueSync({ kind: 'ping' })const sent = worker.sent[0]worker.emit('message', { id: sent.id, ok: true, result: 'pong' } satisfies WorkerReply)@@ -32,7 +37,11 @@ describe('PersistQueue', () => {it('rejects enqueueSync when worker replies with error', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 200,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })const promise = q.enqueueSync({ kind: 'ping' })const sent = worker.sent[0]worker.emit('message', {@@ -45,13 +54,21 @@ describe('PersistQueue', () => {it('times out enqueueSync', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 30,+ asyncTimeoutMs: 30,+ stallTimeoutMs: 30_000,+ })await expect(q.enqueueSync({ kind: 'ping' })).rejects.toThrow(/timeout/)})it('awaitOpen outlasts the sync timeout and resolves on the worker reply', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 30,+ asyncTimeoutMs: 30,+ stallTimeoutMs: 30_000,+ })let settled = falseconst open = q.awaitOpen().finally(() => {settled = true@@ -70,7 +87,11 @@ describe('PersistQueue', () => {it('awaitOpen rejects when the worker exits before answering', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 30,+ asyncTimeoutMs: 30,+ stallTimeoutMs: 30_000,+ })const open = q.awaitOpen()expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])worker.emit('exit', 1)@@ -79,7 +100,11 @@ describe('PersistQueue', () => {it('enqueueAsync preserves order per sessionKey', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 200,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })q.enqueueAsync('s1', { kind: 'ping' })q.enqueueAsync('s1', { kind: 'ping' })// After microtasks the first one is sent but the second is awaiting the@@ -94,7 +119,11 @@ describe('PersistQueue', () => {it('drainPrefix waits only for chains whose sessionKey matches the prefix', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 200,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })// Two in-flight async writes under the same prefix + one unrelated key.q.enqueueAsync('alice:rpc:a:1', { kind: 'ping' })q.enqueueAsync('alice:rpc:b:2', { kind: 'ping' })@@ -120,7 +149,11 @@ describe('PersistQueue', () => {it('rejects pending writes when the worker exits', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })const promise = q.enqueueSync({ kind: 'ping' })worker.emit('exit', 137)await expect(promise).rejects.toThrow(/db worker exited/)@@ -128,7 +161,11 @@ describe('PersistQueue', () => {it('rejects async writes admitted after the worker exited instead of dropping them', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })q.enqueueAsync('s1', { kind: 'ping' })await Promise.resolve()// Liveness witness: while the worker is alive the async write is posted.@@ -141,7 +178,11 @@ describe('PersistQueue', () => {it('fails the next write on the same queue after an async write failed', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })// Witness: a sync write before any failure resolves.const first = q.enqueueSync({ kind: 'ping' })worker.emit('message', {@@ -173,6 +214,7 @@ describe('PersistQueue', () => {const q = new PersistQueue(worker, {syncTimeoutMs: 500,asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,onFailure: err => failures.push(err),onExit: code => exits.push(code),})@@ -201,6 +243,7 @@ describe('PersistQueue', () => {const q = new PersistQueue(worker, {syncTimeoutMs: 500,asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,onFailure: err => failures.push(err),})q.enqueueAsync('s1', { kind: 'ping' })@@ -238,6 +281,7 @@ describe('PersistQueue', () => {const q = new PersistQueue(worker, {syncTimeoutMs: 500,asyncTimeoutMs: 20,+ stallTimeoutMs: 30_000,onFailure: err => failures.push(err),})q.enqueueAsync('s1', { kind: 'ping' })@@ -262,28 +306,6 @@ describe('PersistQueue', () => {expect(worker.sent).toHaveLength(3)})- it('bounds the close() drain of an unacknowledged async write by asyncTimeoutMs', async () => {- const worker = new FakeWorker()- const failures: Error[] = []- const exits: number[] = []- worker.on('exit', code => exits.push(code))- const q = new PersistQueue(worker, {- syncTimeoutMs: 500,- asyncTimeoutMs: 20,- onFailure: err => failures.push(err),- })- q.enqueueAsync('s1', { kind: 'ping' })- await Promise.resolve()- expect(worker.sent).toHaveLength(1)- const started = Date.now()- await expect(q.close()).rejects.toThrow(/not acknowledged within 20ms of close/)- expect(Date.now() - started).toBeLessThan(400)- // The worker was terminated and the failure reported once.- expect(exits).toEqual([0])- expect(failures).toHaveLength(1)- expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])- })-it('reports neither onFailure nor onExit for the worker exit that follows a shutdown ACK', async () => {const worker = new FakeWorker()const failures: Error[] = []@@ -293,6 +315,7 @@ describe('PersistQueue', () => {const q = new PersistQueue(worker, {syncTimeoutMs: 500,asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,onFailure: err => failures.push(err),onExit: code => reportedExits.push(code),})@@ -315,7 +338,11 @@ describe('PersistQueue', () => {worker.terminate = () => {terminateCalls += 1}- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })const closing = q.close()await new Promise(r => setImmediate(r))@@ -331,7 +358,11 @@ describe('PersistQueue', () => {it('drains queued async writes before posting shutdown and rejects close on a failed one', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })q.enqueueAsync('s1', { kind: 'ping' })await Promise.resolve()const closing = q.close()@@ -353,7 +384,11 @@ describe('PersistQueue', () => {it('never posts a write chained behind a failed async write on the same key', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })q.enqueueAsync('s1', { kind: 'ping' })q.enqueueAsync('s1', { kind: 'ping' })const chainedSync = q.enqueueSync({ kind: 'ping' }, 's1')@@ -375,7 +410,11 @@ describe('PersistQueue', () => {it('resolves a concurrent second close only after the shutdown ACK', async () => {const worker = new FakeWorker()- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,+ })let secondSettled = falseconst first = q.close()const second = q.close().finally(() => {@@ -398,6 +437,7 @@ describe('PersistQueue', () => {const q = new PersistQueue(worker, {syncTimeoutMs: 500,asyncTimeoutMs: 500,+ stallTimeoutMs: 30_000,onFailure: err => failures.push(err),})const closing = q.close()@@ -415,3 +455,284 @@ describe('PersistQueue', () => {expect(failures.map(e => e.message)).toEqual(['database is locked'])})})++// A slow worker and a wedged worker differ in progress, not in how long a+// write waits: a slow worker keeps completing operations, a wedged one stops.+// These cases pin both, in service and at close(), so a change that fixes one+// by breaking another fails here.+describe('PersistQueue progress bound: slow vs wedged, in service and at close()', () => {+ const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))+ const ack = (worker: FakeWorker, msg: WorkerMessage) =>+ worker.emit('message', { id: msg.id, ok: true, result: null } satisfies WorkerReply)++ /** A queue whose open ping was answered, so the progress bound is armed. */+ async function openQueue(opts: {+ stallTimeoutMs: number+ asyncTimeoutMs?: number+ syncTimeoutMs?: number+ }) {+ const worker = new FakeWorker()+ const failures: Error[] = []+ let terminateCalls = 0+ worker.terminate = () => {+ terminateCalls += 1+ worker.emit('exit', 1)+ }+ const q = new PersistQueue(worker, {+ syncTimeoutMs: opts.syncTimeoutMs ?? 5000,+ asyncTimeoutMs: opts.asyncTimeoutMs ?? 5000,+ stallTimeoutMs: opts.stallTimeoutMs,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen()+ ack(worker, worker.sent[0])+ await open+ return { worker, q, failures, terminateCalls: () => terminateCalls }+ }++ /**+ * Answers every posted operation in order, one at a time, `delayMs` after+ * the previous answer: a single-writer worker that is slow but alive.+ */+ function autoAck(worker: FakeWorker, delayMs: number): WorkerMessage[] {+ const acked: WorkerMessage[] = []+ let busy = false+ const queue: WorkerMessage[] = []+ const next = () => {+ const msg = queue.shift()+ if (!msg) {+ busy = false+ return+ }+ busy = true+ setTimeout(() => {+ acked.push(msg)+ ack(worker, msg)+ next()+ }, delayMs)+ }+ const post = worker.postMessage.bind(worker)+ worker.postMessage = msg => {+ post(msg)+ queue.push(msg)+ if (!busy) next()+ }+ return acked+ }++ it('fails a wedged worker in service within the bound and releases everything queued behind it', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({+ stallTimeoutMs: 100,+ asyncTimeoutMs: 20,+ })+ // Liveness witness: a write on the same key is acknowledged first.+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ ack(worker, worker.sent[1])+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)++ // The worker stops answering.+ q.enqueueAsync('s1', { kind: 'ping' })+ const keyedSync = q.enqueueSync({ kind: 'ping' }, 's1')+ await Promise.resolve()+ expect(worker.sent).toHaveLength(3)+ const started = Date.now()+ const prefixDrained = q.drainPrefix('s')++ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(Date.now() - started).toBeLessThan(400)+ expect(failures[0].message).toMatch(/db worker made no progress for 100ms/)+ await expect(keyedSync).rejects.toBe(failures[0])+ await prefixDrained+ expect(terminateCalls()).toBe(1)+ // The write chained behind the stalled one was never posted.+ expect(worker.sent).toHaveLength(3)+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow(failures[0].message)+ })++ it('never fails a slow but alive worker with a backlog far longer than the bound', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 100, asyncTimeoutMs: 20 })+ const acked = autoAck(worker, 50)+ const keys = ['a', 'b', 'c']+ // The tag only identifies the write in this test; the queue never reads it.+ const tagged = (tag: string) => ({ kind: 'ping', tag }) as WorkerOp+ for (let i = 0; i < 10; i++) {+ for (const key of keys) q.enqueueAsync(key, tagged(`${key}${i}`))+ }+ await Promise.all(keys.map(k => q.drainSessionKey(k)))+ const tags = acked.map(m => (m.op as WorkerOp & { tag: string }).tag)+ // Liveness witness: all 30 writes were acknowledged, 1.5 s of backlog+ // against a 100 ms bound, each key in its enqueue order.+ expect(tags).toHaveLength(30)+ for (const key of keys) {+ expect(tags.filter(t => t.startsWith(key))).toEqual(+ Array.from({ length: 10 }, (_, i) => `${key}${i}`)+ )+ }+ expect(failures).toHaveLength(0)+ })++ it('drains a slow backlog completely on close() before posting shutdown', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, asyncTimeoutMs: 50 })+ const acked = autoAck(worker, 10)+ for (let i = 0; i < 10; i++) q.enqueueAsync('s1', { kind: 'ping' })+ await q.close()+ const kinds = acked.map(m => m.op.kind)+ // Liveness witness: the ten writes and then shutdown were acknowledged.+ expect(kinds).toEqual([...Array(10).fill('ping'), 'shutdown'])+ expect(failures).toHaveLength(0)+ })++ it('fails close() within the bound when the worker is wedged', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })+ // Liveness witness: a write is acknowledged before the worker stops.+ const first = q.enqueueSync({ kind: 'ping' })+ ack(worker, worker.sent[1])+ await first+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ const started = Date.now()+ await expect(q.close()).rejects.toThrow(/db worker made no progress for 100ms/)+ expect(Date.now() - started).toBeLessThan(400)+ expect(terminateCalls()).toBe(1)+ expect(failures).toHaveLength(1)+ expect(worker.sent.map(m => m.op.kind)).not.toContain('shutdown')+ })++ it('waits on close() for a sync write without a sessionKey before posting shutdown', async () => {+ const { worker, q } = await openQueue({ stallTimeoutMs: 1000 })+ const unkeyed = q.enqueueSync<string>({ kind: 'ping' })+ const closing = q.close()+ await sleep(20)+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping', 'ping'])++ worker.emit('message', {+ id: worker.sent[1].id,+ ok: true,+ result: 'pong',+ } satisfies WorkerReply)+ // Liveness witness: the sync write resolves with the worker's result.+ await expect(unkeyed).resolves.toBe('pong')+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))+ ack(worker, worker.sent[2])+ await closing+ })++ it('does not fail a healthy worker whose answer waited behind a main-thread stall', async () => {+ const realWorker = new Worker(+ `const { parentPort } = require('node:worker_threads')+ parentPort.on('message', msg => setTimeout(() => parentPort.postMessage({ id: msg.id, ok: true, result: null }), 10))`,+ { eval: true }+ )+ const failures: Error[] = []+ const q = new PersistQueue(realWorker as unknown as WorkerLike, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 100,+ onFailure: err => failures.push(err),+ })+ try {+ await q.awaitOpen()+ const write = q.enqueueSync({ kind: 'ping' }, 's1')+ await sleep(1)+ // The main thread is blocked well past the bound while the answer+ // waits in the port. Blocking in the check phase makes the next loop+ // iteration run the overdue timer before the poll phase reads the+ // answer (blocking in the timers phase would not: libuv compares+ // timers against the loop time cached when that phase started).+ await new Promise<void>(resolve =>+ setImmediate(() => {+ const until = Date.now() + 300+ while (Date.now() < until) {+ /* block the event loop */+ }+ resolve()+ })+ )+ // Liveness witness: the write resolves.+ await expect(write).resolves.toBeNull()+ await sleep(150)+ expect(failures).toHaveLength(0)+ } finally {+ await realWorker.terminate()+ }+ })++ it('does not start the bound before the database is open', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 5000,+ asyncTimeoutMs: 5000,+ stallTimeoutMs: 100,+ onFailure: err => failures.push(err),+ })+ const open = q.awaitOpen()+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(2)+ await sleep(300)+ expect(failures).toHaveLength(0)++ ack(worker, worker.sent[0])+ await open+ ack(worker, worker.sent[1])+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)++ // Liveness witness: once open, a wedged write does fail the queue.+ q.enqueueAsync('s1', { kind: 'ping' })+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(failures[0].message).toMatch(/no progress for 100ms/)+ })++ it('stays failed when the stalled operation is acknowledged late', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 100 })+ const stalled = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })+ expect(await stalled).toBe(failures[0])++ worker.emit('message', {+ id: worker.sent[1].id,+ ok: true,+ result: 'late',+ } satisfies WorkerReply)+ await sleep(1)+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(failures[0].message)+ await expect(q.enqueueSync({ kind: 'ping' })).rejects.toBe(failures[0])+ expect(failures).toHaveLength(1)+ })++ it('waits for a shutdown that takes longer than syncTimeoutMs while within the bound', async () => {+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, syncTimeoutMs: 30 })+ const closing = q.close()+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))+ await sleep(90)+ ack(worker, worker.sent.find(m => m.op.kind === 'shutdown')!)+ // Liveness witness: close() resolves after the late shutdown ACK.+ await expect(closing).resolves.toBeUndefined()+ expect(failures).toHaveLength(0)+ })++ it('keeps the bound after an earlier failure, so close() does not wait forever', async () => {+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })+ q.enqueueAsync('a', { kind: 'ping' })+ await vi.waitFor(() => expect(worker.sent).toHaveLength(2))+ const hung = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)+ expect(worker.sent).toHaveLength(3)+ worker.emit('message', {+ id: worker.sent[1].id,+ ok: false,+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },+ } satisfies WorkerReply)+ // Liveness witness: the async failure is latched first.+ await vi.waitFor(() => expect(failures).toHaveLength(1))+ const started = Date.now()+ await expect(q.close()).rejects.toThrow(/database or disk is full/)+ expect(Date.now() - started).toBeLessThan(400)+ expect(terminateCalls()).toBe(1)+ expect(await hung).toBeInstanceOf(Error)+ expect(failures).toHaveLength(1)+ })+})diff --git a/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts b/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.tsindex 7e362674c..6d46bd163 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts@@ -121,6 +121,7 @@ export function makeSqliteStore(const persistQueue = new PersistQueue(worker.worker, {syncTimeoutMs: 2000,asyncTimeoutMs: 5000,+ stallTimeoutMs: 30_000,})const store = new SqliteConversationStore(persistQueue, {cacheSize: opts.cacheSize ?? 8,diff --git a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.tsindex d44914bf0..e0079a46e 100644--- a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts+++ b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts@@ -25,6 +25,8 @@ export interface ConversationStoreFactoryOptions {cacheSize: numbersyncTimeoutMs: numberasyncTimeoutMs: number+ /** Progress bound of the persist queue (`PersistQueueOptions.stallTimeoutMs`). */+ stallTimeoutMs: numbercheckpointEveryWrites: numberheartbeatMs: number/**@@ -95,6 +97,7 @@ export function createConversationStore(const persistQueue = new PersistQueue(worker, {syncTimeoutMs: opts.syncTimeoutMs,asyncTimeoutMs: opts.asyncTimeoutMs,+ stallTimeoutMs: opts.stallTimeoutMs,onTransportError: err => {logger.error({ err }, '[ConversationStore] worker transport error')},diff --git a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts b/mcp-host/src/core/conversation/persistence/persistFailureExit.tsindex 4a7591b43..e1ef2eacf 100644--- a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts+++ b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts@@ -10,6 +10,21 @@ export interface PersistFailureExitOptions {beforeExitTimeoutMs: number}+/**+ * Exit code of a graceful shutdown: 1 when closing the conversation store+ * rejects (a write was lost or the worker stalled), so the loss shows in the+ * container's termination state; 0 otherwise.+ */+export async function storeShutdownExitCode(shutdown: () => Promise<void>): Promise<number> {+ try {+ await shutdown()+ return 0+ } catch (err) {+ logger.error({ err }, 'ConversationStore shutdown failed; exiting with code 1')+ return 1+ }+}+/*** `onPersistFailure` handler for the Host process. A persist queue that can* no longer write never recovers in-process, so the process exits 1 and thediff --git a/mcp-host/src/core/conversation/persistence/persistQueue.ts b/mcp-host/src/core/conversation/persistence/persistQueue.tsindex d92dd13a9..23a1913c1 100644--- a/mcp-host/src/core/conversation/persistence/persistQueue.ts+++ b/mcp-host/src/core/conversation/persistence/persistQueue.ts@@ -12,6 +12,12 @@* with that failure and `onFailure` fires once, so the Host stops instead* of accepting writes it can no longer persist. The same holds after an* unexpected worker exit.+ * - Progress bound: once the database is open, while the worker has+ * operations in flight it must complete one within `stallTimeoutMs` of the+ * previous completion (or of the moment work became in flight). A slow+ * worker keeps completing operations, however long its backlog; a wedged+ * one stops, and the queue fails the same way as after a worker exit. The+ * bound measures worker progress, never how long a write has waited.** The implementation is transport-agnostic: callers inject a `WorkerLike`* (typically a `node:worker_threads.Worker`, but tests inject an in-process@@ -38,11 +44,14 @@ export interface WorkerLike {export interface PersistQueueOptions {syncTimeoutMs: number+ /** An async write not ACKed within this time is logged and kept pending. */+ asyncTimeoutMs: number/**- * An async write not ACKed within this time is logged and kept pending.- * `close()` waits this long for the queued async writes to settle.+ * Longest time the worker may go without completing an operation while it+ * has operations in flight, in service and during `close()`. Not applied+ * before the `awaitOpen()` ping is answered.*/- asyncTimeoutMs: number+ stallTimeoutMs: number/** Called whenever a heartbeat reply is observed (id === HEARTBEAT_ID). */onHeartbeat?: (payload: HeartbeatPayload) => void/** Called on transport-level errors (worker crashed, message channel broke). */@@ -55,7 +64,8 @@ export interface PersistQueueOptions {onExit?: (code: number) => void/*** Called once, with the first failure that makes the queue unusable: an- * async write failed, or the worker exited before ACKing `shutdown`.+ * async write failed, the worker made no progress within `stallTimeoutMs`,+ * or the worker exited before ACKing `shutdown`.*/onFailure?: (err: Error) => void}@@ -79,6 +89,19 @@ export class PersistQueue {/** First failure that makes further writes unsafe; never cleared. */private failure: Error | null = nullprivate closePromise: Promise<void> | null = null+ /**+ * Ids posted to the worker and not yet answered. Unlike `pending`, an id+ * stays here after its caller's timeout: the worker is still busy with it.+ */+ private readonly inFlight = new Set<string>()+ private inFlightWaiters: Array<() => void> = []+ /** The `awaitOpen()` ping was answered; the progress bound applies. */+ private opened = false+ /** `performance.now()` of the last completion, or of in-flight going 0 → 1. */+ private lastProgressAt = 0+ private stallTimer: ReturnType<typeof setTimeout> | undefined+ /** The progress bound expired and the worker was terminated. */+ private stalled = falseconstructor(private readonly worker: WorkerLike,@@ -99,6 +122,7 @@ export class PersistQueue {)}this.pending.clear()+ this.settleInFlight()if (this.shutdownAcked) returnif (this.opts.onExit) this.opts.onExit(code)this.recordFailure(new Error(`db worker exited (code=${code})`))@@ -186,7 +210,8 @@ export class PersistQueue {* Resolves when the worker answers a `ping`. The worker registers its* message handler only after it has opened and migrated the database, so* this resolves once the database is open. No timeout: opening (spawn,- * migrations, recovery of a large WAL) can outlast any write budget.+ * migrations, recovery of a large WAL) can outlast any write budget, so+ * the progress bound applies only once this ping is answered.* Rejects on a worker error reply, a worker exit, or `close()`.*/awaitOpen(): Promise<void> {@@ -195,7 +220,11 @@ export class PersistQueue {} catch (err) {return Promise.reject(err as Error)}- return this.send<unknown>({ kind: 'ping' }, null).then(() => undefined)+ return this.send<unknown>({ kind: 'ping' }, null).then(() => {+ this.opened = true+ this.lastProgressAt = performance.now()+ this.armStallTimer()+ })}/**@@ -240,16 +269,14 @@ export class PersistQueue {}/**- * Graceful shutdown: close admission, drain the keyed writes in- * `writeChain` (async writes and sync writes with a `sessionKey`) for at- * most `asyncTimeoutMs` (past it, a failure is recorded and the worker is- * terminated without `shutdown`), wait for- * the worker to ACK `shutdown` (it closes the database before replying),- * then terminate it. A sync write without a `sessionKey` is not in- * `writeChain`; if the worker is still retrying it on a busy database,- * `shutdown` can close the database first and the write is rejected.- * Rejects with the first write failure, or with the shutdown failure, so- * the caller sees that the durable trail is incomplete.+ * Graceful shutdown: close admission, wait until every accepted write has+ * settled (the keyed chains and every operation in flight, sync writes+ * without a `sessionKey` included), wait for the worker to ACK `shutdown`+ * (it closes the database before replying), then terminate it. There is no+ * total deadline: a slow worker drains its whole backlog, and only the+ * progress bound (`stallTimeoutMs`) ends the wait for a wedged one.+ * Rejects with the first failure, so the caller sees that the durable+ * trail is incomplete.*/close(): Promise<void> {// A second caller waits for the same shutdown instead of returning early.@@ -259,23 +286,26 @@ export class PersistQueue {private async shutdown(): Promise<void> {this.closed = true- if (!(await this.drainWithin(this.opts.asyncTimeoutMs))) {- // The worker is stuck on a write; terminate() below rejects it.- this.recordFailure(- new Error(`async writes not acknowledged within ${this.opts.asyncTimeoutMs}ms of close()`)- )- } else if (!this.exited) {+ // Admission is closed, so no chain starts after this snapshot.+ await Promise.allSettled(Array.from(this.writeChain.values()))+ await this.inFlightSettled()+ if (!this.exited && !this.stalled) {try {- await this.send({ kind: 'shutdown' }, this.opts.syncTimeoutMs)+ // Checkpointing on close can outlast `syncTimeoutMs`; the progress+ // bound covers this operation like any other.+ await this.send({ kind: 'shutdown' }, null)} catch (err) {// Recorded before terminate(), so the exit it causes is not reported// as a second failure.this.recordFailure(err as Error)}}- const terminated = this.worker.terminate()- if (terminated && typeof (terminated as Promise<number>).then === 'function') {- await terminated+ if (!this.stalled) {+ // A stalled worker was already terminated by checkStall().+ const terminated = this.worker.terminate()+ if (terminated && typeof (terminated as Promise<number>).then === 'function') {+ await terminated+ }}for (const entry of this.pending.values()) {clearTimeout(entry.timeoutHandle)@@ -285,15 +315,71 @@ export class PersistQueue {if (this.failure) throw this.failure}- private async drainWithin(ms: number): Promise<boolean> {- let timer: ReturnType<typeof setTimeout> | undefined- const expired = new Promise<boolean>(resolve => {- timer = setTimeout(() => resolve(false), ms)- })- try {- return await Promise.race([this.drain().then(() => true), expired])- } finally {- clearTimeout(timer)+ private inFlightSettled(): Promise<void> {+ if (this.inFlight.size === 0) return Promise.resolve()+ return new Promise(resolve => this.inFlightWaiters.push(resolve))+ }++ /** Forget every in-flight id (worker gone or stalled) and wake waiters. */+ private settleInFlight(): void {+ this.inFlight.clear()+ clearTimeout(this.stallTimer)+ this.stallTimer = undefined+ this.notifyInFlightSettled()+ }++ private notifyInFlightSettled(): void {+ if (this.inFlight.size !== 0) return+ const waiters = this.inFlightWaiters+ this.inFlightWaiters = []+ for (const resolve of waiters) resolve()+ }++ private armStallTimer(): void {+ // Still armed after another failure: close() waits for in-flight work.+ if (!this.opened || this.stalled || this.stallTimer || this.inFlight.size === 0) return+ const remaining = this.opts.stallTimeoutMs - (performance.now() - this.lastProgressAt)+ this.stallTimer = setTimeout(+ () => {+ this.stallTimer = undefined+ // Timers run before the poll phase that delivers worker replies. After+ // a main-thread stall an answer can be waiting; read it first.+ setImmediate(() => this.checkStall())+ },+ Math.max(0, remaining)+ )+ }++ private checkStall(): void {+ if (this.stalled || this.inFlight.size === 0) return+ if (performance.now() - this.lastProgressAt < this.opts.stallTimeoutMs) {+ this.armStallTimer()+ return+ }+ const count = this.inFlight.size+ this.stalled = true+ const failure = new Error(+ `db worker made no progress for ${this.opts.stallTimeoutMs}ms with ${count} operation(s) in flight`+ )+ // The logger keeps only an error's name, so the cause is logged as fields.+ logger.error(+ { stallTimeoutMs: this.opts.stallTimeoutMs, inFlight: count },+ '[PersistQueue] db worker made no progress; terminating it'+ )+ this.recordFailure(failure)+ for (const entry of this.pending.values()) {+ clearTimeout(entry.timeoutHandle)+ entry.reject(failure)+ }+ this.pending.clear()+ this.settleInFlight()+ // The failure is recorded first, so the exit that follows is not reported+ // as a second one. Not awaited: a wedged thread may take a while to stop.+ const terminated = this.worker.terminate()+ if (terminated && typeof (terminated as Promise<number>).then === 'function') {+ ;(terminated as Promise<number>).catch((err: unknown) => {+ logger.warn({ err }, '[PersistQueue] terminating the stalled db worker failed')+ })}}@@ -360,7 +446,11 @@ export class PersistQueue {this.pending.delete(id)clearTimeout(timeoutHandle)reject(err as Error)+ return}+ if (this.inFlight.size === 0) this.lastProgressAt = performance.now()+ this.inFlight.add(id)+ this.armStallTimer()})}@@ -371,6 +461,13 @@ export class PersistQueue {}return}+ if (this.inFlight.delete(reply.id)) {+ this.lastProgressAt = performance.now()+ clearTimeout(this.stallTimer)+ this.stallTimer = undefined+ this.armStallTimer()+ this.notifyInFlightSettled()+ }const entry = this.pending.get(reply.id)if (!entry) returnclearTimeout(entry.timeoutHandle)diff --git a/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts b/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.tsindex 49404cc66..fc5e1d63c 100644--- a/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts+++ b/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts@@ -28,6 +28,7 @@ function makeHarness(): Harness {const persistQueue = new PersistQueue(handle.worker, {syncTimeoutMs: 2000,asyncTimeoutMs: 2000,+ stallTimeoutMs: 30_000,})const service = new SessionSearchService({ persistQueue })return {diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.tsindex e9800eea8..bd96f7412 100644--- a/mcp-host/src/main.ts+++ b/mcp-host/src/main.ts@@ -45,7 +45,10 @@ import {SqliteColdStartLoader,createConversationStore,} from './core/conversation/persistence'-import { exitOnPersistFailure } from './core/conversation/persistence/persistFailureExit'+import {+ exitOnPersistFailure,+ storeShutdownExitCode,+} from './core/conversation/persistence/persistFailureExit'import { validateApprovalConfig } from './core/extensions'import type { ApprovalDecision } from './core/extensions/approvalTypes'import { BasicSafety } from './core/safety/safety'@@ -1627,6 +1630,7 @@ async function initializeConversationStore(): Promise<ConversationStoreHandle |cacheSize: config.conversationCacheSize,syncTimeoutMs: config.dbPersistSyncTimeoutMs,asyncTimeoutMs: config.dbPersistAsyncTimeoutMs,+ stallTimeoutMs: config.dbWorkerStallTimeoutMs,checkpointEveryWrites: config.dbCheckpointEveryWrites,heartbeatMs: config.dbWorkerHeartbeatMs,pendingApprovalTtlMs: config.pendingApprovalTtlMs,@@ -3109,13 +3113,12 @@ async function shutdown(signal: string): Promise<void> {await pluginWorkloadSdkServer?.stop()await rpcServer?.stop()- // T2.1 — drain pending DB writes, then terminate the SQLite worker.+ // T2.1 — drain pending DB writes, then terminate the SQLite worker. A+ // write that could not be persisted makes the process exit 1.+ let exitCode = 0if (conversationStoreHandle) {- try {- await conversationStoreHandle.shutdown()- } catch (err) {- logger.warn({ err: err }, '[Main] ConversationStore shutdown raised:')- }+ const handle = conversationStoreHandle+ exitCode = await storeShutdownExitCode(() => handle.shutdown())}if (gfsRuntimeStop) {@@ -3128,7 +3131,7 @@ async function shutdown(signal: string): Promise<void> {}}- process.exit(0)+ process.exit(exitCode)}/**