Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): bound db worker progress instead of write residence

Alfredo Lopezdc1a299be92b
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index edd790f5e..5ae4f745d 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.574",
+ "version": "0.1.575",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.574",
+ "version": "0.1.575",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 6fd48fd24..7c1cbc29d 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.574",
+ "version": "0.1.575",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.ts b/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.ts
new file mode 100644
index 000000000..ed205b15c
--- /dev/null
+++ b/mcp-host/src/__tests__/config.dbWorkerStallTimeout.test.ts
@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest'
+import { parseDbWorkerStallTimeoutMs } from '../config'
+
+describe('CLERUM_DB_WORKER_STALL_TIMEOUT_MS', () => {
+ it('defaults to 30000 when unset or blank', () => {
+ expect(parseDbWorkerStallTimeoutMs(undefined, 2000)).toBe(30_000)
+ expect(parseDbWorkerStallTimeoutMs(' ', 2000)).toBe(30_000)
+ })
+
+ it('accepts an integer of at least 1000 that exceeds the sync timeout', () => {
+ expect(parseDbWorkerStallTimeoutMs('15000', 2000)).toBe(15_000)
+ expect(parseDbWorkerStallTimeoutMs('1000', 999)).toBe(1000)
+ })
+
+ it.each(['abc', '1.5', '-1', '0', '999', '30s'])('rejects %s', raw => {
+ expect(() => parseDbWorkerStallTimeoutMs(raw, 500)).toThrow(
+ `CLERUM_DB_WORKER_STALL_TIMEOUT_MS='${raw}' must be an integer of at least 1000`
+ )
+ })
+
+ it('rejects a value that does not exceed the sync timeout', () => {
+ expect(() => parseDbWorkerStallTimeoutMs('2000', 2000)).toThrow(
+ "CLERUM_DB_WORKER_STALL_TIMEOUT_MS='2000' must exceed CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS (2000)"
+ )
+ })
+})
diff --git a/mcp-host/src/config.ts b/mcp-host/src/config.ts
index 07ed07891..25133690f 100644
--- a/mcp-host/src/config.ts
+++ b/mcp-host/src/config.ts
@@ -233,6 +233,7 @@ export interface Config {
dbWorkerHeartbeatMs: number
dbPersistSyncTimeoutMs: number
dbPersistAsyncTimeoutMs: number
+ dbWorkerStallTimeoutMs: number
dbCheckpointEveryWrites: number
dbWalSizeAlarmBytes: number
@@ -473,6 +474,31 @@ export function parseStatelessHeartbeatIntervalMs(raw: string | undefined): numb
return parsed
}
+/**
+ * Parse CLERUM_DB_WORKER_STALL_TIMEOUT_MS, the longest time the db worker may
+ * go without completing an operation while it has work in flight. It must
+ * exceed the sync write timeout, so a sync write times out on its own budget
+ * before the worker is declared stalled. An explicit invalid value fails at
+ * boot.
+ */
+export function parseDbWorkerStallTimeoutMs(
+ raw: string | undefined,
+ syncTimeoutMs: number
+): number {
+ if (raw === undefined || raw.trim() === '') return 30_000
+ const name = 'CLERUM_DB_WORKER_STALL_TIMEOUT_MS'
+ const value = /^\d+$/.test(raw) ? Number(raw) : NaN
+ if (!Number.isSafeInteger(value) || value < 1000) {
+ throw new Error(`${name}='${raw}' must be an integer of at least 1000`)
+ }
+ if (value <= syncTimeoutMs) {
+ throw new Error(
+ `${name}='${raw}' must exceed CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS (${syncTimeoutMs})`
+ )
+ }
+ return value
+}
+
/**
* Parse an MCP status-heartbeat duration. These values participate in the
* liveness scheduler, so an explicit typo must fail at boot rather than turn
@@ -1069,6 +1095,10 @@ export const config: Config = {
dbWorkerHeartbeatMs: parseInt(getEnv('CLERUM_DB_WORKER_HEARTBEAT_MS', '5000')!, 10),
dbPersistSyncTimeoutMs: parseInt(getEnv('CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS', '2000')!, 10),
dbPersistAsyncTimeoutMs: parseInt(getEnv('CLERUM_DB_PERSIST_ASYNC_TIMEOUT_MS', '5000')!, 10),
+ dbWorkerStallTimeoutMs: parseDbWorkerStallTimeoutMs(
+ getEnv('CLERUM_DB_WORKER_STALL_TIMEOUT_MS', ''),
+ parseInt(getEnv('CLERUM_DB_PERSIST_SYNC_TIMEOUT_MS', '2000')!, 10)
+ ),
dbCheckpointEveryWrites: parseInt(getEnv('CLERUM_DB_CHECKPOINT_EVERY_WRITES', '100')!, 10),
dbWalSizeAlarmBytes: parseInt(getEnv('CLERUM_DB_WAL_SIZE_ALARM_BYTES', '52428800')!, 10),
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts
index a133b5388..ba00dc84a 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/concurrentSessionCreate.test.ts
@@ -26,6 +26,7 @@ function setup() {
const queue = new PersistQueue(worker, {
syncTimeoutMs: 2000,
asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
onFailure,
})
const manager = new ConversationManager(new SqliteConversationStore(queue, { cacheSize: 8 }))
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts
index df236a599..74ecde176 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts
@@ -47,6 +47,7 @@ describe('createConversationStore persist failure', () => {
cacheSize: 10,
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -86,6 +87,7 @@ describe('createConversationStore persist failure', () => {
cacheSize: 10,
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -128,6 +130,7 @@ describe('createConversationStore store identity', () => {
cacheSize: 10,
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -173,6 +176,7 @@ describe('createConversationStore store identity', () => {
cacheSize: 10,
syncTimeoutMs: 5000,
asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -232,6 +236,7 @@ describe('createConversationStore store identity', () => {
cacheSize: 10,
syncTimeoutMs: 100,
asyncTimeoutMs: 100,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -267,6 +272,7 @@ describe('createConversationStore store identity', () => {
cacheSize: 10,
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
workerScriptPath: script,
@@ -300,6 +306,7 @@ describe('createConversationStore store identity', () => {
cacheSize: 10,
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
checkpointEveryWrites: 100,
heartbeatMs: 60_000,
})
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts
index afb4002f8..140643e74 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/durabilityBarrier.test.ts
@@ -241,7 +241,11 @@ describe('D3 durability barrier', () => {
it('a worker failure applies NEITHER the assistant message NOR the active_task_id clear', async () => {
const raw = createInProcessWorker(tmpDbPath())
const faulty = new FaultInjectingWorker(raw.worker)
- const queue = new PersistQueue(faulty, { syncTimeoutMs: 2000, asyncTimeoutMs: 5000 })
+ const queue = new PersistQueue(faulty, {
+ syncTimeoutMs: 2000,
+ asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
+ })
const store = new SqliteConversationStore(queue, { cacheSize: 8 })
const manager = new ConversationManager(store)
@@ -263,7 +267,11 @@ describe('D3 durability barrier', () => {
it('a worker failure on turn-start applies NOTHING and rolls back the in-RAM turn', async () => {
const raw = createInProcessWorker(tmpDbPath())
const faulty = new FaultInjectingWorker(raw.worker)
- const queue = new PersistQueue(faulty, { syncTimeoutMs: 2000, asyncTimeoutMs: 5000 })
+ const queue = new PersistQueue(faulty, {
+ syncTimeoutMs: 2000,
+ asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
+ })
const store = new SqliteConversationStore(queue, { cacheSize: 8 })
const manager = new ConversationManager(store)
@@ -382,7 +390,11 @@ describe('D3 durability barrier', () => {
const dbPath = tmpDbPath()
const raw = createInProcessWorker(dbPath, { barrierMode: true })
const faulty = new FaultInjectingWorker(raw.worker)
- const queue = new PersistQueue(faulty, { syncTimeoutMs: 500, asyncTimeoutMs: 5000 })
+ const queue = new PersistQueue(faulty, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
+ })
const store = new SqliteConversationStore(queue, { cacheSize: 8 })
const manager = new ConversationManager(store)
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
index 100ebab20..7258b1032 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
@@ -3,12 +3,13 @@ import { spawnSync } from 'node:child_process'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
-import { exitOnPersistFailure } from '../persistFailureExit'
+import { exitOnPersistFailure, storeShutdownExitCode } from '../persistFailureExit'
// Runs the real factory, PersistQueue and exitOnPersistFailure in a child
// Node process and reads that process's exit code. The worker thread is a
-// JS stand-in: it acknowledges writes by appending them to a file, fails the
-// `fail` op and exits on `crash`.
+// JS stand-in: it answers the open ping, acknowledges writes by appending them
+// to a file (`slow` after 30 ms), fails the `fail` op, never answers `hang`
+// and exits on `crash`.
const SRC = path.resolve(__dirname, '..', '..', '..', '..')
const HOOK = path.join(__dirname, 'fixtures', 'tsResolveHook.cjs')
@@ -23,6 +24,13 @@ const fs = require('node:fs')
const { parentPort, workerData } = require('node:worker_threads')
parentPort.on('message', msg => {
const kind = msg.op.kind
+ if (kind === 'ping') parentPort.postMessage({ id: msg.id, ok: true, result: null })
+ if (kind === 'slow') {
+ setTimeout(() => {
+ fs.appendFileSync(workerData.dbPath, msg.op.value + '\\n')
+ parentPort.postMessage({ id: msg.id, ok: true, result: null })
+ }, 30)
+ }
if (kind === 'write') {
fs.appendFileSync(workerData.dbPath, msg.op.value + '\\n')
parentPort.postMessage({ id: msg.id, ok: true, result: null })
@@ -44,17 +52,25 @@ const path = require('node:path')
const [srcRoot, workerPath, dataPath, scenario] = process.argv.slice(2)
const persistence = path.join(srcRoot, 'core/conversation/persistence')
const { createConversationStore } = require(path.join(persistence, 'conversationStoreFactory.ts'))
-const { exitOnPersistFailure } = require(path.join(persistence, 'persistFailureExit.ts'))
+const { exitOnPersistFailure, storeShutdownExitCode } = require(path.join(persistence, 'persistFailureExit.ts'))
+// Mirrors main.ts: the graceful path marks the Host as shutting down, closes
+// the store and exits with the code storeShutdownExitCode returns.
+let shuttingDown = false
+async function gracefulExit() {
+ shuttingDown = true
+ process.exit(await storeShutdownExitCode(() => handle.shutdown()))
+}
const handle = createConversationStore({
mode: 'sqlite',
dbPath: dataPath,
cacheSize: 10,
syncTimeoutMs: 5000,
- asyncTimeoutMs: 5000,
+ asyncTimeoutMs: 50,
+ stallTimeoutMs: 300,
checkpointEveryWrites: 100,
heartbeatMs: 60000,
workerScriptPath: workerPath,
- onPersistFailure: exitOnPersistFailure(() => false, {
+ onPersistFailure: exitOnPersistFailure(() => shuttingDown, {
beforeExit: async () => console.log('PRE_EXIT_FLUSH'),
beforeExitTimeoutMs: 1000,
}),
@@ -69,6 +85,24 @@ const queue = handle.persistQueue
console.log('SHUTDOWN_CLEAN')
return
}
+ if (scenario === 'shutdown-slow-backlog') {
+ for (let i = 0; i < 20; i++) queue.enqueueAsync('s1', { kind: 'slow', value: 'slow' + i })
+ await gracefulExit()
+ return
+ }
+ if (scenario === 'shutdown-wedged') {
+ queue.enqueueAsync('s1', { kind: 'hang' })
+ await gracefulExit()
+ return
+ }
+ if (scenario === 'wedged') {
+ queue.enqueueAsync('s1', { kind: 'hang' })
+ setTimeout(() => {
+ console.log('STILL_RUNNING')
+ process.exit(0)
+ }, 5000)
+ return
+ }
queue.enqueueAsync('s1', { kind: scenario === 'async-write-failure' ? 'fail' : 'crash' })
setTimeout(() => {
console.log('STILL_RUNNING')
@@ -77,7 +111,15 @@ const queue = handle.persistQueue
})()
`
-function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {
+type Scenario =
+ | 'alive'
+ | 'async-write-failure'
+ | 'worker-exit'
+ | 'wedged'
+ | 'shutdown-slow-backlog'
+ | 'shutdown-wedged'
+
+function runChild(scenario: Scenario) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'persist-failure-exit-'))
roots.push(root)
const worker = path.join(root, 'worker.cjs')
@@ -85,6 +127,7 @@ function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {
const data = path.join(root, 'writes.log')
fs.writeFileSync(worker, WORKER)
fs.writeFileSync(child, CHILD)
+ const started = Date.now()
const result = spawnSync(
process.execPath,
[
@@ -100,10 +143,11 @@ function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {
],
{ encoding: 'utf8', timeout: 20_000 }
)
+ const elapsedMs = Date.now() - started
const writes = fs.existsSync(data)
? fs.readFileSync(data, 'utf8').split('\n').filter(Boolean)
: []
- return { status: result.status, out: `${result.stdout}${result.stderr}`, writes }
+ return { status: result.status, out: `${result.stdout}${result.stderr}`, writes, elapsedMs }
}
describe('exitOnPersistFailure in a Host process', () => {
@@ -132,6 +176,51 @@ describe('exitOnPersistFailure in a Host process', () => {
expect(run.out).not.toContain('STILL_RUNNING')
expect(run.status).toBe(1)
})
+
+ it('exits 1, bounded, when the worker stays alive but stops answering', () => {
+ const run = runChild('wedged')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ expect(run.out).toContain('db worker made no progress')
+ expect(run.out).toContain('"stallTimeoutMs":300')
+ expect(run.out).toContain('PRE_EXIT_FLUSH')
+ expect(run.out).not.toContain('STILL_RUNNING')
+ expect(run.status).toBe(1)
+ // 300 ms bound + 1 s flush deadline + process start-up.
+ expect(run.elapsedMs).toBeLessThan(4000)
+ })
+})
+
+describe('Host process exit on a graceful shutdown', () => {
+ it('drains a slow backlog longer than asyncTimeoutMs and exits 0', () => {
+ const run = runChild('shutdown-slow-backlog')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ // Liveness witness: every queued write reached the store.
+ expect(run.writes).toEqual(['first', ...Array.from({ length: 20 }, (_, i) => `slow${i}`)])
+ expect(run.out).not.toContain('ConversationStore shutdown failed')
+ expect(run.status).toBe(0)
+ })
+
+ it('exits 1 when the store cannot drain because the worker is wedged', () => {
+ const run = runChild('shutdown-wedged')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ expect(run.out).toContain('ConversationStore shutdown failed; exiting with code 1')
+ expect(run.out).toContain('db worker made no progress')
+ expect(run.out).toContain('"stallTimeoutMs":300')
+ expect(run.status).toBe(1)
+ expect(run.elapsedMs).toBeLessThan(4000)
+ })
+})
+
+describe('storeShutdownExitCode', () => {
+ it('returns 0 when the store closes and 1 when closing rejects', async () => {
+ // Liveness witness: a clean close is 0, so the 1 below comes from the rejection.
+ await expect(storeShutdownExitCode(async () => undefined)).resolves.toBe(0)
+ await expect(
+ storeShutdownExitCode(async () => {
+ throw new Error('async write failed (sessionKey=s1): disk I/O error')
+ })
+ ).resolves.toBe(1)
+ })
})
describe('exitOnPersistFailure during shutdown', () => {
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
index 80365eebe..7211bd0c7 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
@@ -1,6 +1,7 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { EventEmitter } from 'node:events'
-import type { WorkerMessage, WorkerReply } from '../../../../db/worker/protocol'
+import { Worker } from 'node:worker_threads'
+import type { WorkerMessage, WorkerOp, WorkerReply } from '../../../../db/worker/protocol'
import { PersistQueue, type WorkerLike } from '../persistQueue'
class FakeWorker extends EventEmitter implements WorkerLike {
@@ -23,7 +24,11 @@ class FakeWorker extends EventEmitter implements WorkerLike {
describe('PersistQueue', () => {
it('resolves enqueueSync when worker replies ok', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 200,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
const promise = q.enqueueSync({ kind: 'ping' })
const sent = worker.sent[0]
worker.emit('message', { id: sent.id, ok: true, result: 'pong' } satisfies WorkerReply)
@@ -32,7 +37,11 @@ describe('PersistQueue', () => {
it('rejects enqueueSync when worker replies with error', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 200,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
const promise = q.enqueueSync({ kind: 'ping' })
const sent = worker.sent[0]
worker.emit('message', {
@@ -45,13 +54,21 @@ describe('PersistQueue', () => {
it('times out enqueueSync', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 30,
+ asyncTimeoutMs: 30,
+ stallTimeoutMs: 30_000,
+ })
await expect(q.enqueueSync({ kind: 'ping' })).rejects.toThrow(/timeout/)
})
it('awaitOpen outlasts the sync timeout and resolves on the worker reply', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 30,
+ asyncTimeoutMs: 30,
+ stallTimeoutMs: 30_000,
+ })
let settled = false
const open = q.awaitOpen().finally(() => {
settled = true
@@ -70,7 +87,11 @@ describe('PersistQueue', () => {
it('awaitOpen rejects when the worker exits before answering', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 30, asyncTimeoutMs: 30 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 30,
+ asyncTimeoutMs: 30,
+ stallTimeoutMs: 30_000,
+ })
const open = q.awaitOpen()
expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])
worker.emit('exit', 1)
@@ -79,7 +100,11 @@ describe('PersistQueue', () => {
it('enqueueAsync preserves order per sessionKey', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 200,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
q.enqueueAsync('s1', { kind: 'ping' })
q.enqueueAsync('s1', { kind: 'ping' })
// After microtasks the first one is sent but the second is awaiting the
@@ -94,7 +119,11 @@ describe('PersistQueue', () => {
it('drainPrefix waits only for chains whose sessionKey matches the prefix', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 200, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 200,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
// Two in-flight async writes under the same prefix + one unrelated key.
q.enqueueAsync('alice:rpc:a:1', { kind: 'ping' })
q.enqueueAsync('alice:rpc:b:2', { kind: 'ping' })
@@ -120,7 +149,11 @@ describe('PersistQueue', () => {
it('rejects pending writes when the worker exits', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
const promise = q.enqueueSync({ kind: 'ping' })
worker.emit('exit', 137)
await expect(promise).rejects.toThrow(/db worker exited/)
@@ -128,7 +161,11 @@ describe('PersistQueue', () => {
it('rejects async writes admitted after the worker exited instead of dropping them', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
q.enqueueAsync('s1', { kind: 'ping' })
await Promise.resolve()
// Liveness witness: while the worker is alive the async write is posted.
@@ -141,7 +178,11 @@ describe('PersistQueue', () => {
it('fails the next write on the same queue after an async write failed', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
// Witness: a sync write before any failure resolves.
const first = q.enqueueSync({ kind: 'ping' })
worker.emit('message', {
@@ -173,6 +214,7 @@ describe('PersistQueue', () => {
const q = new PersistQueue(worker, {
syncTimeoutMs: 500,
asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
onFailure: err => failures.push(err),
onExit: code => exits.push(code),
})
@@ -201,6 +243,7 @@ describe('PersistQueue', () => {
const q = new PersistQueue(worker, {
syncTimeoutMs: 500,
asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
onFailure: err => failures.push(err),
})
q.enqueueAsync('s1', { kind: 'ping' })
@@ -238,6 +281,7 @@ describe('PersistQueue', () => {
const q = new PersistQueue(worker, {
syncTimeoutMs: 500,
asyncTimeoutMs: 20,
+ stallTimeoutMs: 30_000,
onFailure: err => failures.push(err),
})
q.enqueueAsync('s1', { kind: 'ping' })
@@ -262,28 +306,6 @@ describe('PersistQueue', () => {
expect(worker.sent).toHaveLength(3)
})
- it('bounds the close() drain of an unacknowledged async write by asyncTimeoutMs', async () => {
- const worker = new FakeWorker()
- const failures: Error[] = []
- const exits: number[] = []
- worker.on('exit', code => exits.push(code))
- const q = new PersistQueue(worker, {
- syncTimeoutMs: 500,
- asyncTimeoutMs: 20,
- onFailure: err => failures.push(err),
- })
- q.enqueueAsync('s1', { kind: 'ping' })
- await Promise.resolve()
- expect(worker.sent).toHaveLength(1)
- const started = Date.now()
- await expect(q.close()).rejects.toThrow(/not acknowledged within 20ms of close/)
- expect(Date.now() - started).toBeLessThan(400)
- // The worker was terminated and the failure reported once.
- expect(exits).toEqual([0])
- expect(failures).toHaveLength(1)
- expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])
- })
-
it('reports neither onFailure nor onExit for the worker exit that follows a shutdown ACK', async () => {
const worker = new FakeWorker()
const failures: Error[] = []
@@ -293,6 +315,7 @@ describe('PersistQueue', () => {
const q = new PersistQueue(worker, {
syncTimeoutMs: 500,
asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
onFailure: err => failures.push(err),
onExit: code => reportedExits.push(code),
})
@@ -315,7 +338,11 @@ describe('PersistQueue', () => {
worker.terminate = () => {
terminateCalls += 1
}
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
const closing = q.close()
await new Promise(r => setImmediate(r))
@@ -331,7 +358,11 @@ describe('PersistQueue', () => {
it('drains queued async writes before posting shutdown and rejects close on a failed one', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
q.enqueueAsync('s1', { kind: 'ping' })
await Promise.resolve()
const closing = q.close()
@@ -353,7 +384,11 @@ describe('PersistQueue', () => {
it('never posts a write chained behind a failed async write on the same key', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
q.enqueueAsync('s1', { kind: 'ping' })
q.enqueueAsync('s1', { kind: 'ping' })
const chainedSync = q.enqueueSync({ kind: 'ping' }, 's1')
@@ -375,7 +410,11 @@ describe('PersistQueue', () => {
it('resolves a concurrent second close only after the shutdown ACK', async () => {
const worker = new FakeWorker()
- const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
+ })
let secondSettled = false
const first = q.close()
const second = q.close().finally(() => {
@@ -398,6 +437,7 @@ describe('PersistQueue', () => {
const q = new PersistQueue(worker, {
syncTimeoutMs: 500,
asyncTimeoutMs: 500,
+ stallTimeoutMs: 30_000,
onFailure: err => failures.push(err),
})
const closing = q.close()
@@ -415,3 +455,284 @@ describe('PersistQueue', () => {
expect(failures.map(e => e.message)).toEqual(['database is locked'])
})
})
+
+// A slow worker and a wedged worker differ in progress, not in how long a
+// write waits: a slow worker keeps completing operations, a wedged one stops.
+// These cases pin both, in service and at close(), so a change that fixes one
+// by breaking another fails here.
+describe('PersistQueue progress bound: slow vs wedged, in service and at close()', () => {
+ const sleep = (ms: number) => new Promise(r => setTimeout(r, ms))
+ const ack = (worker: FakeWorker, msg: WorkerMessage) =>
+ worker.emit('message', { id: msg.id, ok: true, result: null } satisfies WorkerReply)
+
+ /** A queue whose open ping was answered, so the progress bound is armed. */
+ async function openQueue(opts: {
+ stallTimeoutMs: number
+ asyncTimeoutMs?: number
+ syncTimeoutMs?: number
+ }) {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ let terminateCalls = 0
+ worker.terminate = () => {
+ terminateCalls += 1
+ worker.emit('exit', 1)
+ }
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: opts.syncTimeoutMs ?? 5000,
+ asyncTimeoutMs: opts.asyncTimeoutMs ?? 5000,
+ stallTimeoutMs: opts.stallTimeoutMs,
+ onFailure: err => failures.push(err),
+ })
+ const open = q.awaitOpen()
+ ack(worker, worker.sent[0])
+ await open
+ return { worker, q, failures, terminateCalls: () => terminateCalls }
+ }
+
+ /**
+ * Answers every posted operation in order, one at a time, `delayMs` after
+ * the previous answer: a single-writer worker that is slow but alive.
+ */
+ function autoAck(worker: FakeWorker, delayMs: number): WorkerMessage[] {
+ const acked: WorkerMessage[] = []
+ let busy = false
+ const queue: WorkerMessage[] = []
+ const next = () => {
+ const msg = queue.shift()
+ if (!msg) {
+ busy = false
+ return
+ }
+ busy = true
+ setTimeout(() => {
+ acked.push(msg)
+ ack(worker, msg)
+ next()
+ }, delayMs)
+ }
+ const post = worker.postMessage.bind(worker)
+ worker.postMessage = msg => {
+ post(msg)
+ queue.push(msg)
+ if (!busy) next()
+ }
+ return acked
+ }
+
+ it('fails a wedged worker in service within the bound and releases everything queued behind it', async () => {
+ const { worker, q, failures, terminateCalls } = await openQueue({
+ stallTimeoutMs: 100,
+ asyncTimeoutMs: 20,
+ })
+ // Liveness witness: a write on the same key is acknowledged first.
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ ack(worker, worker.sent[1])
+ await q.drainSessionKey('s1')
+ expect(failures).toHaveLength(0)
+
+ // The worker stops answering.
+ q.enqueueAsync('s1', { kind: 'ping' })
+ const keyedSync = q.enqueueSync({ kind: 'ping' }, 's1')
+ await Promise.resolve()
+ expect(worker.sent).toHaveLength(3)
+ const started = Date.now()
+ const prefixDrained = q.drainPrefix('s')
+
+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })
+ expect(Date.now() - started).toBeLessThan(400)
+ expect(failures[0].message).toMatch(/db worker made no progress for 100ms/)
+ await expect(keyedSync).rejects.toBe(failures[0])
+ await prefixDrained
+ expect(terminateCalls()).toBe(1)
+ // The write chained behind the stalled one was never posted.
+ expect(worker.sent).toHaveLength(3)
+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow(failures[0].message)
+ })
+
+ it('never fails a slow but alive worker with a backlog far longer than the bound', async () => {
+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 100, asyncTimeoutMs: 20 })
+ const acked = autoAck(worker, 50)
+ const keys = ['a', 'b', 'c']
+ // The tag only identifies the write in this test; the queue never reads it.
+ const tagged = (tag: string) => ({ kind: 'ping', tag }) as WorkerOp
+ for (let i = 0; i < 10; i++) {
+ for (const key of keys) q.enqueueAsync(key, tagged(`${key}${i}`))
+ }
+ await Promise.all(keys.map(k => q.drainSessionKey(k)))
+ const tags = acked.map(m => (m.op as WorkerOp & { tag: string }).tag)
+ // Liveness witness: all 30 writes were acknowledged, 1.5 s of backlog
+ // against a 100 ms bound, each key in its enqueue order.
+ expect(tags).toHaveLength(30)
+ for (const key of keys) {
+ expect(tags.filter(t => t.startsWith(key))).toEqual(
+ Array.from({ length: 10 }, (_, i) => `${key}${i}`)
+ )
+ }
+ expect(failures).toHaveLength(0)
+ })
+
+ it('drains a slow backlog completely on close() before posting shutdown', async () => {
+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, asyncTimeoutMs: 50 })
+ const acked = autoAck(worker, 10)
+ for (let i = 0; i < 10; i++) q.enqueueAsync('s1', { kind: 'ping' })
+ await q.close()
+ const kinds = acked.map(m => m.op.kind)
+ // Liveness witness: the ten writes and then shutdown were acknowledged.
+ expect(kinds).toEqual([...Array(10).fill('ping'), 'shutdown'])
+ expect(failures).toHaveLength(0)
+ })
+
+ it('fails close() within the bound when the worker is wedged', async () => {
+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })
+ // Liveness witness: a write is acknowledged before the worker stops.
+ const first = q.enqueueSync({ kind: 'ping' })
+ ack(worker, worker.sent[1])
+ await first
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ const started = Date.now()
+ await expect(q.close()).rejects.toThrow(/db worker made no progress for 100ms/)
+ expect(Date.now() - started).toBeLessThan(400)
+ expect(terminateCalls()).toBe(1)
+ expect(failures).toHaveLength(1)
+ expect(worker.sent.map(m => m.op.kind)).not.toContain('shutdown')
+ })
+
+ it('waits on close() for a sync write without a sessionKey before posting shutdown', async () => {
+ const { worker, q } = await openQueue({ stallTimeoutMs: 1000 })
+ const unkeyed = q.enqueueSync<string>({ kind: 'ping' })
+ const closing = q.close()
+ await sleep(20)
+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping', 'ping'])
+
+ worker.emit('message', {
+ id: worker.sent[1].id,
+ ok: true,
+ result: 'pong',
+ } satisfies WorkerReply)
+ // Liveness witness: the sync write resolves with the worker's result.
+ await expect(unkeyed).resolves.toBe('pong')
+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))
+ ack(worker, worker.sent[2])
+ await closing
+ })
+
+ it('does not fail a healthy worker whose answer waited behind a main-thread stall', async () => {
+ const realWorker = new Worker(
+ `const { parentPort } = require('node:worker_threads')
+ parentPort.on('message', msg => setTimeout(() => parentPort.postMessage({ id: msg.id, ok: true, result: null }), 10))`,
+ { eval: true }
+ )
+ const failures: Error[] = []
+ const q = new PersistQueue(realWorker as unknown as WorkerLike, {
+ syncTimeoutMs: 5000,
+ asyncTimeoutMs: 5000,
+ stallTimeoutMs: 100,
+ onFailure: err => failures.push(err),
+ })
+ try {
+ await q.awaitOpen()
+ const write = q.enqueueSync({ kind: 'ping' }, 's1')
+ await sleep(1)
+ // The main thread is blocked well past the bound while the answer
+ // waits in the port. Blocking in the check phase makes the next loop
+ // iteration run the overdue timer before the poll phase reads the
+ // answer (blocking in the timers phase would not: libuv compares
+ // timers against the loop time cached when that phase started).
+ await new Promise<void>(resolve =>
+ setImmediate(() => {
+ const until = Date.now() + 300
+ while (Date.now() < until) {
+ /* block the event loop */
+ }
+ resolve()
+ })
+ )
+ // Liveness witness: the write resolves.
+ await expect(write).resolves.toBeNull()
+ await sleep(150)
+ expect(failures).toHaveLength(0)
+ } finally {
+ await realWorker.terminate()
+ }
+ })
+
+ it('does not start the bound before the database is open', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 5000,
+ asyncTimeoutMs: 5000,
+ stallTimeoutMs: 100,
+ onFailure: err => failures.push(err),
+ })
+ const open = q.awaitOpen()
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ expect(worker.sent).toHaveLength(2)
+ await sleep(300)
+ expect(failures).toHaveLength(0)
+
+ ack(worker, worker.sent[0])
+ await open
+ ack(worker, worker.sent[1])
+ await q.drainSessionKey('s1')
+ expect(failures).toHaveLength(0)
+
+ // Liveness witness: once open, a wedged write does fail the queue.
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })
+ expect(failures[0].message).toMatch(/no progress for 100ms/)
+ })
+
+ it('stays failed when the stalled operation is acknowledged late', async () => {
+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 100 })
+ const stalled = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)
+ await vi.waitFor(() => expect(failures).toHaveLength(1), { timeout: 1000, interval: 5 })
+ expect(await stalled).toBe(failures[0])
+
+ worker.emit('message', {
+ id: worker.sent[1].id,
+ ok: true,
+ result: 'late',
+ } satisfies WorkerReply)
+ await sleep(1)
+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(failures[0].message)
+ await expect(q.enqueueSync({ kind: 'ping' })).rejects.toBe(failures[0])
+ expect(failures).toHaveLength(1)
+ })
+
+ it('waits for a shutdown that takes longer than syncTimeoutMs while within the bound', async () => {
+ const { worker, q, failures } = await openQueue({ stallTimeoutMs: 1000, syncTimeoutMs: 30 })
+ const closing = q.close()
+ await vi.waitFor(() => expect(worker.sent.map(m => m.op.kind)).toContain('shutdown'))
+ await sleep(90)
+ ack(worker, worker.sent.find(m => m.op.kind === 'shutdown')!)
+ // Liveness witness: close() resolves after the late shutdown ACK.
+ await expect(closing).resolves.toBeUndefined()
+ expect(failures).toHaveLength(0)
+ })
+
+ it('keeps the bound after an earlier failure, so close() does not wait forever', async () => {
+ const { worker, q, failures, terminateCalls } = await openQueue({ stallTimeoutMs: 100 })
+ q.enqueueAsync('a', { kind: 'ping' })
+ await vi.waitFor(() => expect(worker.sent).toHaveLength(2))
+ const hung = q.enqueueSync({ kind: 'ping' }).catch((err: unknown) => err)
+ expect(worker.sent).toHaveLength(3)
+ worker.emit('message', {
+ id: worker.sent[1].id,
+ ok: false,
+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },
+ } satisfies WorkerReply)
+ // Liveness witness: the async failure is latched first.
+ await vi.waitFor(() => expect(failures).toHaveLength(1))
+ const started = Date.now()
+ await expect(q.close()).rejects.toThrow(/database or disk is full/)
+ expect(Date.now() - started).toBeLessThan(400)
+ expect(terminateCalls()).toBe(1)
+ expect(await hung).toBeInstanceOf(Error)
+ expect(failures).toHaveLength(1)
+ })
+})
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts b/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts
index 7e362674c..6d46bd163 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/testHelpers.ts
@@ -121,6 +121,7 @@ export function makeSqliteStore(
const persistQueue = new PersistQueue(worker.worker, {
syncTimeoutMs: 2000,
asyncTimeoutMs: 5000,
+ stallTimeoutMs: 30_000,
})
const store = new SqliteConversationStore(persistQueue, {
cacheSize: opts.cacheSize ?? 8,
diff --git a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
index d44914bf0..e0079a46e 100644
--- a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
+++ b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
@@ -25,6 +25,8 @@ export interface ConversationStoreFactoryOptions {
cacheSize: number
syncTimeoutMs: number
asyncTimeoutMs: number
+ /** Progress bound of the persist queue (`PersistQueueOptions.stallTimeoutMs`). */
+ stallTimeoutMs: number
checkpointEveryWrites: number
heartbeatMs: number
/**
@@ -95,6 +97,7 @@ export function createConversationStore(
const persistQueue = new PersistQueue(worker, {
syncTimeoutMs: opts.syncTimeoutMs,
asyncTimeoutMs: opts.asyncTimeoutMs,
+ stallTimeoutMs: opts.stallTimeoutMs,
onTransportError: err => {
logger.error({ err }, '[ConversationStore] worker transport error')
},
diff --git a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
index 4a7591b43..e1ef2eacf 100644
--- a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
+++ b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
@@ -10,6 +10,21 @@ export interface PersistFailureExitOptions {
beforeExitTimeoutMs: number
}
+/**
+ * Exit code of a graceful shutdown: 1 when closing the conversation store
+ * rejects (a write was lost or the worker stalled), so the loss shows in the
+ * container's termination state; 0 otherwise.
+ */
+export async function storeShutdownExitCode(shutdown: () => Promise<void>): Promise<number> {
+ try {
+ await shutdown()
+ return 0
+ } catch (err) {
+ logger.error({ err }, 'ConversationStore shutdown failed; exiting with code 1')
+ return 1
+ }
+}
+
/**
* `onPersistFailure` handler for the Host process. A persist queue that can
* no longer write never recovers in-process, so the process exits 1 and the
diff --git a/mcp-host/src/core/conversation/persistence/persistQueue.ts b/mcp-host/src/core/conversation/persistence/persistQueue.ts
index d92dd13a9..23a1913c1 100644
--- a/mcp-host/src/core/conversation/persistence/persistQueue.ts
+++ b/mcp-host/src/core/conversation/persistence/persistQueue.ts
@@ -12,6 +12,12 @@
* with that failure and `onFailure` fires once, so the Host stops instead
* of accepting writes it can no longer persist. The same holds after an
* unexpected worker exit.
+ * - Progress bound: once the database is open, while the worker has
+ * operations in flight it must complete one within `stallTimeoutMs` of the
+ * previous completion (or of the moment work became in flight). A slow
+ * worker keeps completing operations, however long its backlog; a wedged
+ * one stops, and the queue fails the same way as after a worker exit. The
+ * bound measures worker progress, never how long a write has waited.
*
* The implementation is transport-agnostic: callers inject a `WorkerLike`
* (typically a `node:worker_threads.Worker`, but tests inject an in-process
@@ -38,11 +44,14 @@ export interface WorkerLike {
export interface PersistQueueOptions {
syncTimeoutMs: number
+ /** An async write not ACKed within this time is logged and kept pending. */
+ asyncTimeoutMs: number
/**
- * An async write not ACKed within this time is logged and kept pending.
- * `close()` waits this long for the queued async writes to settle.
+ * Longest time the worker may go without completing an operation while it
+ * has operations in flight, in service and during `close()`. Not applied
+ * before the `awaitOpen()` ping is answered.
*/
- asyncTimeoutMs: number
+ stallTimeoutMs: number
/** Called whenever a heartbeat reply is observed (id === HEARTBEAT_ID). */
onHeartbeat?: (payload: HeartbeatPayload) => void
/** Called on transport-level errors (worker crashed, message channel broke). */
@@ -55,7 +64,8 @@ export interface PersistQueueOptions {
onExit?: (code: number) => void
/**
* Called once, with the first failure that makes the queue unusable: an
- * async write failed, or the worker exited before ACKing `shutdown`.
+ * async write failed, the worker made no progress within `stallTimeoutMs`,
+ * or the worker exited before ACKing `shutdown`.
*/
onFailure?: (err: Error) => void
}
@@ -79,6 +89,19 @@ export class PersistQueue {
/** First failure that makes further writes unsafe; never cleared. */
private failure: Error | null = null
private closePromise: Promise<void> | null = null
+ /**
+ * Ids posted to the worker and not yet answered. Unlike `pending`, an id
+ * stays here after its caller's timeout: the worker is still busy with it.
+ */
+ private readonly inFlight = new Set<string>()
+ private inFlightWaiters: Array<() => void> = []
+ /** The `awaitOpen()` ping was answered; the progress bound applies. */
+ private opened = false
+ /** `performance.now()` of the last completion, or of in-flight going 0 → 1. */
+ private lastProgressAt = 0
+ private stallTimer: ReturnType<typeof setTimeout> | undefined
+ /** The progress bound expired and the worker was terminated. */
+ private stalled = false
constructor(
private readonly worker: WorkerLike,
@@ -99,6 +122,7 @@ export class PersistQueue {
)
}
this.pending.clear()
+ this.settleInFlight()
if (this.shutdownAcked) return
if (this.opts.onExit) this.opts.onExit(code)
this.recordFailure(new Error(`db worker exited (code=${code})`))
@@ -186,7 +210,8 @@ export class PersistQueue {
* Resolves when the worker answers a `ping`. The worker registers its
* message handler only after it has opened and migrated the database, so
* this resolves once the database is open. No timeout: opening (spawn,
- * migrations, recovery of a large WAL) can outlast any write budget.
+ * migrations, recovery of a large WAL) can outlast any write budget, so
+ * the progress bound applies only once this ping is answered.
* Rejects on a worker error reply, a worker exit, or `close()`.
*/
awaitOpen(): Promise<void> {
@@ -195,7 +220,11 @@ export class PersistQueue {
} catch (err) {
return Promise.reject(err as Error)
}
- return this.send<unknown>({ kind: 'ping' }, null).then(() => undefined)
+ return this.send<unknown>({ kind: 'ping' }, null).then(() => {
+ this.opened = true
+ this.lastProgressAt = performance.now()
+ this.armStallTimer()
+ })
}
/**
@@ -240,16 +269,14 @@ export class PersistQueue {
}
/**
- * Graceful shutdown: close admission, drain the keyed writes in
- * `writeChain` (async writes and sync writes with a `sessionKey`) for at
- * most `asyncTimeoutMs` (past it, a failure is recorded and the worker is
- * terminated without `shutdown`), wait for
- * the worker to ACK `shutdown` (it closes the database before replying),
- * then terminate it. A sync write without a `sessionKey` is not in
- * `writeChain`; if the worker is still retrying it on a busy database,
- * `shutdown` can close the database first and the write is rejected.
- * Rejects with the first write failure, or with the shutdown failure, so
- * the caller sees that the durable trail is incomplete.
+ * Graceful shutdown: close admission, wait until every accepted write has
+ * settled (the keyed chains and every operation in flight, sync writes
+ * without a `sessionKey` included), wait for the worker to ACK `shutdown`
+ * (it closes the database before replying), then terminate it. There is no
+ * total deadline: a slow worker drains its whole backlog, and only the
+ * progress bound (`stallTimeoutMs`) ends the wait for a wedged one.
+ * Rejects with the first failure, so the caller sees that the durable
+ * trail is incomplete.
*/
close(): Promise<void> {
// A second caller waits for the same shutdown instead of returning early.
@@ -259,23 +286,26 @@ export class PersistQueue {
private async shutdown(): Promise<void> {
this.closed = true
- if (!(await this.drainWithin(this.opts.asyncTimeoutMs))) {
- // The worker is stuck on a write; terminate() below rejects it.
- this.recordFailure(
- new Error(`async writes not acknowledged within ${this.opts.asyncTimeoutMs}ms of close()`)
- )
- } else if (!this.exited) {
+ // Admission is closed, so no chain starts after this snapshot.
+ await Promise.allSettled(Array.from(this.writeChain.values()))
+ await this.inFlightSettled()
+ if (!this.exited && !this.stalled) {
try {
- await this.send({ kind: 'shutdown' }, this.opts.syncTimeoutMs)
+ // Checkpointing on close can outlast `syncTimeoutMs`; the progress
+ // bound covers this operation like any other.
+ await this.send({ kind: 'shutdown' }, null)
} catch (err) {
// Recorded before terminate(), so the exit it causes is not reported
// as a second failure.
this.recordFailure(err as Error)
}
}
- const terminated = this.worker.terminate()
- if (terminated && typeof (terminated as Promise<number>).then === 'function') {
- await terminated
+ if (!this.stalled) {
+ // A stalled worker was already terminated by checkStall().
+ const terminated = this.worker.terminate()
+ if (terminated && typeof (terminated as Promise<number>).then === 'function') {
+ await terminated
+ }
}
for (const entry of this.pending.values()) {
clearTimeout(entry.timeoutHandle)
@@ -285,15 +315,71 @@ export class PersistQueue {
if (this.failure) throw this.failure
}
- private async drainWithin(ms: number): Promise<boolean> {
- let timer: ReturnType<typeof setTimeout> | undefined
- const expired = new Promise<boolean>(resolve => {
- timer = setTimeout(() => resolve(false), ms)
- })
- try {
- return await Promise.race([this.drain().then(() => true), expired])
- } finally {
- clearTimeout(timer)
+ private inFlightSettled(): Promise<void> {
+ if (this.inFlight.size === 0) return Promise.resolve()
+ return new Promise(resolve => this.inFlightWaiters.push(resolve))
+ }
+
+ /** Forget every in-flight id (worker gone or stalled) and wake waiters. */
+ private settleInFlight(): void {
+ this.inFlight.clear()
+ clearTimeout(this.stallTimer)
+ this.stallTimer = undefined
+ this.notifyInFlightSettled()
+ }
+
+ private notifyInFlightSettled(): void {
+ if (this.inFlight.size !== 0) return
+ const waiters = this.inFlightWaiters
+ this.inFlightWaiters = []
+ for (const resolve of waiters) resolve()
+ }
+
+ private armStallTimer(): void {
+ // Still armed after another failure: close() waits for in-flight work.
+ if (!this.opened || this.stalled || this.stallTimer || this.inFlight.size === 0) return
+ const remaining = this.opts.stallTimeoutMs - (performance.now() - this.lastProgressAt)
+ this.stallTimer = setTimeout(
+ () => {
+ this.stallTimer = undefined
+ // Timers run before the poll phase that delivers worker replies. After
+ // a main-thread stall an answer can be waiting; read it first.
+ setImmediate(() => this.checkStall())
+ },
+ Math.max(0, remaining)
+ )
+ }
+
+ private checkStall(): void {
+ if (this.stalled || this.inFlight.size === 0) return
+ if (performance.now() - this.lastProgressAt < this.opts.stallTimeoutMs) {
+ this.armStallTimer()
+ return
+ }
+ const count = this.inFlight.size
+ this.stalled = true
+ const failure = new Error(
+ `db worker made no progress for ${this.opts.stallTimeoutMs}ms with ${count} operation(s) in flight`
+ )
+ // The logger keeps only an error's name, so the cause is logged as fields.
+ logger.error(
+ { stallTimeoutMs: this.opts.stallTimeoutMs, inFlight: count },
+ '[PersistQueue] db worker made no progress; terminating it'
+ )
+ this.recordFailure(failure)
+ for (const entry of this.pending.values()) {
+ clearTimeout(entry.timeoutHandle)
+ entry.reject(failure)
+ }
+ this.pending.clear()
+ this.settleInFlight()
+ // The failure is recorded first, so the exit that follows is not reported
+ // as a second one. Not awaited: a wedged thread may take a while to stop.
+ const terminated = this.worker.terminate()
+ if (terminated && typeof (terminated as Promise<number>).then === 'function') {
+ ;(terminated as Promise<number>).catch((err: unknown) => {
+ logger.warn({ err }, '[PersistQueue] terminating the stalled db worker failed')
+ })
}
}
@@ -360,7 +446,11 @@ export class PersistQueue {
this.pending.delete(id)
clearTimeout(timeoutHandle)
reject(err as Error)
+ return
}
+ if (this.inFlight.size === 0) this.lastProgressAt = performance.now()
+ this.inFlight.add(id)
+ this.armStallTimer()
})
}
@@ -371,6 +461,13 @@ export class PersistQueue {
}
return
}
+ if (this.inFlight.delete(reply.id)) {
+ this.lastProgressAt = performance.now()
+ clearTimeout(this.stallTimer)
+ this.stallTimer = undefined
+ this.armStallTimer()
+ this.notifyInFlightSettled()
+ }
const entry = this.pending.get(reply.id)
if (!entry) return
clearTimeout(entry.timeoutHandle)
diff --git a/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts b/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts
index 49404cc66..fc5e1d63c 100644
--- a/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts
+++ b/mcp-host/src/core/sessionSearch/__tests__/sessionSearch.test.ts
@@ -28,6 +28,7 @@ function makeHarness(): Harness {
const persistQueue = new PersistQueue(handle.worker, {
syncTimeoutMs: 2000,
asyncTimeoutMs: 2000,
+ stallTimeoutMs: 30_000,
})
const service = new SessionSearchService({ persistQueue })
return {
diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.ts
index e9800eea8..bd96f7412 100644
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@ -45,7 +45,10 @@ import {
SqliteColdStartLoader,
createConversationStore,
} from './core/conversation/persistence'
-import { exitOnPersistFailure } from './core/conversation/persistence/persistFailureExit'
+import {
+ exitOnPersistFailure,
+ storeShutdownExitCode,
+} from './core/conversation/persistence/persistFailureExit'
import { validateApprovalConfig } from './core/extensions'
import type { ApprovalDecision } from './core/extensions/approvalTypes'
import { BasicSafety } from './core/safety/safety'
@@ -1627,6 +1630,7 @@ async function initializeConversationStore(): Promise<ConversationStoreHandle |
cacheSize: config.conversationCacheSize,
syncTimeoutMs: config.dbPersistSyncTimeoutMs,
asyncTimeoutMs: config.dbPersistAsyncTimeoutMs,
+ stallTimeoutMs: config.dbWorkerStallTimeoutMs,
checkpointEveryWrites: config.dbCheckpointEveryWrites,
heartbeatMs: config.dbWorkerHeartbeatMs,
pendingApprovalTtlMs: config.pendingApprovalTtlMs,
@@ -3109,13 +3113,12 @@ async function shutdown(signal: string): Promise<void> {
await pluginWorkloadSdkServer?.stop()
await rpcServer?.stop()
- // T2.1 — drain pending DB writes, then terminate the SQLite worker.
+ // T2.1 — drain pending DB writes, then terminate the SQLite worker. A
+ // write that could not be persisted makes the process exit 1.
+ let exitCode = 0
if (conversationStoreHandle) {
- try {
- await conversationStoreHandle.shutdown()
- } catch (err) {
- logger.warn({ err: err }, '[Main] ConversationStore shutdown raised:')
- }
+ const handle = conversationStoreHandle
+ exitCode = await storeShutdownExitCode(() => handle.shutdown())
}
if (gfsRuntimeStop) {
@@ -3128,7 +3131,7 @@ async function shutdown(signal: string): Promise<void> {
}
}
- process.exit(0)
+ process.exit(exitCode)
}
/**