Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): stop dropping writes after a db worker exit

Alfredo Lopez56b10b573a72
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 1928bc87d..0e6daaf99 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.557",
+ "version": "0.1.558",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.557",
+ "version": "0.1.558",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 2e9f5111a..17bcdd016 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.557",
+ "version": "0.1.558",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts
new file mode 100644
index 000000000..22a202791
--- /dev/null
+++ b/mcp-host/src/core/conversation/persistence/__tests__/conversationStoreFactory.test.ts
@@ -0,0 +1,67 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { createConversationStore } from '../conversationStoreFactory'
+
+const roots: string[] = []
+
+afterEach(() => {
+ for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true })
+})
+
+function workerScript(body: string): string {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'store-factory-'))
+ roots.push(root)
+ const file = path.join(root, 'worker.cjs')
+ fs.writeFileSync(file, body)
+ return file
+}
+
+function waitFor<T>(register: (resolve: (v: T) => void) => void, ms: number): Promise<T> {
+ return new Promise<T>((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error(`no callback within ${ms}ms`)), ms)
+ register(v => {
+ clearTimeout(timer)
+ resolve(v)
+ })
+ })
+}
+
+describe('createConversationStore persist failure', () => {
+ it('reports an unexpected exit of a real worker thread through onPersistFailure', async () => {
+ const script = workerScript(`
+ const { parentPort } = require('node:worker_threads')
+ parentPort.on('message', msg => {
+ if (msg.op.kind === 'ping') parentPort.postMessage({ id: msg.id, ok: true, result: 'pong' })
+ if (msg.op.kind === 'crash') process.exit(3)
+ })
+ `)
+ let onFailure: (err: Error) => void = () => {
+ throw new Error('onPersistFailure fired before the test registered it')
+ }
+ const handle = createConversationStore({
+ mode: 'sqlite',
+ dbPath: path.join(path.dirname(script), 'state.db'),
+ cacheSize: 10,
+ syncTimeoutMs: 2000,
+ asyncTimeoutMs: 2000,
+ checkpointEveryWrites: 100,
+ heartbeatMs: 60_000,
+ workerScriptPath: script,
+ onPersistFailure: err => onFailure(err),
+ })
+ const queue = handle.persistQueue
+ if (!queue) throw new Error('sqlite mode returned no persist queue')
+
+ // Liveness witness: the live worker acknowledges a write.
+ await expect(queue.enqueueSync({ kind: 'ping' })).resolves.toBe('pong')
+
+ const failure = waitFor<Error>(resolve => {
+ onFailure = resolve
+ }, 5000)
+ queue.enqueueAsync('s1', { kind: 'crash' } as never)
+ await expect(failure).resolves.toMatchObject({ message: 'db worker exited (code=3)' })
+ expect(() => queue.enqueueAsync('s1', { kind: 'ping' })).toThrow(/db worker exited/)
+ })
+})
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/fixtures/tsResolveHook.cjs b/mcp-host/src/core/conversation/persistence/__tests__/fixtures/tsResolveHook.cjs
new file mode 100644
index 000000000..ee6b56ea1
--- /dev/null
+++ b/mcp-host/src/core/conversation/persistence/__tests__/fixtures/tsResolveHook.cjs
@@ -0,0 +1,19 @@
+// Lets a child `node --experimental-transform-types` process require the
+// TypeScript sources directly: resolves extensionless relative imports made
+// from a .ts file to `<path>.ts` or `<path>/index.ts`.
+const fs = require('node:fs')
+const path = require('node:path')
+const { registerHooks } = require('node:module')
+const { fileURLToPath } = require('node:url')
+
+registerHooks({
+ resolve(specifier, context, nextResolve) {
+ if (specifier.startsWith('.') && context.parentURL?.endsWith('.ts')) {
+ const base = path.resolve(path.dirname(fileURLToPath(context.parentURL)), specifier)
+ for (const candidate of [`${base}.ts`, path.join(base, 'index.ts')]) {
+ if (fs.existsSync(candidate)) return nextResolve(candidate, context)
+ }
+ }
+ return nextResolve(specifier, context)
+ },
+})
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
new file mode 100644
index 000000000..7c46ecada
--- /dev/null
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
@@ -0,0 +1,130 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { spawnSync } from 'node:child_process'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+
+// Runs the real factory, PersistQueue and exitOnPersistFailure in a child
+// Node process and reads that process's exit code. The worker thread is a
+// JS stand-in: it acknowledges writes by appending them to a file, fails the
+// `fail` op and exits on `crash`.
+
+const SRC = path.resolve(__dirname, '..', '..', '..', '..')
+const HOOK = path.join(__dirname, 'fixtures', 'tsResolveHook.cjs')
+const roots: string[] = []
+
+afterEach(() => {
+ for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true })
+})
+
+const WORKER = `
+const fs = require('node:fs')
+const { parentPort, workerData } = require('node:worker_threads')
+parentPort.on('message', msg => {
+ const kind = msg.op.kind
+ if (kind === 'write') {
+ fs.appendFileSync(workerData.dbPath, msg.op.value + '\\n')
+ parentPort.postMessage({ id: msg.id, ok: true, result: null })
+ }
+ if (kind === 'fail') {
+ const error = { code: 'SQLITE_CONSTRAINT', message: 'FOREIGN KEY constraint failed' }
+ parentPort.postMessage({ id: msg.id, ok: false, error })
+ }
+ if (kind === 'crash') process.exit(3)
+ if (kind === 'shutdown') {
+ parentPort.postMessage({ id: msg.id, ok: true, result: null })
+ process.exit(0)
+ }
+})
+`
+
+const CHILD = `
+const path = require('node:path')
+const [srcRoot, workerPath, dataPath, scenario] = process.argv.slice(2)
+const persistence = path.join(srcRoot, 'core/conversation/persistence')
+const { createConversationStore } = require(path.join(persistence, 'conversationStoreFactory.ts'))
+const { exitOnPersistFailure } = require(path.join(persistence, 'persistFailureExit.ts'))
+const handle = createConversationStore({
+ mode: 'sqlite',
+ dbPath: dataPath,
+ cacheSize: 10,
+ syncTimeoutMs: 5000,
+ asyncTimeoutMs: 5000,
+ checkpointEveryWrites: 100,
+ heartbeatMs: 60000,
+ workerScriptPath: workerPath,
+ onPersistFailure: exitOnPersistFailure(() => false),
+})
+const queue = handle.persistQueue
+;(async () => {
+ await queue.enqueueSync({ kind: 'write', value: 'first' })
+ console.log('FIRST_WRITE_ACKED')
+ if (scenario === 'alive') {
+ queue.enqueueAsync('s1', { kind: 'write', value: 'second' })
+ await handle.shutdown()
+ console.log('SHUTDOWN_CLEAN')
+ return
+ }
+ queue.enqueueAsync('s1', { kind: scenario === 'async-write-failure' ? 'fail' : 'crash' })
+ setTimeout(() => {
+ console.log('STILL_RUNNING')
+ process.exit(0)
+ }, 5000)
+})()
+`
+
+function runChild(scenario: 'alive' | 'async-write-failure' | 'worker-exit') {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'persist-failure-exit-'))
+ roots.push(root)
+ const worker = path.join(root, 'worker.cjs')
+ const child = path.join(root, 'child.cjs')
+ const data = path.join(root, 'writes.log')
+ fs.writeFileSync(worker, WORKER)
+ fs.writeFileSync(child, CHILD)
+ const result = spawnSync(
+ process.execPath,
+ [
+ '--no-warnings',
+ '--experimental-transform-types',
+ '--require',
+ HOOK,
+ child,
+ SRC,
+ worker,
+ data,
+ scenario,
+ ],
+ { encoding: 'utf8', timeout: 20_000 }
+ )
+ const writes = fs.existsSync(data)
+ ? fs.readFileSync(data, 'utf8').split('\n').filter(Boolean)
+ : []
+ return { status: result.status, out: `${result.stdout}${result.stderr}`, writes }
+}
+
+describe('exitOnPersistFailure in a Host process', () => {
+ it('keeps the process running and persists writes while the worker is healthy', () => {
+ const run = runChild('alive')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ expect(run.out).toContain('SHUTDOWN_CLEAN')
+ expect(run.writes).toEqual(['first', 'second'])
+ expect(run.status).toBe(0)
+ })
+
+ it('exits 1 when an async write fails', () => {
+ const run = runChild('async-write-failure')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ expect(run.out).toContain('Conversation store can no longer persist writes')
+ expect(run.out).not.toContain('STILL_RUNNING')
+ expect(run.status).toBe(1)
+ })
+
+ it('exits 1 when the worker exits unexpectedly', () => {
+ const run = runChild('worker-exit')
+ expect(run.out).toContain('FIRST_WRITE_ACKED')
+ expect(run.out).toContain('worker exited; writes are rejected')
+ expect(run.out).toContain('Conversation store can no longer persist writes')
+ expect(run.out).not.toContain('STILL_RUNNING')
+ expect(run.status).toBe(1)
+ })
+})
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
index 484df5044..0239e3b30 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
@@ -97,4 +97,171 @@ describe('PersistQueue', () => {
worker.emit('exit', 137)
await expect(promise).rejects.toThrow(/db worker exited/)
})
+
+ it('rejects async writes admitted after the worker exited instead of dropping them', async () => {
+ const worker = new FakeWorker()
+ const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ // Liveness witness: while the worker is alive the async write is posted.
+ expect(worker.sent).toHaveLength(1)
+
+ worker.emit('exit', 137)
+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(/db worker exited \(code=137\)/)
+ expect(worker.sent).toHaveLength(1)
+ })
+
+ it('fails the next write on the same queue after an async write failed', async () => {
+ const worker = new FakeWorker()
+ const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ // Witness: a sync write before any failure resolves.
+ const first = q.enqueueSync({ kind: 'ping' })
+ worker.emit('message', {
+ id: worker.sent[0].id,
+ ok: true,
+ result: 'pong',
+ } satisfies WorkerReply)
+ await expect(first).resolves.toBe('pong')
+
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ worker.emit('message', {
+ id: worker.sent[1].id,
+ ok: false,
+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },
+ } satisfies WorkerReply)
+ await new Promise(r => setImmediate(r))
+
+ await expect(q.enqueueSync({ kind: 'ping' })).rejects.toThrow(
+ /async write failed \(sessionKey=s1\): database or disk is full/
+ )
+ expect(() => q.enqueueAsync('s2', { kind: 'ping' })).toThrow(/database or disk is full/)
+ })
+
+ it('calls onFailure once when the worker exits unexpectedly', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ onFailure: err => failures.push(err),
+ })
+ // Liveness witness: with the worker alive a write is acknowledged and
+ // onFailure has not fired.
+ const ok = q.enqueueSync({ kind: 'ping' })
+ worker.emit('message', {
+ id: worker.sent[0].id,
+ ok: true,
+ result: 'pong',
+ } satisfies WorkerReply)
+ await expect(ok).resolves.toBe('pong')
+ expect(failures).toHaveLength(0)
+
+ worker.emit('exit', 1)
+ expect(failures.map(e => e.message)).toEqual(['db worker exited (code=1)'])
+ // A later rejected write does not report a second failure.
+ expect(() => q.enqueueAsync('s1', { kind: 'ping' })).toThrow(/db worker exited/)
+ expect(failures).toHaveLength(1)
+ })
+
+ it('calls onFailure once when an async write fails', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ onFailure: err => failures.push(err),
+ })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ q.enqueueAsync('s2', { kind: 'ping' })
+ await Promise.resolve()
+ expect(worker.sent).toHaveLength(2)
+
+ // Liveness witness: an acknowledged async write does not fire onFailure.
+ worker.emit('message', {
+ id: worker.sent[0].id,
+ ok: true,
+ result: 'pong',
+ } satisfies WorkerReply)
+ await new Promise(r => setImmediate(r))
+ expect(failures).toHaveLength(0)
+
+ worker.emit('message', {
+ id: worker.sent[1].id,
+ ok: false,
+ error: { code: 'SQLITE_FULL', message: 'database or disk is full' },
+ } satisfies WorkerReply)
+ await new Promise(r => setImmediate(r))
+ expect(failures.map(e => e.message)).toEqual([
+ 'async write failed (sessionKey=s2): database or disk is full',
+ ])
+
+ // A worker exit after the failure does not report a second one.
+ worker.emit('exit', 1)
+ expect(failures).toHaveLength(1)
+ })
+
+ it('does not call onFailure for the worker exit that follows a shutdown ACK', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const exits: number[] = []
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 500,
+ onFailure: err => failures.push(err),
+ onExit: code => exits.push(code),
+ })
+ const closing = q.close()
+ await new Promise(r => setImmediate(r))
+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')
+ expect(shutdown).toBeDefined()
+ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)
+ // FakeWorker.terminate() emits the exit.
+ await closing
+ // Liveness witness: the exit was observed.
+ expect(exits).toEqual([0])
+ expect(failures).toHaveLength(0)
+ })
+
+ it('waits for the worker shutdown ACK before terminating it', async () => {
+ const worker = new FakeWorker()
+ let terminateCalls = 0
+ worker.terminate = () => {
+ terminateCalls += 1
+ }
+ const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ const closing = q.close()
+ await new Promise(r => setImmediate(r))
+
+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')
+ // Witness: the shutdown op was posted.
+ expect(shutdown).toBeDefined()
+ expect(terminateCalls).toBe(0)
+
+ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)
+ await closing
+ expect(terminateCalls).toBe(1)
+ })
+
+ it('drains queued async writes before posting shutdown and rejects close on a failed one', async () => {
+ const worker = new FakeWorker()
+ const q = new PersistQueue(worker, { syncTimeoutMs: 500, asyncTimeoutMs: 500 })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ const closing = q.close()
+ await new Promise(r => setImmediate(r))
+ // The queued write is still in flight, so shutdown is not posted yet.
+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])
+
+ worker.emit('message', {
+ id: worker.sent[0].id,
+ ok: false,
+ error: { code: 'SQLITE_IOERR', message: 'disk I/O error' },
+ } satisfies WorkerReply)
+ await new Promise(r => setImmediate(r))
+ const shutdown = worker.sent.find(m => m.op.kind === 'shutdown')
+ expect(shutdown).toBeDefined()
+ worker.emit('message', { id: shutdown!.id, ok: true, result: null } satisfies WorkerReply)
+ await expect(closing).rejects.toThrow(/disk I\/O error/)
+ })
})
diff --git a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
index 200a83610..dbe4dcbad 100644
--- a/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
+++ b/mcp-host/src/core/conversation/persistence/conversationStoreFactory.ts
@@ -10,6 +10,7 @@
*/
import * as path from 'node:path'
import { Worker } from 'node:worker_threads'
+import { logger } from '../../../logger'
import { type ConversationStore, InMemoryConversationStore } from '../conversationStore'
import { DualConversationStore } from './dualConversationStore'
import { PersistQueue, type WorkerLike } from './persistQueue'
@@ -35,6 +36,11 @@ export interface ConversationStoreFactoryOptions {
pendingApprovalTtlMs?: number
/** Optional override for the compiled worker script path (used by tests). */
workerScriptPath?: string
+ /**
+ * Called once when the persist queue becomes unusable (unexpected worker
+ * exit or a failed async write). `main.ts` ends the process here.
+ */
+ onPersistFailure?: (err: Error) => void
}
export interface ConversationStoreHandle {
@@ -90,11 +96,12 @@ export function createConversationStore(
syncTimeoutMs: opts.syncTimeoutMs,
asyncTimeoutMs: opts.asyncTimeoutMs,
onTransportError: err => {
- console.error('[ConversationStore] worker transport error:', err)
+ logger.error({ err }, '[ConversationStore] worker transport error')
},
onExit: code => {
- console.warn(`[ConversationStore] worker exited (code=${code})`)
+ logger.error({ code }, '[ConversationStore] worker exited; writes are rejected')
},
+ onFailure: opts.onPersistFailure,
})
const sqliteStore = new SqliteConversationStore(persistQueue, {
diff --git a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
new file mode 100644
index 000000000..d93f737b2
--- /dev/null
+++ b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
@@ -0,0 +1,18 @@
+import { logger } from '../../../logger'
+
+/**
+ * `onPersistFailure` handler for the Host process. A persist queue that can
+ * no longer write never recovers in-process, so the process exits 1 and the
+ * kubelet restarts the container. During shutdown `close()` reports the
+ * failure instead.
+ */
+export function exitOnPersistFailure(isShuttingDown: () => boolean): (err: Error) => void {
+ return err => {
+ if (isShuttingDown()) return
+ logger.error(
+ { err },
+ '[Main] Conversation store can no longer persist writes; exiting with code 1'
+ )
+ process.exit(1)
+ }
+}
diff --git a/mcp-host/src/core/conversation/persistence/persistQueue.ts b/mcp-host/src/core/conversation/persistence/persistQueue.ts
index 74afe557a..9aa255f1b 100644
--- a/mcp-host/src/core/conversation/persistence/persistQueue.ts
+++ b/mcp-host/src/core/conversation/persistence/persistQueue.ts
@@ -4,9 +4,12 @@
* - `enqueueSync(op)` resolves when the worker ACKs (or rejects on error /
* timeout). Used for IronClaw-critical writes: suspendForApproval, approve,
* deny, completeTurn with billed usage.
- * - `enqueueAsync(sessionKey, op)` is fire-and-forget but preserves the
+ * - `enqueueAsync(sessionKey, op)` does not wait for the ACK but preserves the
* `sessionKey`-scoped ordering chain (per `SessionProcessor`'s FIFO
- * invariant). Errors are logged.
+ * invariant). A failed async write marks the queue failed: every later
+ * write rejects with that failure and `onFailure` fires once, so the Host
+ * stops instead of accepting writes it can no longer persist. The same
+ * holds after an unexpected worker exit.
*
* The implementation is transport-agnostic: callers inject a `WorkerLike`
* (typically a `node:worker_threads.Worker`, but tests inject an in-process
@@ -21,6 +24,7 @@ import type {
WorkerReply,
} from '../../../db/worker/protocol'
import { HEARTBEAT_ID } from '../../../db/worker/protocol'
+import { logger } from '../../../logger'
export interface WorkerLike {
postMessage(msg: WorkerMessage): void
@@ -39,6 +43,11 @@ export interface PersistQueueOptions {
onTransportError?: (err: Error) => void
/** Called when the worker exits. Drives reconnect/degraded handling in main. */
onExit?: (code: number) => void
+ /**
+ * Called once, with the first failure that makes the queue unusable: an
+ * async write failed, or the worker exited before ACKing `shutdown`.
+ */
+ onFailure?: (err: Error) => void
}
interface PendingEntry {
@@ -51,7 +60,13 @@ interface PendingEntry {
export class PersistQueue {
private readonly pending = new Map<string, PendingEntry>()
private readonly writeChain = new Map<string, Promise<void>>()
+ /** Set once admission closes (graceful `close()` or worker exit). */
private closed = false
+ private exited = false
+ /** The worker ACKed `shutdown`; its exit that follows is expected. */
+ private shutdownAcked = false
+ /** First failure that makes further writes unsafe; never cleared. */
+ private failure: Error | null = null
constructor(
private readonly worker: WorkerLike,
@@ -63,6 +78,7 @@ export class PersistQueue {
})
this.worker.on('exit', code => {
this.closed = true
+ this.exited = true
// Flush pending with an explicit error so callers don't hang.
for (const entry of this.pending.values()) {
clearTimeout(entry.timeoutHandle)
@@ -72,24 +88,31 @@ export class PersistQueue {
}
this.pending.clear()
if (this.opts.onExit) this.opts.onExit(code)
+ if (!this.shutdownAcked) this.recordFailure(new Error(`db worker exited (code=${code})`))
})
}
/**
- * Fire-and-forget write. Errors are logged. Ordering is preserved per
- * `sessionKey` so a `startTurn` precedes its `completeTurn` on disk
- * regardless of scheduling jitter. Returns immediately.
+ * Write without waiting for the ACK. Ordering is preserved per `sessionKey`
+ * so a `startTurn` precedes its `completeTurn` on disk regardless of
+ * scheduling jitter. Throws when the queue is closed or an earlier write
+ * failed; a failure of this write is recorded and surfaces on the next
+ * write, through `onFailure` and from `close()`.
*/
enqueueAsync(sessionKey: string, op: WorkerOp): void {
- if (this.closed) return
+ this.assertAdmissible()
const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()
const next = prev
.then(() => this.send(op, this.opts.asyncTimeoutMs))
.then(() => {
/* discard the result — async path */
})
- .catch(err => {
- console.error(`[PersistQueue] async write failed (sessionKey=${sessionKey}):`, err)
+ .catch((err: Error) => {
+ const failure = new Error(
+ `async write failed (sessionKey=${sessionKey}): ${err.message ?? String(err)}`
+ )
+ logger.error({ sessionKey, err }, '[PersistQueue] async write failed')
+ this.recordFailure(failure)
})
.finally(() => {
// Drop the chain reference once it settles so we don't keep arbitrarily
@@ -114,8 +137,10 @@ export class PersistQueue {
* race the async one and the snapshot reads inconsistent state.
*/
enqueueSync<T = unknown>(op: WorkerOp, sessionKey?: string): Promise<T> {
- if (this.closed) {
- return Promise.reject(new Error('PersistQueue is closed'))
+ try {
+ this.assertAdmissible()
+ } catch (err) {
+ return Promise.reject(err as Error)
}
if (sessionKey) {
const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()
@@ -183,28 +208,46 @@ export class PersistQueue {
await Promise.allSettled(chains)
}
+ /**
+ * Graceful shutdown: close admission, drain every queued write, wait for the
+ * worker to ACK `shutdown` (it closes the database before replying), then
+ * terminate it. Rejects with the first write failure, or with the shutdown
+ * failure, so the caller sees that the durable trail is incomplete.
+ */
async close(): Promise<void> {
- if (this.closed) return
+ if (this.closed && !this.exited) return
this.closed = true
await this.drain()
- try {
- this.worker.postMessage({ id: randomUUID(), op: { kind: 'shutdown' } })
- } catch {
- /* ignore — worker may already be gone */
- }
- try {
- const terminated = this.worker.terminate()
- if (terminated && typeof (terminated as Promise<number>).then === 'function') {
- await terminated
+ let shutdownError: Error | null = null
+ if (!this.exited) {
+ try {
+ await this.send({ kind: 'shutdown' }, this.opts.syncTimeoutMs)
+ } catch (err) {
+ shutdownError = err as Error
}
- } catch (err) {
- console.warn('[PersistQueue] terminate() raised:', err)
+ }
+ const terminated = this.worker.terminate()
+ if (terminated && typeof (terminated as Promise<number>).then === 'function') {
+ await terminated
}
for (const entry of this.pending.values()) {
clearTimeout(entry.timeoutHandle)
entry.reject(new Error('PersistQueue closed'))
}
this.pending.clear()
+ if (this.failure) throw this.failure
+ if (shutdownError) throw shutdownError
+ }
+
+ private recordFailure(err: Error): void {
+ if (this.failure) return
+ this.failure = err
+ if (this.opts.onFailure) this.opts.onFailure(err)
+ }
+
+ private assertAdmissible(): void {
+ if (this.failure) throw this.failure
+ if (this.closed) throw new Error('PersistQueue is closed')
}
private send<T>(op: WorkerOp, timeoutMs: number): Promise<T> {
@@ -244,6 +287,9 @@ export class PersistQueue {
clearTimeout(entry.timeoutHandle)
this.pending.delete(reply.id)
if (reply.ok) {
+ // Set synchronously: the worker's exit can be delivered before the
+ // awaiting `close()` resumes.
+ if (entry.op.kind === 'shutdown') this.shutdownAcked = true
entry.resolve(reply.result)
} else {
const err = new Error(reply.error?.message ?? 'unknown worker error')
diff --git a/mcp-host/src/db/worker/dbWorker.ts b/mcp-host/src/db/worker/dbWorker.ts
index 41939934a..eb247dd95 100644
--- a/mcp-host/src/db/worker/dbWorker.ts
+++ b/mcp-host/src/db/worker/dbWorker.ts
@@ -84,12 +84,15 @@ async function main(): Promise<void> {
writesSinceCheckpoint = 0
}
}
- const reply: WorkerReply = { id: msg.id, ok: true, result }
- parentPort.postMessage(reply)
if (msg.op.kind === 'shutdown') {
+ // Close (and checkpoint) before the ACK: the main thread terminates
+ // the worker as soon as it sees the reply.
db.close()
+ parentPort.postMessage({ id: msg.id, ok: true, result } satisfies WorkerReply)
process.exit(0)
}
+ const reply: WorkerReply = { id: msg.id, ok: true, result }
+ parentPort.postMessage(reply)
} catch (err) {
const e = err as { code?: string; message?: string; stack?: string }
const reply: WorkerReply = {
diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.ts
index e47331bd2..50fcfc7db 100644
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@ -45,6 +45,7 @@ import {
SqliteColdStartLoader,
createConversationStore,
} from './core/conversation/persistence'
+import { exitOnPersistFailure } from './core/conversation/persistence/persistFailureExit'
import { validateApprovalConfig } from './core/extensions'
import type { ApprovalDecision } from './core/extensions/approvalTypes'
import { BasicSafety } from './core/safety/safety'
@@ -1628,6 +1629,7 @@ async function initializeConversationStore(): Promise<ConversationStoreHandle |
heartbeatMs: config.dbWorkerHeartbeatMs,
pendingApprovalTtlMs: config.pendingApprovalTtlMs,
barrierMode,
+ onPersistFailure: exitOnPersistFailure(() => isShuttingDown),
})
agent.setConversationStore(handle.store)