Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): keep slow writes pending, flush reporters before exit

Alfredo Lopez87ac332c53a9
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 6206db95d..d0f7b6125 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.571",
+ "version": "0.1.572",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.571",
+ "version": "0.1.572",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 5a3eab112..a0e5ddbe0 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.571",
+ "version": "0.1.572",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
index 526d965a2..100ebab20 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts
@@ -54,7 +54,10 @@ const handle = createConversationStore({
checkpointEveryWrites: 100,
heartbeatMs: 60000,
workerScriptPath: workerPath,
- onPersistFailure: exitOnPersistFailure(() => false),
+ onPersistFailure: exitOnPersistFailure(() => false, {
+ beforeExit: async () => console.log('PRE_EXIT_FLUSH'),
+ beforeExitTimeoutMs: 1000,
+ }),
})
const queue = handle.persistQueue
;(async () => {
@@ -116,6 +119,7 @@ describe('exitOnPersistFailure in a Host process', () => {
const run = runChild('async-write-failure')
expect(run.out).toContain('FIRST_WRITE_ACKED')
expect(run.out).toContain('Conversation store can no longer persist writes')
+ expect(run.out).toContain('PRE_EXIT_FLUSH')
expect(run.out).not.toContain('STILL_RUNNING')
expect(run.status).toBe(1)
})
@@ -131,18 +135,95 @@ describe('exitOnPersistFailure in a Host process', () => {
})
describe('exitOnPersistFailure during shutdown', () => {
- it('does not exit while the Host is shutting down', () => {
+ it('does not exit while the Host is shutting down', async () => {
const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)
+ const beforeExit = vi.fn(async () => undefined)
try {
let shuttingDown = true
- const handler = exitOnPersistFailure(() => shuttingDown)
+ const handler = exitOnPersistFailure(() => shuttingDown, {
+ beforeExit,
+ beforeExitTimeoutMs: 1000,
+ })
handler(new Error('db worker exited (code=1)'))
+ expect(beforeExit).not.toHaveBeenCalled()
expect(exit).not.toHaveBeenCalled()
// Liveness witness: the same handler exits once shutdown is not running.
shuttingDown = false
handler(new Error('db worker exited (code=1)'))
- expect(exit).toHaveBeenCalledExactlyOnceWith(1)
+ await vi.waitFor(() => expect(exit).toHaveBeenCalledExactlyOnceWith(1))
+ expect(beforeExit).toHaveBeenCalledOnce()
+ } finally {
+ exit.mockRestore()
+ }
+ })
+})
+
+describe('exitOnPersistFailure pre-exit flush', () => {
+ function spyExit(events: string[]) {
+ return vi.spyOn(process, 'exit').mockImplementation(code => {
+ events.push(`exit ${code}`)
+ return undefined as never
+ })
+ }
+
+ it('flushes before exiting 1', async () => {
+ const events: string[] = []
+ const exit = spyExit(events)
+ try {
+ const handler = exitOnPersistFailure(() => false, {
+ beforeExit: async () => {
+ events.push('flush start')
+ await new Promise(r => setTimeout(r, 20))
+ events.push('flushed')
+ },
+ beforeExitTimeoutMs: 1000,
+ })
+ handler(new Error('async write failed'))
+ // A second failure during the flush neither flushes nor exits again.
+ handler(new Error('db worker exited (code=1)'))
+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())
+ expect(events).toEqual(['flush start', 'flushed', 'exit 1'])
+ } finally {
+ exit.mockRestore()
+ }
+ })
+
+ it('exits 1 when the flush hangs past the deadline', async () => {
+ const events: string[] = []
+ const exit = spyExit(events)
+ try {
+ const handler = exitOnPersistFailure(() => false, {
+ beforeExit: () => {
+ events.push('flush start')
+ return new Promise<void>(() => undefined)
+ },
+ beforeExitTimeoutMs: 30,
+ })
+ const started = Date.now()
+ handler(new Error('async write failed'))
+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())
+ expect(Date.now() - started).toBeLessThan(500)
+ expect(events).toEqual(['flush start', 'exit 1'])
+ } finally {
+ exit.mockRestore()
+ }
+ })
+
+ it('exits 1 when the flush fails', async () => {
+ const events: string[] = []
+ const exit = spyExit(events)
+ try {
+ const handler = exitOnPersistFailure(() => false, {
+ beforeExit: async () => {
+ events.push('flush start')
+ throw new Error('usage endpoint unreachable')
+ },
+ beforeExitTimeoutMs: 1000,
+ })
+ handler(new Error('async write failed'))
+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())
+ expect(events).toEqual(['flush start', 'exit 1'])
} finally {
exit.mockRestore()
}
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
index c0cf8dd4c..80365eebe 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts
@@ -232,6 +232,58 @@ describe('PersistQueue', () => {
expect(failures).toHaveLength(1)
})
+ it('keeps a slow async write pending past asyncTimeoutMs and its key ordered until the ACK', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 20,
+ onFailure: err => failures.push(err),
+ })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ // Liveness witness: the first write was posted.
+ expect(worker.sent).toHaveLength(1)
+ await new Promise(r => setTimeout(r, 80))
+ // Past the threshold: still pending, the second write still queued behind it.
+ expect(failures).toHaveLength(0)
+ expect(worker.sent).toHaveLength(1)
+
+ worker.emit('message', { id: worker.sent[0].id, ok: true, result: null } satisfies WorkerReply)
+ await new Promise(r => setImmediate(r))
+ expect(worker.sent).toHaveLength(2)
+ worker.emit('message', { id: worker.sent[1].id, ok: true, result: null } satisfies WorkerReply)
+ await q.drainSessionKey('s1')
+ expect(failures).toHaveLength(0)
+ // A later write on the same key is still admitted.
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ expect(worker.sent).toHaveLength(3)
+ })
+
+ it('bounds the close() drain of an unacknowledged async write by asyncTimeoutMs', async () => {
+ const worker = new FakeWorker()
+ const failures: Error[] = []
+ const exits: number[] = []
+ worker.on('exit', code => exits.push(code))
+ const q = new PersistQueue(worker, {
+ syncTimeoutMs: 500,
+ asyncTimeoutMs: 20,
+ onFailure: err => failures.push(err),
+ })
+ q.enqueueAsync('s1', { kind: 'ping' })
+ await Promise.resolve()
+ expect(worker.sent).toHaveLength(1)
+ const started = Date.now()
+ await expect(q.close()).rejects.toThrow(/not acknowledged within 20ms of close/)
+ expect(Date.now() - started).toBeLessThan(400)
+ // The worker was terminated and the failure reported once.
+ expect(exits).toEqual([0])
+ expect(failures).toHaveLength(1)
+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])
+ })
+
it('reports neither onFailure nor onExit for the worker exit that follows a shutdown ACK', async () => {
const worker = new FakeWorker()
const failures: Error[] = []
diff --git a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
index d93f737b2..4a7591b43 100644
--- a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
+++ b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts
@@ -1,18 +1,51 @@
import { logger } from '../../../logger'
+export interface PersistFailureExitOptions {
+ /**
+ * Best-effort work before the exit, e.g. flushing buffered usage events.
+ * A rejection is logged; the process exits 1 either way.
+ */
+ beforeExit: () => Promise<void>
+ /** The exit happens after this long even if `beforeExit` has not settled. */
+ beforeExitTimeoutMs: number
+}
+
/**
* `onPersistFailure` handler for the Host process. A persist queue that can
* no longer write never recovers in-process, so the process exits 1 and the
- * kubelet restarts the container. During shutdown `close()` reports the
- * failure instead.
+ * kubelet restarts the container, after a bounded `beforeExit`. During
+ * shutdown `close()` reports the failure instead.
*/
-export function exitOnPersistFailure(isShuttingDown: () => boolean): (err: Error) => void {
+export function exitOnPersistFailure(
+ isShuttingDown: () => boolean,
+ opts: PersistFailureExitOptions
+): (err: Error) => void {
+ let exiting = false
return err => {
- if (isShuttingDown()) return
+ if (exiting || isShuttingDown()) return
+ exiting = true
logger.error(
{ err },
'[Main] Conversation store can no longer persist writes; exiting with code 1'
)
- process.exit(1)
+ let timer: ReturnType<typeof setTimeout> | undefined
+ const expired = new Promise<void>(resolve => {
+ timer = setTimeout(() => {
+ logger.warn(
+ { timeoutMs: opts.beforeExitTimeoutMs },
+ '[Main] Pre-exit flush did not finish in time'
+ )
+ resolve()
+ }, opts.beforeExitTimeoutMs)
+ })
+ const flushed = Promise.resolve()
+ .then(opts.beforeExit)
+ .catch((flushErr: unknown) => {
+ logger.warn({ err: flushErr }, '[Main] Pre-exit flush failed')
+ })
+ void Promise.race([flushed, expired]).then(() => {
+ clearTimeout(timer)
+ process.exit(1)
+ })
}
}
diff --git a/mcp-host/src/core/conversation/persistence/persistQueue.ts b/mcp-host/src/core/conversation/persistence/persistQueue.ts
index b089fc968..d92dd13a9 100644
--- a/mcp-host/src/core/conversation/persistence/persistQueue.ts
+++ b/mcp-host/src/core/conversation/persistence/persistQueue.ts
@@ -6,10 +6,12 @@
* deny, completeTurn with billed usage.
* - `enqueueAsync(sessionKey, op)` does not wait for the ACK but preserves the
* `sessionKey`-scoped ordering chain (per `SessionProcessor`'s FIFO
- * invariant). A failed async write marks the queue failed: every later
- * write rejects with that failure and `onFailure` fires once, so the Host
- * stops instead of accepting writes it can no longer persist. The same
- * holds after an unexpected worker exit.
+ * invariant). An async write stays pending until the worker answers or
+ * exits; a slow ACK is only logged, so its outcome is never guessed. A
+ * failed async write marks the queue failed: every later write rejects
+ * with that failure and `onFailure` fires once, so the Host stops instead
+ * of accepting writes it can no longer persist. The same holds after an
+ * unexpected worker exit.
*
* The implementation is transport-agnostic: callers inject a `WorkerLike`
* (typically a `node:worker_threads.Worker`, but tests inject an in-process
@@ -36,6 +38,10 @@ export interface WorkerLike {
export interface PersistQueueOptions {
syncTimeoutMs: number
+ /**
+ * An async write not ACKed within this time is logged and kept pending.
+ * `close()` waits this long for the queued async writes to settle.
+ */
asyncTimeoutMs: number
/** Called whenever a heartbeat reply is observed (id === HEARTBEAT_ID). */
onHeartbeat?: (payload: HeartbeatPayload) => void
@@ -110,7 +116,7 @@ export class PersistQueue {
this.assertAdmissible()
const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()
const next = prev
- .then(() => this.sendUnlessFailed(op, this.opts.asyncTimeoutMs))
+ .then(() => this.sendUnlessFailed(op, this.opts.asyncTimeoutMs, 'warn'))
.then(() => {
/* discard the result — async path */
})
@@ -235,7 +241,9 @@ export class PersistQueue {
/**
* Graceful shutdown: close admission, drain the keyed writes in
- * `writeChain` (async writes and sync writes with a `sessionKey`), wait for
+ * `writeChain` (async writes and sync writes with a `sessionKey`) for at
+ * most `asyncTimeoutMs` (past it, a failure is recorded and the worker is
+ * terminated without `shutdown`), wait for
* the worker to ACK `shutdown` (it closes the database before replying),
* then terminate it. A sync write without a `sessionKey` is not in
* `writeChain`; if the worker is still retrying it on a busy database,
@@ -251,8 +259,12 @@ export class PersistQueue {
private async shutdown(): Promise<void> {
this.closed = true
- await this.drain()
- if (!this.exited) {
+ if (!(await this.drainWithin(this.opts.asyncTimeoutMs))) {
+ // The worker is stuck on a write; terminate() below rejects it.
+ this.recordFailure(
+ new Error(`async writes not acknowledged within ${this.opts.asyncTimeoutMs}ms of close()`)
+ )
+ } else if (!this.exited) {
try {
await this.send({ kind: 'shutdown' }, this.opts.syncTimeoutMs)
} catch (err) {
@@ -273,6 +285,18 @@ export class PersistQueue {
if (this.failure) throw this.failure
}
+ private async drainWithin(ms: number): Promise<boolean> {
+ let timer: ReturnType<typeof setTimeout> | undefined
+ const expired = new Promise<boolean>(resolve => {
+ timer = setTimeout(() => resolve(false), ms)
+ })
+ try {
+ return await Promise.race([this.drain().then(() => true), expired])
+ } finally {
+ clearTimeout(timer)
+ }
+ }
+
private recordFailure(err: Error): void {
if (this.failure) return
this.failure = err
@@ -288,19 +312,38 @@ export class PersistQueue {
* Post a write that waited on its `sessionKey` chain. The queue may have
* failed while it waited, and admission was checked before that.
*/
- private sendUnlessFailed<T>(op: WorkerOp, timeoutMs: number): Promise<T> {
+ private sendUnlessFailed<T>(
+ op: WorkerOp,
+ timeoutMs: number,
+ onTimeout: 'reject' | 'warn' = 'reject'
+ ): Promise<T> {
if (this.failure) return Promise.reject(this.failure)
- return this.send<T>(op, timeoutMs)
+ return this.send<T>(op, timeoutMs, onTimeout)
}
- private send<T>(op: WorkerOp, timeoutMs: number | null): Promise<T> {
+ /**
+ * `onTimeout: 'warn'` logs once when `timeoutMs` elapses and keeps waiting
+ * for the ACK or the worker exit.
+ */
+ private send<T>(
+ op: WorkerOp,
+ timeoutMs: number | null,
+ onTimeout: 'reject' | 'warn' = 'reject'
+ ): Promise<T> {
const id = randomUUID()
return new Promise<T>((resolve, reject) => {
const timeoutHandle =
timeoutMs === null
? undefined
: setTimeout(() => {
- if (this.pending.delete(id)) {
+ if (onTimeout === 'warn') {
+ if (this.pending.has(id)) {
+ logger.warn(
+ { op: op.kind, timeoutMs },
+ '[PersistQueue] async write not acknowledged yet; still waiting'
+ )
+ }
+ } else if (this.pending.delete(id)) {
reject(new Error(`db worker timeout (${timeoutMs}ms) for op=${op.kind}`))
}
}, timeoutMs)
diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.ts
index 86bf7cbb2..e9800eea8 100644
--- a/mcp-host/src/main.ts
+++ b/mcp-host/src/main.ts
@@ -251,6 +251,8 @@ let agent: AgentStateMachine | null = null
let cronScheduler: CronScheduler | null = null
let sessionProcessor: SessionProcessor | null = null
let isShuttingDown = false
+/** Bound on the reporter flush that precedes a persist-failure exit 1. */
+const PERSIST_FAILURE_FLUSH_TIMEOUT_MS = 5_000
let mcpInitializationGeneration = 0
// Stage 3 (stateless-agents) — heartbeat emitter + DRAINING fence holder.
let statelessHeartbeat: StatelessHeartbeat | null = null
@@ -1629,7 +1631,18 @@ async function initializeConversationStore(): Promise<ConversationStoreHandle |
heartbeatMs: config.dbWorkerHeartbeatMs,
pendingApprovalTtlMs: config.pendingApprovalTtlMs,
barrierMode,
- onPersistFailure: exitOnPersistFailure(() => isShuttingDown),
+ onPersistFailure: exitOnPersistFailure(() => isShuttingDown, {
+ // Flush buffered usage and governed-run events before the exit 1.
+ // Marking the Host as shutting down keeps a SIGTERM during the flush
+ // from running the graceful path, which exits 0.
+ beforeExit: async () => {
+ isShuttingDown = true
+ usageReporter?.stop()
+ governedRunReporter?.stop()
+ await Promise.all([usageReporter?.drain(), governedRunReporter?.drain()])
+ },
+ beforeExitTimeoutMs: PERSIST_FAILURE_FLUSH_TIMEOUT_MS,
+ }),
})
agent.setConversationStore(handle.store)
diff --git a/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts b/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts
index 1dc79534a..dcc6848da 100644
--- a/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts
+++ b/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts
@@ -99,6 +99,24 @@ describe('conversation-store files are not agent-accessible', () => {
expect(hits).not.toContain('alias.md')
})
+ it('list skips a link that cannot be resolved and keeps its siblings', async () => {
+ fs.symlinkSync('loop', path.join(root, 'loop'))
+ fs.symlinkSync('notes.md/child', path.join(root, 'through-file'))
+ fs.symlinkSync('missing', path.join(root, 'dangling'))
+ const workspace = new WorkspaceService(root)
+ const names = (await workspace.list()).map(e => e.name)
+ // Witness: the ordinary files and the dangling link stay listed.
+ expect(names).toEqual(expect.arrayContaining(['mystate.db.txt', 'notes.md', 'dangling']))
+ // ELOOP and ENOTDIR omit only their own entry.
+ expect(names).not.toContain('loop')
+ expect(names).not.toContain('through-file')
+ for (const p of PROTECTED) expect(names).not.toContain(p)
+
+ const all = await workspace.listAll()
+ expect(all).toEqual(expect.arrayContaining(['mystate.db.txt', 'notes.md']))
+ for (const p of PROTECTED) expect(all).not.toContain(p)
+ })
+
it('file_read and file_write deny them, directly and through a symlink', async () => {
const reader = new FileReadTool(root)
const writer = new FileWriteTool(root)
@@ -126,17 +144,20 @@ describe('conversation-store files are not agent-accessible', () => {
return new ShellTool(root, 5000, ['PATH'])
}
- it.each(['cat state.db.pre-1.bak', 'cat state.db-wal', 'echo y > state.db.pre-2.bak'])(
- 'rejects by name: %s',
- async command => {
- const out = await shell().execute({ command })
- expect(out.is_error).toBe(true)
- expect(out.content).toContain('Command rejected')
- expect(out.content).not.toContain(STORE_BYTES)
- storeFilesUntouched()
- expect(fs.existsSync(path.join(root, 'state.db.pre-2.bak'))).toBe(false)
- }
- )
+ it.each([
+ 'cat state.db.pre-1.bak',
+ 'cat state.db-wal',
+ 'echo y > state.db.pre-2.bak',
+ 'cat state.db_old',
+ 'cat state.db2',
+ ])('rejects by name: %s', async command => {
+ const out = await shell().execute({ command })
+ expect(out.is_error).toBe(true)
+ expect(out.content).toContain('Command rejected')
+ expect(out.content).not.toContain(STORE_BYTES)
+ storeFilesUntouched()
+ expect(fs.existsSync(path.join(root, 'state.db.pre-2.bak'))).toBe(false)
+ })
it('runs commands on the lookalike and the ordinary file', async () => {
const out = await shell().execute({
diff --git a/mcp-host/src/workspace/service.ts b/mcp-host/src/workspace/service.ts
index 5c0401990..60c02708c 100644
--- a/mcp-host/src/workspace/service.ts
+++ b/mcp-host/src/workspace/service.ts
@@ -316,8 +316,9 @@ export class WorkspaceService implements Workspace {
const absPath = path.join(resolved, entry.name)
// A symlink with a harmless name can resolve to store data; the listing
// omits it as read and write already deny it. A dangling link reaches no
- // file and stays listed.
- if (await this.resolvesToProtectedPath(absPath, realWorkspaceRoot)) continue
+ // file and stays listed. A link that cannot be resolved (a loop, a path
+ // through a file) is omitted without hiding its siblings.
+ if (await this.omitFromListing(absPath, realWorkspaceRoot)) continue
let size: number | undefined
let modifiedAt: string | undefined
try {
@@ -338,16 +339,12 @@ export class WorkspaceService implements Workspace {
return results
}
- private async resolvesToProtectedPath(
- absPath: string,
- realWorkspaceRoot: string
- ): Promise<boolean> {
+ private async omitFromListing(absPath: string, realWorkspaceRoot: string): Promise<boolean> {
let realPath: string
try {
realPath = await fs.realpath(absPath)
} catch (err) {
- if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false
- throw err
+ return (err as NodeJS.ErrnoException).code !== 'ENOENT'
}
return isProtectedRealPath(realPath, realWorkspaceRoot)
}