Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): keep slow writes pending, flush reporters before exit
Alfredo Lopez87ac332c53a9
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 6206db95d..d0f7b6125 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.571",+ "version": "0.1.572","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.571",+ "version": "0.1.572","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 5a3eab112..a0e5ddbe0 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.571",+ "version": "0.1.572","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.tsindex 526d965a2..100ebab20 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistFailureExit.test.ts@@ -54,7 +54,10 @@ const handle = createConversationStore({checkpointEveryWrites: 100,heartbeatMs: 60000,workerScriptPath: workerPath,- onPersistFailure: exitOnPersistFailure(() => false),+ onPersistFailure: exitOnPersistFailure(() => false, {+ beforeExit: async () => console.log('PRE_EXIT_FLUSH'),+ beforeExitTimeoutMs: 1000,+ }),})const queue = handle.persistQueue;(async () => {@@ -116,6 +119,7 @@ describe('exitOnPersistFailure in a Host process', () => {const run = runChild('async-write-failure')expect(run.out).toContain('FIRST_WRITE_ACKED')expect(run.out).toContain('Conversation store can no longer persist writes')+ expect(run.out).toContain('PRE_EXIT_FLUSH')expect(run.out).not.toContain('STILL_RUNNING')expect(run.status).toBe(1)})@@ -131,18 +135,95 @@ describe('exitOnPersistFailure in a Host process', () => {})describe('exitOnPersistFailure during shutdown', () => {- it('does not exit while the Host is shutting down', () => {+ it('does not exit while the Host is shutting down', async () => {const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never)+ const beforeExit = vi.fn(async () => undefined)try {let shuttingDown = true- const handler = exitOnPersistFailure(() => shuttingDown)+ const handler = exitOnPersistFailure(() => shuttingDown, {+ beforeExit,+ beforeExitTimeoutMs: 1000,+ })handler(new Error('db worker exited (code=1)'))+ expect(beforeExit).not.toHaveBeenCalled()expect(exit).not.toHaveBeenCalled()// Liveness witness: the same handler exits once shutdown is not running.shuttingDown = falsehandler(new Error('db worker exited (code=1)'))- expect(exit).toHaveBeenCalledExactlyOnceWith(1)+ await vi.waitFor(() => expect(exit).toHaveBeenCalledExactlyOnceWith(1))+ expect(beforeExit).toHaveBeenCalledOnce()+ } finally {+ exit.mockRestore()+ }+ })+})++describe('exitOnPersistFailure pre-exit flush', () => {+ function spyExit(events: string[]) {+ return vi.spyOn(process, 'exit').mockImplementation(code => {+ events.push(`exit ${code}`)+ return undefined as never+ })+ }++ it('flushes before exiting 1', async () => {+ const events: string[] = []+ const exit = spyExit(events)+ try {+ const handler = exitOnPersistFailure(() => false, {+ beforeExit: async () => {+ events.push('flush start')+ await new Promise(r => setTimeout(r, 20))+ events.push('flushed')+ },+ beforeExitTimeoutMs: 1000,+ })+ handler(new Error('async write failed'))+ // A second failure during the flush neither flushes nor exits again.+ handler(new Error('db worker exited (code=1)'))+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())+ expect(events).toEqual(['flush start', 'flushed', 'exit 1'])+ } finally {+ exit.mockRestore()+ }+ })++ it('exits 1 when the flush hangs past the deadline', async () => {+ const events: string[] = []+ const exit = spyExit(events)+ try {+ const handler = exitOnPersistFailure(() => false, {+ beforeExit: () => {+ events.push('flush start')+ return new Promise<void>(() => undefined)+ },+ beforeExitTimeoutMs: 30,+ })+ const started = Date.now()+ handler(new Error('async write failed'))+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())+ expect(Date.now() - started).toBeLessThan(500)+ expect(events).toEqual(['flush start', 'exit 1'])+ } finally {+ exit.mockRestore()+ }+ })++ it('exits 1 when the flush fails', async () => {+ const events: string[] = []+ const exit = spyExit(events)+ try {+ const handler = exitOnPersistFailure(() => false, {+ beforeExit: async () => {+ events.push('flush start')+ throw new Error('usage endpoint unreachable')+ },+ beforeExitTimeoutMs: 1000,+ })+ handler(new Error('async write failed'))+ await vi.waitFor(() => expect(exit).toHaveBeenCalled())+ expect(events).toEqual(['flush start', 'exit 1'])} finally {exit.mockRestore()}diff --git a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.tsindex c0cf8dd4c..80365eebe 100644--- a/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts+++ b/mcp-host/src/core/conversation/persistence/__tests__/persistQueue.test.ts@@ -232,6 +232,58 @@ describe('PersistQueue', () => {expect(failures).toHaveLength(1)})+ it('keeps a slow async write pending past asyncTimeoutMs and its key ordered until the ACK', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 20,+ onFailure: err => failures.push(err),+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ // Liveness witness: the first write was posted.+ expect(worker.sent).toHaveLength(1)+ await new Promise(r => setTimeout(r, 80))+ // Past the threshold: still pending, the second write still queued behind it.+ expect(failures).toHaveLength(0)+ expect(worker.sent).toHaveLength(1)++ worker.emit('message', { id: worker.sent[0].id, ok: true, result: null } satisfies WorkerReply)+ await new Promise(r => setImmediate(r))+ expect(worker.sent).toHaveLength(2)+ worker.emit('message', { id: worker.sent[1].id, ok: true, result: null } satisfies WorkerReply)+ await q.drainSessionKey('s1')+ expect(failures).toHaveLength(0)+ // A later write on the same key is still admitted.+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(3)+ })++ it('bounds the close() drain of an unacknowledged async write by asyncTimeoutMs', async () => {+ const worker = new FakeWorker()+ const failures: Error[] = []+ const exits: number[] = []+ worker.on('exit', code => exits.push(code))+ const q = new PersistQueue(worker, {+ syncTimeoutMs: 500,+ asyncTimeoutMs: 20,+ onFailure: err => failures.push(err),+ })+ q.enqueueAsync('s1', { kind: 'ping' })+ await Promise.resolve()+ expect(worker.sent).toHaveLength(1)+ const started = Date.now()+ await expect(q.close()).rejects.toThrow(/not acknowledged within 20ms of close/)+ expect(Date.now() - started).toBeLessThan(400)+ // The worker was terminated and the failure reported once.+ expect(exits).toEqual([0])+ expect(failures).toHaveLength(1)+ expect(worker.sent.map(m => m.op.kind)).toEqual(['ping'])+ })+it('reports neither onFailure nor onExit for the worker exit that follows a shutdown ACK', async () => {const worker = new FakeWorker()const failures: Error[] = []diff --git a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts b/mcp-host/src/core/conversation/persistence/persistFailureExit.tsindex d93f737b2..4a7591b43 100644--- a/mcp-host/src/core/conversation/persistence/persistFailureExit.ts+++ b/mcp-host/src/core/conversation/persistence/persistFailureExit.ts@@ -1,18 +1,51 @@import { logger } from '../../../logger'+export interface PersistFailureExitOptions {+ /**+ * Best-effort work before the exit, e.g. flushing buffered usage events.+ * A rejection is logged; the process exits 1 either way.+ */+ beforeExit: () => Promise<void>+ /** The exit happens after this long even if `beforeExit` has not settled. */+ beforeExitTimeoutMs: number+}+/*** `onPersistFailure` handler for the Host process. A persist queue that can* no longer write never recovers in-process, so the process exits 1 and the- * kubelet restarts the container. During shutdown `close()` reports the- * failure instead.+ * kubelet restarts the container, after a bounded `beforeExit`. During+ * shutdown `close()` reports the failure instead.*/-export function exitOnPersistFailure(isShuttingDown: () => boolean): (err: Error) => void {+export function exitOnPersistFailure(+ isShuttingDown: () => boolean,+ opts: PersistFailureExitOptions+): (err: Error) => void {+ let exiting = falsereturn err => {- if (isShuttingDown()) return+ if (exiting || isShuttingDown()) return+ exiting = truelogger.error({ err },'[Main] Conversation store can no longer persist writes; exiting with code 1')- process.exit(1)+ let timer: ReturnType<typeof setTimeout> | undefined+ const expired = new Promise<void>(resolve => {+ timer = setTimeout(() => {+ logger.warn(+ { timeoutMs: opts.beforeExitTimeoutMs },+ '[Main] Pre-exit flush did not finish in time'+ )+ resolve()+ }, opts.beforeExitTimeoutMs)+ })+ const flushed = Promise.resolve()+ .then(opts.beforeExit)+ .catch((flushErr: unknown) => {+ logger.warn({ err: flushErr }, '[Main] Pre-exit flush failed')+ })+ void Promise.race([flushed, expired]).then(() => {+ clearTimeout(timer)+ process.exit(1)+ })}}diff --git a/mcp-host/src/core/conversation/persistence/persistQueue.ts b/mcp-host/src/core/conversation/persistence/persistQueue.tsindex b089fc968..d92dd13a9 100644--- a/mcp-host/src/core/conversation/persistence/persistQueue.ts+++ b/mcp-host/src/core/conversation/persistence/persistQueue.ts@@ -6,10 +6,12 @@* deny, completeTurn with billed usage.* - `enqueueAsync(sessionKey, op)` does not wait for the ACK but preserves the* `sessionKey`-scoped ordering chain (per `SessionProcessor`'s FIFO- * invariant). A failed async write marks the queue failed: every later- * write rejects with that failure and `onFailure` fires once, so the Host- * stops instead of accepting writes it can no longer persist. The same- * holds after an unexpected worker exit.+ * invariant). An async write stays pending until the worker answers or+ * exits; a slow ACK is only logged, so its outcome is never guessed. A+ * failed async write marks the queue failed: every later write rejects+ * with that failure and `onFailure` fires once, so the Host stops instead+ * of accepting writes it can no longer persist. The same holds after an+ * unexpected worker exit.** The implementation is transport-agnostic: callers inject a `WorkerLike`* (typically a `node:worker_threads.Worker`, but tests inject an in-process@@ -36,6 +38,10 @@ export interface WorkerLike {export interface PersistQueueOptions {syncTimeoutMs: number+ /**+ * An async write not ACKed within this time is logged and kept pending.+ * `close()` waits this long for the queued async writes to settle.+ */asyncTimeoutMs: number/** Called whenever a heartbeat reply is observed (id === HEARTBEAT_ID). */onHeartbeat?: (payload: HeartbeatPayload) => void@@ -110,7 +116,7 @@ export class PersistQueue {this.assertAdmissible()const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()const next = prev- .then(() => this.sendUnlessFailed(op, this.opts.asyncTimeoutMs))+ .then(() => this.sendUnlessFailed(op, this.opts.asyncTimeoutMs, 'warn')).then(() => {/* discard the result — async path */})@@ -235,7 +241,9 @@ export class PersistQueue {/*** Graceful shutdown: close admission, drain the keyed writes in- * `writeChain` (async writes and sync writes with a `sessionKey`), wait for+ * `writeChain` (async writes and sync writes with a `sessionKey`) for at+ * most `asyncTimeoutMs` (past it, a failure is recorded and the worker is+ * terminated without `shutdown`), wait for* the worker to ACK `shutdown` (it closes the database before replying),* then terminate it. A sync write without a `sessionKey` is not in* `writeChain`; if the worker is still retrying it on a busy database,@@ -251,8 +259,12 @@ export class PersistQueue {private async shutdown(): Promise<void> {this.closed = true- await this.drain()- if (!this.exited) {+ if (!(await this.drainWithin(this.opts.asyncTimeoutMs))) {+ // The worker is stuck on a write; terminate() below rejects it.+ this.recordFailure(+ new Error(`async writes not acknowledged within ${this.opts.asyncTimeoutMs}ms of close()`)+ )+ } else if (!this.exited) {try {await this.send({ kind: 'shutdown' }, this.opts.syncTimeoutMs)} catch (err) {@@ -273,6 +285,18 @@ export class PersistQueue {if (this.failure) throw this.failure}+ private async drainWithin(ms: number): Promise<boolean> {+ let timer: ReturnType<typeof setTimeout> | undefined+ const expired = new Promise<boolean>(resolve => {+ timer = setTimeout(() => resolve(false), ms)+ })+ try {+ return await Promise.race([this.drain().then(() => true), expired])+ } finally {+ clearTimeout(timer)+ }+ }+private recordFailure(err: Error): void {if (this.failure) returnthis.failure = err@@ -288,19 +312,38 @@ export class PersistQueue {* Post a write that waited on its `sessionKey` chain. The queue may have* failed while it waited, and admission was checked before that.*/- private sendUnlessFailed<T>(op: WorkerOp, timeoutMs: number): Promise<T> {+ private sendUnlessFailed<T>(+ op: WorkerOp,+ timeoutMs: number,+ onTimeout: 'reject' | 'warn' = 'reject'+ ): Promise<T> {if (this.failure) return Promise.reject(this.failure)- return this.send<T>(op, timeoutMs)+ return this.send<T>(op, timeoutMs, onTimeout)}- private send<T>(op: WorkerOp, timeoutMs: number | null): Promise<T> {+ /**+ * `onTimeout: 'warn'` logs once when `timeoutMs` elapses and keeps waiting+ * for the ACK or the worker exit.+ */+ private send<T>(+ op: WorkerOp,+ timeoutMs: number | null,+ onTimeout: 'reject' | 'warn' = 'reject'+ ): Promise<T> {const id = randomUUID()return new Promise<T>((resolve, reject) => {const timeoutHandle =timeoutMs === null? undefined: setTimeout(() => {- if (this.pending.delete(id)) {+ if (onTimeout === 'warn') {+ if (this.pending.has(id)) {+ logger.warn(+ { op: op.kind, timeoutMs },+ '[PersistQueue] async write not acknowledged yet; still waiting'+ )+ }+ } else if (this.pending.delete(id)) {reject(new Error(`db worker timeout (${timeoutMs}ms) for op=${op.kind}`))}}, timeoutMs)diff --git a/mcp-host/src/main.ts b/mcp-host/src/main.tsindex 86bf7cbb2..e9800eea8 100644--- a/mcp-host/src/main.ts+++ b/mcp-host/src/main.ts@@ -251,6 +251,8 @@ let agent: AgentStateMachine | null = nulllet cronScheduler: CronScheduler | null = nulllet sessionProcessor: SessionProcessor | null = nulllet isShuttingDown = false+/** Bound on the reporter flush that precedes a persist-failure exit 1. */+const PERSIST_FAILURE_FLUSH_TIMEOUT_MS = 5_000let mcpInitializationGeneration = 0// Stage 3 (stateless-agents) — heartbeat emitter + DRAINING fence holder.let statelessHeartbeat: StatelessHeartbeat | null = null@@ -1629,7 +1631,18 @@ async function initializeConversationStore(): Promise<ConversationStoreHandle |heartbeatMs: config.dbWorkerHeartbeatMs,pendingApprovalTtlMs: config.pendingApprovalTtlMs,barrierMode,- onPersistFailure: exitOnPersistFailure(() => isShuttingDown),+ onPersistFailure: exitOnPersistFailure(() => isShuttingDown, {+ // Flush buffered usage and governed-run events before the exit 1.+ // Marking the Host as shutting down keeps a SIGTERM during the flush+ // from running the graceful path, which exits 0.+ beforeExit: async () => {+ isShuttingDown = true+ usageReporter?.stop()+ governedRunReporter?.stop()+ await Promise.all([usageReporter?.drain(), governedRunReporter?.drain()])+ },+ beforeExitTimeoutMs: PERSIST_FAILURE_FLUSH_TIMEOUT_MS,+ }),})agent.setConversationStore(handle.store)diff --git a/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts b/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.tsindex 1dc79534a..dcc6848da 100644--- a/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts+++ b/mcp-host/src/workspace/__tests__/conversationStoreFiles.test.ts@@ -99,6 +99,24 @@ describe('conversation-store files are not agent-accessible', () => {expect(hits).not.toContain('alias.md')})+ it('list skips a link that cannot be resolved and keeps its siblings', async () => {+ fs.symlinkSync('loop', path.join(root, 'loop'))+ fs.symlinkSync('notes.md/child', path.join(root, 'through-file'))+ fs.symlinkSync('missing', path.join(root, 'dangling'))+ const workspace = new WorkspaceService(root)+ const names = (await workspace.list()).map(e => e.name)+ // Witness: the ordinary files and the dangling link stay listed.+ expect(names).toEqual(expect.arrayContaining(['mystate.db.txt', 'notes.md', 'dangling']))+ // ELOOP and ENOTDIR omit only their own entry.+ expect(names).not.toContain('loop')+ expect(names).not.toContain('through-file')+ for (const p of PROTECTED) expect(names).not.toContain(p)++ const all = await workspace.listAll()+ expect(all).toEqual(expect.arrayContaining(['mystate.db.txt', 'notes.md']))+ for (const p of PROTECTED) expect(all).not.toContain(p)+ })+it('file_read and file_write deny them, directly and through a symlink', async () => {const reader = new FileReadTool(root)const writer = new FileWriteTool(root)@@ -126,17 +144,20 @@ describe('conversation-store files are not agent-accessible', () => {return new ShellTool(root, 5000, ['PATH'])}- it.each(['cat state.db.pre-1.bak', 'cat state.db-wal', 'echo y > state.db.pre-2.bak'])(- 'rejects by name: %s',- async command => {- const out = await shell().execute({ command })- expect(out.is_error).toBe(true)- expect(out.content).toContain('Command rejected')- expect(out.content).not.toContain(STORE_BYTES)- storeFilesUntouched()- expect(fs.existsSync(path.join(root, 'state.db.pre-2.bak'))).toBe(false)- }- )+ it.each([+ 'cat state.db.pre-1.bak',+ 'cat state.db-wal',+ 'echo y > state.db.pre-2.bak',+ 'cat state.db_old',+ 'cat state.db2',+ ])('rejects by name: %s', async command => {+ const out = await shell().execute({ command })+ expect(out.is_error).toBe(true)+ expect(out.content).toContain('Command rejected')+ expect(out.content).not.toContain(STORE_BYTES)+ storeFilesUntouched()+ expect(fs.existsSync(path.join(root, 'state.db.pre-2.bak'))).toBe(false)+ })it('runs commands on the lookalike and the ordinary file', async () => {const out = await shell().execute({diff --git a/mcp-host/src/workspace/service.ts b/mcp-host/src/workspace/service.tsindex 5c0401990..60c02708c 100644--- a/mcp-host/src/workspace/service.ts+++ b/mcp-host/src/workspace/service.ts@@ -316,8 +316,9 @@ export class WorkspaceService implements Workspace {const absPath = path.join(resolved, entry.name)// A symlink with a harmless name can resolve to store data; the listing// omits it as read and write already deny it. A dangling link reaches no- // file and stays listed.- if (await this.resolvesToProtectedPath(absPath, realWorkspaceRoot)) continue+ // file and stays listed. A link that cannot be resolved (a loop, a path+ // through a file) is omitted without hiding its siblings.+ if (await this.omitFromListing(absPath, realWorkspaceRoot)) continuelet size: number | undefinedlet modifiedAt: string | undefinedtry {@@ -338,16 +339,12 @@ export class WorkspaceService implements Workspace {return results}- private async resolvesToProtectedPath(- absPath: string,- realWorkspaceRoot: string- ): Promise<boolean> {+ private async omitFromListing(absPath: string, realWorkspaceRoot: string): Promise<boolean> {let realPath: stringtry {realPath = await fs.realpath(absPath)} catch (err) {- if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false- throw err+ return (err as NodeJS.ErrnoException).code !== 'ENOENT'}return isProtectedRealPath(realPath, realWorkspaceRoot)}