Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): stop GFS ledger race and stacked console adapters in tests

Alfredo Lopez8d6a8c7cfe22
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 2e0a546d6..642e04a30 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.504",
+ "version": "0.1.505",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.504",
+ "version": "0.1.505",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 3881d4e89..2ba1fba6a 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.504",
+ "version": "0.1.505",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/gfsLedgerPublicationRace.test.ts b/mcp-host/src/__tests__/gfsLedgerPublicationRace.test.ts
new file mode 100644
index 000000000..2f86b2f9d
--- /dev/null
+++ b/mcp-host/src/__tests__/gfsLedgerPublicationRace.test.ts
@@ -0,0 +1,115 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { randomUUID } from 'node:crypto'
+import * as fs from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { type GfsRuntime, bootstrapGfsRuntime } from '../gfsRuntime'
+import { GfsDownloadStore } from '../internalTools/gfsDownloadStore'
+
+/**
+ * The live writer publishes every ledger by writing a temporary file and
+ * renaming it over `ledger-v1.json`. A standby that reads the ledger during
+ * writer contention can open the old inode and then `lstat` the name after the
+ * rename. This hook performs that real rename at exactly that point, once.
+ */
+const race = vi.hoisted(() => ({
+ armed: false,
+ fired: 0,
+ publish: undefined as undefined | ((ledgerPath: string) => Promise<void>),
+}))
+
+vi.mock('node:fs/promises', async importOriginal => {
+ const actual = await importOriginal<typeof import('node:fs/promises')>()
+ const lstat = (async (target: Parameters<typeof actual.lstat>[0], ...rest: unknown[]) => {
+ if (race.armed && String(target).endsWith('/.gfs-download-store/ledger-v1.json')) {
+ race.armed = false
+ race.fired += 1
+ await race.publish?.(String(target))
+ }
+ return (actual.lstat as (...args: unknown[]) => unknown)(target, ...rest)
+ }) as typeof actual.lstat
+ return { ...actual, lstat, default: { ...actual, lstat } }
+})
+
+/** Same file operations as the live writer's persist(): new inode, then rename. */
+async function publishLedgerCopy(ledgerPath: string): Promise<void> {
+ const raw = await fs.readFile(ledgerPath, 'utf8')
+ const temporary = `${ledgerPath}.tmp-${randomUUID()}`
+ await fs.writeFile(temporary, raw, { flag: 'wx', mode: 0o600 })
+ await fs.rename(temporary, ledgerPath)
+}
+
+const roots: string[] = []
+const stores: GfsDownloadStore[] = []
+const runtimes: GfsRuntime[] = []
+
+async function root() {
+ const value = await fs.mkdtemp(join(tmpdir(), 'gfs-ledger-race-'))
+ roots.push(value)
+ return value
+}
+
+async function ledgerInode(value: string): Promise<bigint> {
+ const real = await fs.realpath(value)
+ return (await fs.stat(join(real, '.gfs-download-store', 'ledger-v1.json'), { bigint: true })).ino
+}
+
+afterEach(async () => {
+ race.armed = false
+ race.fired = 0
+ race.publish = undefined
+ for (const runtime of runtimes.splice(0)) await runtime.stop()
+ for (const store of stores.splice(0)) await store.close()
+ vi.restoreAllMocks()
+ for (const value of roots.splice(0)) await fs.rm(value, { recursive: true, force: true })
+})
+
+describe('GFS ledger publication during writer contention', () => {
+ it('reports a ledger renamed by the live writer as transient contention, not corruption', async () => {
+ const value = await root()
+ const first = new GfsDownloadStore(value)
+ stores.push(first)
+ await first.initialize()
+ const before = await ledgerInode(value)
+ race.publish = publishLedgerCopy
+ race.armed = true
+
+ const second = new GfsDownloadStore(value)
+ await expect(second.initialize()).rejects.toMatchObject({
+ code: 'writer_locked',
+ transientWriterContention: true,
+ })
+
+ // Witness: the rename really happened between the standby's open and lstat.
+ expect(race.fired).toBe(1)
+ expect(await ledgerInode(value)).not.toBe(before)
+ })
+
+ it('keeps retrying through a ledger publication and starts cleanup after the old writer releases', async () => {
+ const value = await root()
+ const first = new GfsDownloadStore(value)
+ stores.push(first)
+ await first.initialize()
+ let resolveCleanup!: () => void
+ const cleanupStarted = new Promise<void>(resolve => {
+ resolveCleanup = resolve
+ })
+ const cleanup = vi
+ .spyOn(GfsDownloadStore.prototype, 'cleanupExpired')
+ .mockImplementation(async () => {
+ resolveCleanup()
+ })
+ race.publish = publishLedgerCopy
+ race.armed = true
+
+ const runtime = await bootstrapGfsRuntime(value, { retryDelayMs: 25, maxRetryAttempts: 40 })
+ runtimes.push(runtime)
+ expect(race.fired).toBe(1)
+ expect(runtime.store.isAvailable()).toBe(false)
+
+ await first.close()
+ await cleanupStarted
+ expect(runtime.store.isAvailable()).toBe(true)
+ expect(cleanup).toHaveBeenCalledOnce()
+ })
+})
diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.ts
index cbc22f5b5..1a2dd9dad 100644
--- a/mcp-host/src/internalTools/gfsDownloadStore.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts
@@ -14,7 +14,11 @@ import type {
GfsProcessingLeaseAcquisition,
GfsProcessingLeaseProvider,
} from './gfsProcessingLease'
-import { openPrivateStoreObject, verifyPrivateStoreDirectory } from './gfsStorePrivateFiles'
+import {
+ PrivateStoreNameMovedError,
+ openPrivateStoreObject,
+ verifyPrivateStoreDirectory,
+} from './gfsStorePrivateFiles'
import { GfsStoreWriterLease, GfsStoreWriterOwnershipError } from './gfsStoreWriterLease'
export type GfsDownloadStoreErrorCode =
@@ -1447,8 +1451,12 @@ export class GfsDownloadStore {
false
)
parseLedger(await handle.readFile('utf8'))
- } catch {
- throw new GfsDownloadStoreError('corrupt_store_ledger')
+ } catch (readError) {
+ // The live writer publishes every ledger by renaming a new inode over
+ // this name, so a name that moved between open and lstat is that
+ // writer at work, not corruption. The next retry judges the new ledger.
+ if (!(readError instanceof PrivateStoreNameMovedError))
+ throw new GfsDownloadStoreError('corrupt_store_ledger')
} finally {
await handle?.close()
}
diff --git a/mcp-host/src/internalTools/gfsDownloadStoreAdversarialQuota.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreAdversarialQuota.test.ts
index f58b28606..fced13bf9 100644
--- a/mcp-host/src/internalTools/gfsDownloadStoreAdversarialQuota.test.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStoreAdversarialQuota.test.ts
@@ -274,7 +274,9 @@ describe('GFS download store adversarial quota', () => {
).rejects.toMatchObject({ code: 'host_quota_exceeded' })
expect(store.debugUsage()).toMatchObject({ files: 64 })
for (const caller of callers) await store.releaseReceiptOwner(`${caller}-owner`, caller)
- })
+ // 64 durable publications at 5 fsyncs each measured 2.2-4.7 s on a loaded
+ // host, which leaves no margin inside Vitest's 5 s default.
+ }, 30_000)
it('rolls back charge, owner pins, and active slots when mkdir fails with proven absence', async () => {
await fs.chmod(callerRoot, 0o500)
diff --git a/mcp-host/src/internalTools/gfsDownloadStoreFutureLeaseCapacity.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreFutureLeaseCapacity.test.ts
index 6f54720c5..3fb92e75e 100644
--- a/mcp-host/src/internalTools/gfsDownloadStoreFutureLeaseCapacity.test.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStoreFutureLeaseCapacity.test.ts
@@ -334,7 +334,9 @@ describe('future execution protection and whole admission capacity', () => {
).rejects.toMatchObject({ code: 'host_quota_exceeded' })
expect(store.debugUsage().files).toBe(64)
for (const receipt of receipts) expect(store.debugRecord(receipt.id)).toBeDefined()
- })
+ // 64 durable publications at 5 fsyncs each measured 2.2-4.7 s on a loaded
+ // host, which leaves no margin inside Vitest's 5 s default.
+ }, 30_000)
it('respects pending active reservations before any cache-pressure effect', async () => {
const { root, store } = await setup({ MCP_HOST_GFS_CALLER_DOWNLOAD_MAX_FILES: '1' })
diff --git a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts b/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
index b767361ea..c3e948c3a 100644
--- a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
+++ b/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
@@ -22,6 +22,17 @@ export function isPrivateStoreStatTrusted(info: Stats, kind: 'file' | 'directory
)
}
+/**
+ * The opened inode is trusted, but the name now points at another inode. An
+ * atomic rename over the name between open and lstat produces exactly this.
+ */
+export class PrivateStoreNameMovedError extends Error {
+ constructor() {
+ super('Private-store inode changed')
+ this.name = 'PrivateStoreNameMovedError'
+ }
+}
+
/** Restore only the exact kubelet fsGroup expansion of a private, owned inode. */
export async function openPrivateStoreObject(
filename: string,
@@ -53,12 +64,12 @@ export async function openPrivateStoreObject(
if (
afterIdentity.dev !== identity.dev ||
afterIdentity.ino !== identity.ino ||
- named.dev !== identity.dev ||
- named.ino !== identity.ino ||
named.isSymbolicLink() ||
(after.mode & 0o7777) !== (restore ? expected : actual)
)
throw new Error('Private-store inode changed')
+ if (named.dev !== identity.dev || named.ino !== identity.ino)
+ throw new PrivateStoreNameMovedError()
return handle
} catch (error) {
await handle.close()
diff --git a/mcp-host/src/logger.test.ts b/mcp-host/src/logger.test.ts
index 1cdcc20a6..40a23b8a9 100644
--- a/mcp-host/src/logger.test.ts
+++ b/mcp-host/src/logger.test.ts
@@ -74,4 +74,31 @@ describe('mcp-host structured logger redaction', () => {
vi.resetModules()
}
})
+
+ it('writes one JSON layer when the module is evaluated more than once', async () => {
+ const previousConsole = { log: console.log, error: console.error, warn: console.warn }
+ const output = vi.spyOn(console, 'log').mockImplementation(() => {})
+ vi.stubEnv('LOG_LEVEL', 'info')
+ try {
+ for (let load = 0; load < 3; load++) {
+ vi.resetModules()
+ await import('./logger.js')
+ }
+ console.log('[Probe] single layer')
+
+ expect(output).toHaveBeenCalledOnce()
+ expect(JSON.parse(output.mock.calls[0]![0] as string)).toMatchObject({
+ level: 'info',
+ component: 'Probe',
+ msg: 'single layer',
+ })
+ } finally {
+ output.mockRestore()
+ console.log = previousConsole.log
+ console.error = previousConsole.error
+ console.warn = previousConsole.warn
+ vi.unstubAllEnvs()
+ vi.resetModules()
+ }
+ })
})
diff --git a/mcp-host/src/logger.ts b/mcp-host/src/logger.ts
index 9d6e8a490..50c13d280 100644
--- a/mcp-host/src/logger.ts
+++ b/mcp-host/src/logger.ts
@@ -20,9 +20,28 @@ const SENSITIVE_KEY_RE =
const UNSAFE_OBJECT_KEY = /^(?:__proto__|constructor|prototype)$/
const SAFE_OBJECT_KEY = /^[A-Za-z0-9._-]{1,64}$/
-const originalLog = console.log.bind(console)
-const originalError = console.error.bind(console)
-const originalWarn = console.warn.bind(console)
+// Every console adapter installed by any evaluation of this module is
+// registered with the sink it writes to. Evaluating the module again (tests
+// that call vi.resetModules()) must reuse that sink instead of wrapping the
+// previous adapter, or every line is JSON-encoded once per load. The registry
+// is keyed by function identity: a replacement such as a test spy is a sink.
+type ConsoleWriter = (...args: unknown[]) => void
+const ADAPTER_SINKS = Symbol.for('clerum.mcp-host.logger.adapterSinks')
+const registryHost = globalThis as { [ADAPTER_SINKS]?: WeakMap<ConsoleWriter, ConsoleWriter> }
+const adapterSinks = (registryHost[ADAPTER_SINKS] ??= new WeakMap())
+
+function sinkOf(writer: ConsoleWriter): ConsoleWriter {
+ return adapterSinks.get(writer) ?? writer.bind(console)
+}
+
+function adapter(sink: ConsoleWriter, write: ConsoleWriter): ConsoleWriter {
+ adapterSinks.set(write, sink)
+ return write
+}
+
+const originalLog = sinkOf(console.log)
+const originalError = sinkOf(console.error)
+const originalWarn = sinkOf(console.warn)
function isSafeObjectKey(key: string): boolean {
return SAFE_OBJECT_KEY.test(key) && !UNSAFE_OBJECT_KEY.test(key)
@@ -105,9 +124,9 @@ function emit(level: string, args: unknown[]): void {
writer(JSON.stringify(entry))
}
-console.log = (...args: unknown[]) => emit('info', args)
-console.error = (...args: unknown[]) => emit('error', args)
-console.warn = (...args: unknown[]) => emit('warn', args)
+console.log = adapter(originalLog, (...args: unknown[]) => emit('info', args))
+console.error = adapter(originalError, (...args: unknown[]) => emit('error', args))
+console.warn = adapter(originalWarn, (...args: unknown[]) => emit('warn', args))
/** Explicit structured entry point for service code, sharing redaction and sinks. */
function writeStructured(