Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): accept the writer fence across PD reattach and log the refusal reason
Alfredo Lopezb8f01e18e44f
diff --git a/docs/operations/gfs-large-file-host-downloads.md b/docs/operations/gfs-large-file-host-downloads.mdindex 33bba65cd..c60fa18d4 100644--- a/docs/operations/gfs-large-file-host-downloads.md+++ b/docs/operations/gfs-large-file-host-downloads.md@@ -339,6 +339,41 @@ Labels contain bounded enums only. Caller, resource, download, command, path, coIf limits are lowered, existing retained copies remain charged and new admissions are rejected until usage falls below the new policy.+### Writer fence versions and persistent-disk reattach (#1028)++`writer.lock` names the writer database by ownership and inode. The v2 fence+(`{schemaVersion:2, ownership, databaseDevice, databaseInode}`) also recorded+the device number, and a persistent disk reattached under another device node+changes that number while the inode stays the same. A v2 Host whose disk came+back on another `/dev/sdX` was refused with a permanent `writer_locked`.++The current image accepts a v2 or v3 fence when its ownership matches and its+`databaseInode` equals the inode of the open writer database; the device is+ignored. An accepted fence is never rewritten. A v3 fence+(`{schemaVersion:3, ownership, databaseInode}`) is written only when a store is+bootstrapped with no fence. Live writers are still excluded by the SQLite+`BEGIN EXCLUSIVE` lock and by the descriptor/name device and inode checks.+When a v2 fence with a stale device is accepted, the Host logs one info line,+`GFS writer fence v2 accepted with stale device`, with `storedDevice` and+`currentDevice`. A writer refusal is logged with `writerDetail`, a closed value+naming the refusal branch (for example `database_identity_changed`,+`legacy_fence`, `sqlite_busy_verified`, `ownership_lost`).++Operator note: v2 remains on disk on pre-existing Hosts. Do not delete+`writer.lock`. Inspect the store (ledger, writer-fence and source-inventory+hashes) before any transition; the inspected fence hash stays valid because an+accepted v2 fence is not rewritten.++| Image after the fix has run | Store with a v2 fence (pre-existing Host) | Store with a v3 fence (bootstrapped by the fix) |+| ----------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------ |+| Production main image (no GFS store code) | Ignores the store directory; works | Ignores the store directory; works |+| Image with the v2 fence (before #1028) | Works while the device number is unchanged; a reattach under another device locks it again | Refuses the v3 fence as legacy (`writer_locked`) |+| Current image | Accepted by inode | Accepted by inode |++Deleting `writer.lock` to force a rollback does not work: with a ledger present+the store refuses with `unsupported_store_schema`, and the fence is what keeps+older binaries excluded.+### Records quarantined by builds before the #1022 fixA build before this fix quarantined every record on any boot whose ledger helddiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 5aa25a82b..ef2576e2b 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.519",+ "version": "0.1.520","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.519",+ "version": "0.1.520","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 54395b65f..1e2e352b5 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.519",+ "version": "0.1.520","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/gfsRuntime.test.ts b/mcp-host/src/__tests__/gfsRuntime.test.tsindex 092698c28..617a90847 100644--- a/mcp-host/src/__tests__/gfsRuntime.test.ts+++ b/mcp-host/src/__tests__/gfsRuntime.test.ts@@ -8,6 +8,17 @@ import { type GfsRuntime, bootstrapGfsRuntime } from '../gfsRuntime'import { GfsDownloadStore, GfsDownloadStoreError } from '../internalTools/gfsDownloadStore'import { logger } from '../logger'+// The logger binds its error sink when its module is evaluated and keeps only+// name/code/status of an Error. Installing this sink before any import makes+// it the logger's error sink, so T8 reads the exact JSON line written.+const serializedErrorLines = vi.hoisted(() => {+ const lines: string[] = []+ console.error = (...args: unknown[]) => {+ lines.push(args.map(String).join(' '))+ }+ return lines+})+const roots: string[] = []const stores: GfsDownloadStore[] = []const runtimes: GfsRuntime[] = []@@ -322,4 +333,45 @@ describe('GFS runtime bootstrap', () => {expect(cleanup).not.toHaveBeenCalled()expect(vi.getTimerCount()).toBe(0)})++ it('T8: the serialized recovery-required line names the writer refusal branch as writerDetail', async () => {+ const value = await root()+ const bootstrap = new GfsDownloadStore(value)+ await bootstrap.initialize()+ await bootstrap.close()+ const storeRoot = join(await fs.realpath(value), '.gfs-download-store')+ const database = await fs.stat(join(storeRoot, 'writer-v2.sqlite'), { bigint: true })+ await fs.writeFile(+ join(storeRoot, 'writer.lock'),+ JSON.stringify({+ schemaVersion: 2,+ ownership: 'sqlite-exclusive-v1',+ databaseDevice: String(database.dev),+ databaseInode: String(database.ino + 1n),+ }),+ { mode: 0o600 }+ )++ serializedErrorLines.length = 0+ const runtime = await bootstrapGfsRuntime(value)+ runtimes.push(runtime)+ expect(runtime.store.isAvailable()).toBe(false)++ const entries = serializedErrorLines+ .filter(line => line.startsWith('{'))+ .map(line => JSON.parse(line) as Record<string, unknown>)+ const recovery = entries.filter(+ entry =>+ entry.msg ===+ 'GFS download store entered recovery-required state; managed GFS operations are disabled'+ )+ expect(recovery).toHaveLength(1)+ expect(recovery[0]).toMatchObject({+ level: 'error',+ component: 'gfs-runtime',+ writerDetail: 'database_identity_changed',+ err: { name: 'GfsDownloadStoreError', code: 'writer_locked' },+ available: false,+ })+ })})diff --git a/mcp-host/src/gfsRuntime.ts b/mcp-host/src/gfsRuntime.tsindex 15e4dc06c..3b56cf54e 100644--- a/mcp-host/src/gfsRuntime.ts+++ b/mcp-host/src/gfsRuntime.ts@@ -21,6 +21,12 @@ function transientWriterContention(error: unknown): boolean {)}+// The logger keeps only name/code/status of an Error, so the writer refusal+// branch is logged as its own field.+function writerDetail(error: unknown): string | undefined {+ return error instanceof GfsDownloadStoreError ? error.detail : undefined+}+/*** Contain GFS store failure without removing the caller workspace or falling* back to an unprotected shared-root shell. Ordinary RPC startup continues;@@ -70,7 +76,12 @@ export async function bootstrapGfsRuntime(if (stopped) returnif (!transientWriterContention(error)) {logger.error(- { component: 'gfs-runtime', err: error, attempt: attempts },+ {+ component: 'gfs-runtime',+ err: error,+ writerDetail: writerDetail(error),+ attempt: attempts,+ },'GFS writer retry ended in recovery-required state')return@@ -140,7 +151,12 @@ export async function bootstrapGfsRuntime(schedule({ kind: 'retry', delayMs: retryDelayMs })} else {logger.error(- { component: 'gfs-runtime', err: error, available: store.isAvailable() },+ {+ component: 'gfs-runtime',+ err: error,+ writerDetail: writerDetail(error),+ available: store.isAvailable(),+ },'GFS download store entered recovery-required state; managed GFS operations are disabled')}diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.tsindex c153f3132..5e4861cf1 100644--- a/mcp-host/src/internalTools/gfsDownloadStore.ts+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts@@ -20,7 +20,11 @@ import {openPrivateStoreObject,verifyPrivateStoreDirectory,} from './gfsStorePrivateFiles'-import { GfsStoreWriterLease, GfsStoreWriterOwnershipError } from './gfsStoreWriterLease'+import {+ GfsStoreWriterLease,+ GfsStoreWriterOwnershipError,+ type WriterFenceDetail,+} from './gfsStoreWriterLease'const GFS_DOWNLOAD_STORE_LOG_COMPONENT = 'GfsDownloadStore'@@ -41,7 +45,9 @@ export type GfsDownloadStoreErrorCode =export class GfsDownloadStoreError extends Error {constructor(readonly code: GfsDownloadStoreErrorCode,- readonly transientWriterContention = false+ readonly transientWriterContention = false,+ /** Writer-ownership refusal branch; set only when the writer lease named it. */+ readonly detail?: WriterFenceDetail) {super(`GFS download store failed (${code})`)this.name = 'GfsDownloadStoreError'@@ -1300,6 +1306,19 @@ export class GfsDownloadStore {try {await lease.acquire()this.writerLease = lease+ // A v2 fence is accepted by inode only and stays on disk unchanged. Its+ // stale device is the persistent-disk reattach case; device numbers are+ // not secrets.+ const accepted = lease.lastAcquire+ if (accepted?.fenceSchema === 2 && accepted.storedDevice !== accepted.currentDevice)+ logger.info(+ {+ component: GFS_DOWNLOAD_STORE_LOG_COMPONENT,+ storedDevice: accepted.storedDevice,+ currentDevice: accepted.currentDevice,+ },+ 'GFS writer fence v2 accepted with stale device'+ )} catch (error) {if (error instanceof GfsStoreWriterOwnershipError) {if (error.transientWriterContention) {@@ -1324,7 +1343,7 @@ export class GfsDownloadStore {await handle?.close()}}- throw new GfsDownloadStoreError(error.reason, error.transientWriterContention)+ throw new GfsDownloadStoreError(error.reason, error.transientWriterContention, error.detail)}throw new GfsDownloadStoreError('workspace_unavailable')}@@ -1337,11 +1356,16 @@ export class GfsDownloadStore {private async assertWriterOwnership(): Promise<void> {try {- if (!this.writerLease) throw new GfsStoreWriterOwnershipError('Writer ownership lost')+ if (!this.writerLease)+ throw new GfsStoreWriterOwnershipError('Writer ownership lost', 'ownership_lost')await this.writerLease.verifyHeld()- } catch {+ } catch (error) {this.unsafe = true- throw new GfsDownloadStoreError('writer_locked')+ throw new GfsDownloadStoreError(+ 'writer_locked',+ false,+ error instanceof GfsStoreWriterOwnershipError ? error.detail : undefined+ )}}@@ -1408,8 +1432,12 @@ export class GfsDownloadStore {if (this.unsafe) throw new GfsDownloadStoreError('storage_write_failed')try {this.writerLease?.assertHeld()- } catch {- throw new GfsDownloadStoreError('writer_locked')+ } catch (error) {+ throw new GfsDownloadStoreError(+ 'writer_locked',+ false,+ error instanceof GfsStoreWriterOwnershipError ? error.detail : undefined+ )}}}diff --git a/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.tsindex 5561ad984..bfd0c30be 100644--- a/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts+++ b/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts@@ -613,6 +613,108 @@ describe('legacy processing leases after #1019', () => {})})+describe('first boot of a Host locked by a persistent-disk reattach (#1028)', () => {+ afterEach(() => {+ vi.restoreAllMocks()+ })++ it('T7: recovers the store behind a stale-device v2 fence and keeps every record it must keep', async () => {+ const owner = '55555555-5555-4555-8555-555555555555'+ const thirdSource: GfsImageSource = {+ ...source,+ resourceId: 'c'.repeat(32),+ gfsUri: `gfs://main/${'c'.repeat(32)}`,+ name: 'third.csv',+ }+ const crashed = freshStore()+ await crashed.initialize()+ const reusable = await fixture(crashed)+ const pinned = await fixture(crashed, owner, otherSource)+ const callerRoot = path.join(root, 'users', 'caller-a')+ const transfer = await crashed.createTransfer({+ callerIdentity: 'caller-a',+ callerWorkspacePath: callerRoot,+ source: thirdSource,+ sizeBytes: 7,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ })+ // The Host dies holding the writer; releasing the kernel connection models it.+ await (+ crashed as unknown as { writerLease: { release(): Promise<void> } }+ ).writerLease.release()++ // An older build left a processing lease, and the pinned copy expired+ // while the Host was locked.+ const seededLedger = JSON.parse(await fs.readFile(ledgerFile(), 'utf8'))+ seededLedger.records[pinned.id].expiresAt = new Date(Date.now() - 60_000).toISOString()+ seededLedger.processingLeases = {+ [LEGACY_LEASE_ID]: legacyLease({ recordIds: [transfer.id] }),+ }+ await fs.writeFile(ledgerFile(), JSON.stringify(seededLedger), { mode: 0o600 })+ // The disk came back under another device node: the v2 fence keeps the+ // inode and carries a stale device.+ const storeRoot = path.join(await fs.realpath(root), '.gfs-download-store')+ const database = await fs.stat(path.join(storeRoot, 'writer-v2.sqlite'), { bigint: true })+ const staleDevice = String(database.dev + 1n)+ const v2 = JSON.stringify({+ schemaVersion: 2,+ ownership: 'sqlite-exclusive-v1',+ databaseDevice: staleDevice,+ databaseInode: String(database.ino),+ })+ await fs.writeFile(path.join(storeRoot, 'writer.lock'), v2, { mode: 0o600 })+ // Witness: the seeded store holds every state this boot has to handle.+ expect(seededLedger.records[reusable.id].state).toBe('completed')+ expect(seededLedger.records[pinned.id].state).toBe('completed')+ expect(seededLedger.records[transfer.id].state).toBe('transferring')+ expect(seededLedger.retentionOwners[owner].recordIds).toEqual([pinned.id])+ const pinnedPath = path.join(callerRoot, pinned.path)+ expect(await fs.readFile(pinnedPath, 'utf8')).toBe('fixture')++ const warn = vi.spyOn(logger, 'warn')+ const info = vi.spyOn(logger, 'info')+ const recovering = freshStore()+ await recovering.initialize()++ expect(recovering.isAvailable()).toBe(true)+ expect(+ info.mock.calls.filter(call => call[1] === 'GFS writer fence v2 accepted with stale device')+ ).toEqual([+ [+ {+ component: 'GfsDownloadStore',+ storedDevice: staleDevice,+ currentDevice: String(database.dev),+ },+ 'GFS writer fence v2 accepted with stale device',+ ],+ ])+ expect(await fs.readFile(path.join(storeRoot, 'writer.lock'), 'utf8')).toBe(v2)+ expect(discardWarnings(warn.mock.calls)).toEqual([+ [+ { component: 'GfsDownloadStore', discarded: 1 },+ 'GFS download store discarded 1 legacy processing lease(s) at initialize',+ ],+ ])+ expect(recovering.debugRecord(transfer.id)?.state).toBe('quarantined')+ expect(recovering.debugRecord(pinned.id)?.state).toBe('completed')+ // Three copies of 7 bytes stay charged, including the quarantined transfer.+ expect(recovering.debugUsage().bytes).toBe(21)++ const persisted = JSON.parse(await fs.readFile(ledgerFile(), 'utf8'))+ expect(persisted).not.toHaveProperty('processingLeases')+ expect(persisted.records[transfer.id].state).toBe('quarantined')+ expect(persisted.records[pinned.id].state).toBe('completed')+ expect(persisted.retentionOwners[owner]).toEqual(seededLedger.retentionOwners[owner])+ // The first-boot expiry sweep kept the expired copy its owner still pins.+ expect(await fs.readFile(pinnedPath, 'utf8')).toBe('fixture')+ expect(await recovering.reusableReceipt('caller-a', source, 7)).toMatchObject({+ id: reusable.id,+ sha256: reusable.sha256,+ })+ })+})+describe('GFS writer recovery and filesystem invariants', () => {it.each(['SIGKILL', 'SIGTERM'] as const)('excludes another process and recovers after %s on the same inode',@@ -839,7 +941,13 @@ describe('GFS writer recovery and filesystem invariants', () => {await store.close()const databasePath = path.join(root, '.gfs-download-store', 'writer-v2.sqlite')await fs.rename(databasePath, `${databasePath}.old`)- await expect(freshStore().initialize()).rejects.toMatchObject({ code: 'workspace_unavailable' })+ // #1028: a fence naming a database that is gone is a writer refusal with+ // its own detail, never a generic workspace failure.+ await expect(freshStore().initialize()).rejects.toMatchObject({+ code: 'writer_locked',+ transientWriterContention: false,+ detail: 'database_missing',+ })// Exclusive creation proves initialization left the target absent in the// same atomic operation. An unexpected second inode fails with EEXIST.let replacementDatabase: FileHandle | undefined@@ -848,7 +956,10 @@ describe('GFS writer recovery and filesystem invariants', () => {} finally {await replacementDatabase?.close()}- await expect(freshStore().initialize()).rejects.toMatchObject({ code: 'writer_locked' })+ await expect(freshStore().initialize()).rejects.toMatchObject({+ code: 'writer_locked',+ detail: 'database_identity_changed',+ })})it('restores only verified owned fsGroup transformations without relaxing privacy', async () => {diff --git a/mcp-host/src/internalTools/gfsStoreWriterFence.test.ts b/mcp-host/src/internalTools/gfsStoreWriterFence.test.tsnew file mode 100644index 000000000..22357e74a--- /dev/null+++ b/mcp-host/src/internalTools/gfsStoreWriterFence.test.ts@@ -0,0 +1,310 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import Database from 'better-sqlite3'+import { createHash } from 'node:crypto'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import { logger } from '../logger'+import { GfsDownloadStore } from './gfsDownloadStore'+import { GfsStoreWriterLease } from './gfsStoreWriterLease'++/**+ * #1028: a v2 writer fence recorded the database device, and a persistent disk+ * reattached under another device node changed it, which locked the store for+ * good. The fence is now judged by ownership and inode only. A v2 fence is+ * accepted in place and never rewritten; v3 (no device) is written only at+ * bootstrap. Live writers stay excluded by the SQLite exclusive lock.+ */+const OWNERSHIP = 'sqlite-exclusive-v1'+const V2_ACCEPTED = 'GFS writer fence v2 accepted with stale device'++let root: string+let stores: GfsDownloadStore[]+let leases: GfsStoreWriterLease[]+let connections: Database.Database[]++beforeEach(async () => {+ root = await fs.mkdtemp(path.join(tmpdir(), 'gfs-writer-fence-'))+ stores = []+ leases = []+ connections = []+})++afterEach(async () => {+ vi.restoreAllMocks()+ for (const connection of connections) if (connection.open) connection.close()+ for (const value of leases) await value.release()+ for (const store of stores) await store.close(0).catch(() => undefined)+ await fs.rm(root, { recursive: true, force: true })+})++function freshStore(): GfsDownloadStore {+ const store = new GfsDownloadStore(root)+ stores.push(store)+ return store+}++function lease(storeRoot: string): GfsStoreWriterLease {+ const value = new GfsStoreWriterLease(storeRoot, false)+ leases.push(value)+ return value+}++function connection(storeRoot: string): Database.Database {+ const value = new Database(path.join(storeRoot, 'writer-v2.sqlite'), {+ timeout: 0,+ fileMustExist: true,+ })+ connections.push(value)+ return value+}++/** A bootstrapped store whose only writer has closed. */+async function closedStoreRoot(): Promise<string> {+ const store = freshStore()+ await store.initialize()+ await store.close()+ return path.join(await fs.realpath(root), '.gfs-download-store')+}++async function databaseIdentity(storeRoot: string): Promise<{ dev: string; ino: string }> {+ const info = await fs.stat(path.join(storeRoot, 'writer-v2.sqlite'), { bigint: true })+ return { dev: String(info.dev), ino: String(info.ino) }+}++/** Byte-exact v2 fence as the c8f898209 writer serialized it. */+function v2Fence(device: string, inode: string): string {+ return JSON.stringify({+ schemaVersion: 2,+ ownership: OWNERSHIP,+ databaseDevice: device,+ databaseInode: inode,+ })+}++function markerPath(storeRoot: string): string {+ return path.join(storeRoot, 'writer.lock')+}++async function writeFence(storeRoot: string, bytes: string): Promise<void> {+ await fs.writeFile(markerPath(storeRoot), bytes, { mode: 0o600 })+}++function sha256(value: string): string {+ return createHash('sha256').update(value).digest('hex')+}++function acceptanceLogs(spy: { mock: { calls: unknown[][] } }): unknown[][] {+ return spy.mock.calls.filter(call => call[1] === V2_ACCEPTED)+}++describe('GFS writer fence across a persistent-disk reattach (#1028)', () => {+ it('T1: accepts a v2 fence whose device changed when its inode matches, keeps it byte-for-byte and logs both devices', async () => {+ const storeRoot = await closedStoreRoot()+ const { dev, ino } = await databaseIdentity(storeRoot)+ const staleDevice = String(BigInt(dev) + 1n)+ const v2 = v2Fence(staleDevice, ino)+ await writeFence(storeRoot, v2)+ const info = vi.spyOn(logger, 'info')++ const store = freshStore()+ await store.initialize()++ expect(store.isAvailable()).toBe(true)+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v2)+ expect(acceptanceLogs(info)).toEqual([+ [+ { component: 'GfsDownloadStore', storedDevice: staleDevice, currentDevice: dev },+ V2_ACCEPTED,+ ],+ ])+ await store.close()+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v2)+ })++ it('T2: refuses a v2 fence naming another database inode as permanent database_identity_changed', async () => {+ const storeRoot = await closedStoreRoot()+ const { dev, ino } = await databaseIdentity(storeRoot)+ const foreign = v2Fence(dev, String(BigInt(ino) + 1n))+ await writeFence(storeRoot, foreign)++ const store = freshStore()+ await expect(store.initialize()).rejects.toMatchObject({+ code: 'writer_locked',+ transientWriterContention: false,+ detail: 'database_identity_changed',+ })+ expect(store.isAvailable()).toBe(false)+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(foreign)++ // Witness: the same store with only the inode corrected is accepted, so the+ // refusal came from the inode and nothing else.+ await writeFence(storeRoot, v2Fence(dev, ino))+ await store.initialize()+ expect(store.isAvailable()).toBe(true)+ })++ it('T3: bootstrap writes the v3 fence, a v3 reopen acquires, and reacquisition leaves its hash unchanged', async () => {+ const store = freshStore()+ await store.initialize()+ const storeRoot = path.join(await fs.realpath(root), '.gfs-download-store')+ const { dev, ino } = await databaseIdentity(storeRoot)+ const v3 = JSON.stringify({ schemaVersion: 3, ownership: OWNERSHIP, databaseInode: ino })+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v3)+ await store.close()+ const bootstrapHash = sha256(await fs.readFile(markerPath(storeRoot), 'utf8'))++ const info = vi.spyOn(logger, 'info')+ const reopened = freshStore()+ await reopened.initialize()+ expect(reopened.isAvailable()).toBe(true)+ expect(acceptanceLogs(info)).toEqual([])+ await reopened.close()++ const direct = lease(storeRoot)+ await direct.acquire()+ direct.assertHeld()+ expect(direct.lastAcquire).toEqual({ fenceSchema: 3, currentDevice: dev })+ await direct.release()+ expect(sha256(await fs.readFile(markerPath(storeRoot), 'utf8'))).toBe(bootstrapHash)+ })++ it('T4: a second real SQLite connection excludes the writer as verified transient contention while v2 stays on disk', async () => {+ const storeRoot = await closedStoreRoot()+ const { dev, ino } = await databaseIdentity(storeRoot)+ const staleDevice = String(BigInt(dev) + 1n)+ const v2 = v2Fence(staleDevice, ino)+ await writeFence(storeRoot, v2)++ const other = connection(storeRoot)+ other.pragma('journal_mode = DELETE')+ other.exec('BEGIN EXCLUSIVE')+ expect(other.inTransaction).toBe(true)+ await expect(lease(storeRoot).acquire()).rejects.toMatchObject({+ reason: 'writer_locked',+ transientWriterContention: true,+ detail: 'sqlite_busy_verified',+ })+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v2)+ other.close()++ const held = lease(storeRoot)+ await held.acquire()+ held.assertHeld()+ expect(held.lastAcquire).toEqual({+ fenceSchema: 2,+ storedDevice: staleDevice,+ currentDevice: dev,+ })+ // While the lease holds, another real connection cannot take the lock.+ const contender = connection(storeRoot)+ let contenderCode: unknown+ try {+ contender.exec('BEGIN EXCLUSIVE')+ } catch (error) {+ contenderCode = (error as { code?: unknown }).code+ }+ expect(contenderCode).toBe('SQLITE_BUSY')+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v2)+ })++ it.each([+ ['empty', () => ''],+ ['truncated v2', (dev: string, ino: string) => v2Fence(dev, ino).slice(0, 24)],+ [+ 'v1 legacy lease id',+ () => JSON.stringify({ pid: 123, leaseId: 'legacy', acquiredAt: new Date(0).toISOString() }),+ ],+ [+ 'unknown schema version',+ (_dev: string, ino: string) =>+ JSON.stringify({ schemaVersion: 4, ownership: OWNERSHIP, databaseInode: ino }),+ ],+ [+ 'foreign ownership',+ (_dev: string, ino: string) =>+ JSON.stringify({ schemaVersion: 3, ownership: 'other', databaseInode: ino }),+ ],+ ] as const)(+ 'T5: refuses the %s fence as legacy_fence and leaves it in place',+ async (_name, build) => {+ const storeRoot = await closedStoreRoot()+ const { dev, ino } = await databaseIdentity(storeRoot)+ const fence = build(dev, ino)+ await writeFence(storeRoot, fence)++ await expect(lease(storeRoot).acquire()).rejects.toMatchObject({+ reason: 'writer_locked',+ transientWriterContention: false,+ detail: 'legacy_fence',+ })+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(fence)++ // Witness: the same database with a valid fence is acquired.+ await writeFence(storeRoot, v2Fence(dev, ino))+ const valid = lease(storeRoot)+ await valid.acquire()+ valid.assertHeld()+ }+ )++ it('T6: operator recovery accepts a v2 fence with another device in place and refuses another inode', async () => {+ const storeRoot = await closedStoreRoot()+ const { dev, ino } = await databaseIdentity(storeRoot)+ const v2 = v2Fence(String(BigInt(dev) + 1n), ino)+ await writeFence(storeRoot, v2)+ let fenceChecks = 0+ const assertPhysicalFenceHeld = async () => {+ fenceChecks += 1+ }++ const operator = lease(storeRoot)+ await operator.acquireForRecovery({+ expectedWriterFenceSha256: sha256(v2),+ assertPhysicalFenceHeld,+ })+ await operator.verifyHeld()+ expect(fenceChecks).toBeGreaterThan(0)+ expect(operator.lastAcquire?.fenceSchema).toBe(2)+ // The inspected hash stays valid: the v2 file is not rewritten.+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(v2)+ await operator.release()++ const foreign = v2Fence(dev, String(BigInt(ino) + 1n))+ await writeFence(storeRoot, foreign)+ await expect(+ lease(storeRoot).acquireForRecovery({+ expectedWriterFenceSha256: sha256(foreign),+ assertPhysicalFenceHeld,+ })+ ).rejects.toMatchObject({+ reason: 'writer_locked',+ transientWriterContention: false,+ detail: 'database_identity_changed',+ })+ expect(await fs.readFile(markerPath(storeRoot), 'utf8')).toBe(foreign)+ })++ it('T9: assertInitialized and assertWriterOwnership propagate ownership_lost', async () => {+ const store = freshStore()+ await store.initialize()+ expect(store.isAvailable()).toBe(true)+ // Witness: the guarded call succeeds while the writer is held.+ await store.cleanupExpired()++ await (store as unknown as { writerLease: GfsStoreWriterLease }).writerLease.release()+ // cleanupExpired checks assertInitialized first.+ await expect(store.cleanupExpired()).rejects.toMatchObject({+ code: 'writer_locked',+ transientWriterContention: false,+ detail: 'ownership_lost',+ })+ // close() reaches assertWriterOwnership through serialize, without assertInitialized.+ await expect(store.close(0)).rejects.toMatchObject({+ code: 'writer_locked',+ transientWriterContention: false,+ detail: 'ownership_lost',+ })+ expect(store.isAvailable()).toBe(false)+ })+})diff --git a/mcp-host/src/internalTools/gfsStoreWriterLease.ts b/mcp-host/src/internalTools/gfsStoreWriterLease.tsindex 7165b5715..87ac33a67 100644--- a/mcp-host/src/internalTools/gfsStoreWriterLease.ts+++ b/mcp-host/src/internalTools/gfsStoreWriterLease.ts@@ -7,9 +7,31 @@ import { isPrivateStoreStatTrusted, openPrivateStoreObject } from './gfsStorePriconst OWNERSHIP = 'sqlite-exclusive-v1'+/**+ * Closed set naming the branch that refused writer ownership. It carries no+ * path, inode or device value, so it is safe to log as a structured field.+ */+export type WriterFenceDetail =+ | 'local_owner_active'+ | 'local_owner_ambiguous'+ | 'fence_unreadable'+ | 'fence_snapshot_changed'+ | 'legacy_fence'+ | 'missing_fence_with_ledger'+ | 'missing_fence_without_ledger'+ | 'database_missing'+ | 'database_identity_changed'+ | 'sqlite_busy_verified'+ | 'sqlite_busy_unverified'+ | 'ownership_lost'+ | 'path_identity_changed'+ | 'root_path_changed'+ | 'fence_changed'+export class GfsStoreWriterOwnershipError extends Error {constructor(message: string,+ readonly detail: WriterFenceDetail,readonly reason:| 'writer_locked'| 'unsupported_store_schema'@@ -20,6 +42,56 @@ export class GfsStoreWriterOwnershipError extends Error {}}+/**+ * The fence a successful acquisition verified. A v2 fence also recorded the+ * database device, which changes when a persistent disk is reattached under+ * another device node; only its inode is compared.+ */+export interface GfsStoreWriterFenceAcceptance {+ fenceSchema: 2 | 3+ storedDevice?: string+ currentDevice: string+}++interface KnownWriterFence {+ schemaVersion: 2 | 3+ databaseInode: string+ databaseDevice?: string+}++/**+ * v2 fences ({schemaVersion:2, ownership, databaseDevice, databaseInode}) stay+ * on disk unchanged on Hosts that wrote them; v3 omits the device. Both are+ * accepted by ownership and inode. Anything else is legacy or foreign.+ */+function parseKnownFence(marker: string): KnownWriterFence | undefined {+ try {+ const parsed = JSON.parse(marker) as {+ schemaVersion?: unknown+ ownership?: unknown+ databaseInode?: unknown+ databaseDevice?: unknown+ }+ if (+ (parsed.schemaVersion !== 2 && parsed.schemaVersion !== 3) ||+ parsed.ownership !== OWNERSHIP ||+ typeof parsed.databaseInode !== 'string'+ )+ return undefined+ return {+ schemaVersion: parsed.schemaVersion,+ databaseInode: parsed.databaseInode,+ databaseDevice:+ parsed.schemaVersion === 2 && typeof parsed.databaseDevice === 'string'+ ? parsed.databaseDevice+ : undefined,+ }+ } catch {+ /* An operator can repair the exact hashed interrupted fence. */+ return undefined+ }+}+interface OperatorOwnershipTransition {expectedWriterFenceSha256: string | 'absent'assertPhysicalFenceHeld(): Promise<void>@@ -40,6 +112,7 @@ export class GfsStoreWriterLease {// and the next attempt judges the store from scratch.private inTransition = falseprivate verifiedMarker?: string+ private acceptance?: GfsStoreWriterFenceAcceptanceprivate readonly databasePath: stringprivate readonly markerPath: string@@ -60,6 +133,11 @@ export class GfsStoreWriterLease {return this.acquireInternal(transition)}+ /** The fence verified by the last successful acquisition of this lease. */+ get lastAcquire(): GfsStoreWriterFenceAcceptance | undefined {+ return this.acceptance+ }+private async acquireInternal(transition?: OperatorOwnershipTransition): Promise<void> {const priorOwner = GfsStoreWriterLease.localOwners.get(this.root)if (priorOwner) {@@ -75,11 +153,17 @@ export class GfsStoreWriterLease {// owner started releasing while it was being checked.transient = priorOwner.inTransition}- throw new GfsStoreWriterOwnershipError('Writer already active', 'writer_locked', transient)+ throw new GfsStoreWriterOwnershipError(+ 'Writer already active',+ transient ? 'local_owner_active' : 'local_owner_ambiguous',+ 'writer_locked',+ transient+ )}GfsStoreWriterLease.localOwners.set(this.root, this)this.localOwner = truethis.inTransition = true+ this.acceptance = undefinedlet contentionMarker: string | undefinedtry {await transition?.assertPhysicalFenceHeld()@@ -101,25 +185,20 @@ export class GfsStoreWriterLease {marker = await this.markerHandle.readFile('utf8')} catch (error) {if ((error as NodeJS.ErrnoException).code !== 'ENOENT')- throw new GfsStoreWriterOwnershipError('Legacy or ambiguous writer fence')+ throw new GfsStoreWriterOwnershipError(+ 'Legacy or ambiguous writer fence',+ 'fence_unreadable'+ )}if (transition) {const actual =marker === undefined ? 'absent' : createHash('sha256').update(marker).digest('hex')if (actual !== transition.expectedWriterFenceSha256)- throw new GfsStoreWriterOwnershipError('Writer fence changed')- }- let knownV2 = false- if (marker !== undefined) {- try {- const parsed = JSON.parse(marker) as { schemaVersion?: unknown; ownership?: unknown }- knownV2 = parsed.schemaVersion === 2 && parsed.ownership === OWNERSHIP- } catch {- /* An operator can repair the exact hashed interrupted fence. */- }- if (!knownV2 && !transition)- throw new GfsStoreWriterOwnershipError('Legacy or ambiguous writer fence')+ throw new GfsStoreWriterOwnershipError('Writer fence changed', 'fence_snapshot_changed')}+ const knownFence = marker === undefined ? undefined : parseKnownFence(marker)+ if (marker !== undefined && !knownFence && !transition)+ throw new GfsStoreWriterOwnershipError('Legacy or ambiguous writer fence', 'legacy_fence')if (marker === undefined && !this.allowBootstrap && !transition) {let ledgerExists = truetry {@@ -132,10 +211,11 @@ export class GfsStoreWriterLease {// cannot prove their executor stopped; only a physical fence can.throw new GfsStoreWriterOwnershipError('Operator transition required',+ ledgerExists ? 'missing_fence_with_ledger' : 'missing_fence_without_ledger',ledgerExists ? 'unsupported_store_schema' : 'corrupt_store_ledger')}- if (!knownV2) {+ if (!knownFence) {await transition?.assertPhysicalFenceHeld()try {const created = await fs.open(@@ -153,29 +233,43 @@ export class GfsStoreWriterLease {if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error}}- // A missing/replaced database behind an existing v2 fence is never recreated.- this.databaseHandle = await openPrivateStoreObject(- this.databasePath,- 'file',- constants.O_RDWR,- false- )+ // A missing/replaced database behind an existing fence is never recreated.+ try {+ this.databaseHandle = await openPrivateStoreObject(+ this.databasePath,+ 'file',+ constants.O_RDWR,+ false+ )+ } catch (error) {+ if (knownFence && (error as NodeJS.ErrnoException).code === 'ENOENT')+ throw new GfsStoreWriterOwnershipError('Writer database missing', 'database_missing')+ throw error+ }const info = await this.databaseHandle.stat({ bigint: true })- const expectedMarker = JSON.stringify({- schemaVersion: 2,- ownership: OWNERSHIP,- databaseDevice: String(info.dev),- databaseInode: String(info.ino),- })- if (knownV2 && marker !== expectedMarker)- throw new GfsStoreWriterOwnershipError('Writer database inode changed')- if (knownV2 && !transition) contentionMarker = expectedMarker+ const databaseInode = String(info.ino)+ // The device is deliberately not compared: a persistent disk reattached+ // under another device node keeps its inodes. Live writers stay excluded+ // by BEGIN EXCLUSIVE and the descriptor/name identity checks below.+ if (knownFence && knownFence.databaseInode !== databaseInode)+ throw new GfsStoreWriterOwnershipError(+ 'Writer database inode changed',+ 'database_identity_changed'+ )+ if (knownFence && !transition) contentionMarker = markerawait transition?.assertPhysicalFenceHeld()await this.databaseHandle.chmod(0o600)this.database = new Database(this.databasePath, { timeout: 0, fileMustExist: true })this.database.pragma('journal_mode = DELETE')this.database.exec('BEGIN EXCLUSIVE')- if (marker !== expectedMarker) {+ // An accepted v2 or v3 fence is never rewritten; the normal-path marker+ // descriptor is read-only. Only a missing fence (bootstrap) or an operator+ // transition over a hashed legacy fence writes the v3 fence.+ const fence =+ knownFence && marker !== undefined+ ? marker+ : JSON.stringify({ schemaVersion: 3, ownership: OWNERSHIP, databaseInode })+ if (!knownFence) {await transition?.assertPhysicalFenceHeld()if (!this.markerHandle) {this.markerHandle = await fs.open(@@ -186,7 +280,7 @@ export class GfsStoreWriterLease {}// Rewrite through the verified descriptor without unlinking the legacy// fence. A crash leaves its inode present, still excluding old binaries.- const bytes = Buffer.from(expectedMarker)+ const bytes = Buffer.from(fence)await transition?.assertPhysicalFenceHeld()await this.markerHandle.truncate(0)let offset = 0@@ -208,10 +302,15 @@ export class GfsStoreWriterLease {}await transition?.assertPhysicalFenceHeld()await this.markerHandle!.chmod(0o600)- this.verifiedMarker = expectedMarker+ this.verifiedMarker = fenceawait this.verifyHeld()await transition?.assertPhysicalFenceHeld()await this.rootHandle.sync()+ this.acceptance = {+ fenceSchema: knownFence?.schemaVersion ?? 3,+ storedDevice: knownFence?.databaseDevice,+ currentDevice: String(info.dev),+ }this.inTransition = false} catch (error) {let transient = false@@ -225,19 +324,25 @@ export class GfsStoreWriterLease {}await this.release()if ((error as { code?: string }).code === 'SQLITE_BUSY')- throw new GfsStoreWriterOwnershipError('Writer already active', 'writer_locked', transient)+ throw new GfsStoreWriterOwnershipError(+ 'Writer already active',+ transient ? 'sqlite_busy_verified' : 'sqlite_busy_unverified',+ 'writer_locked',+ transient+ )throw error}}assertHeld(): void {if (!this.localOwner || !this.database?.open || !this.database.inTransaction)- throw new GfsStoreWriterOwnershipError('Writer ownership lost')+ throw new GfsStoreWriterOwnershipError('Writer ownership lost', 'ownership_lost')}async verifyHeld(): Promise<void> {this.assertHeld()- if (!this.verifiedMarker) throw new GfsStoreWriterOwnershipError('Writer ownership lost')+ if (!this.verifiedMarker)+ throw new GfsStoreWriterOwnershipError('Writer ownership lost', 'ownership_lost')await this.verifyOwnershipPaths(this.verifiedMarker)this.assertHeld()}@@ -248,7 +353,7 @@ export class GfsStoreWriterLease {[this.markerPath, this.markerHandle, 'file'],[this.databasePath, this.databaseHandle, 'file'],] as const) {- if (!handle) throw new GfsStoreWriterOwnershipError('Writer ownership lost')+ if (!handle) throw new GfsStoreWriterOwnershipError('Writer ownership lost', 'ownership_lost')const metadata = await handle.stat()const opened = await handle.stat({ bigint: true })const named = await fs.lstat(filename, { bigint: true })@@ -258,14 +363,14 @@ export class GfsStoreWriterLease {named.dev !== opened.dev ||named.ino !== opened.ino)- throw new GfsStoreWriterOwnershipError('Writer inode changed')+ throw new GfsStoreWriterOwnershipError('Writer inode changed', 'path_identity_changed')}if ((await fs.realpath(this.root)) !== this.root)- throw new GfsStoreWriterOwnershipError('Writer path changed')+ throw new GfsStoreWriterOwnershipError('Writer path changed', 'root_path_changed')const marker = await openPrivateStoreObject(this.markerPath, 'file', constants.O_RDONLY, false)try {if ((await marker.readFile('utf8')) !== expectedMarker)- throw new GfsStoreWriterOwnershipError('Writer fence changed')+ throw new GfsStoreWriterOwnershipError('Writer fence changed', 'fence_changed')} finally {await marker.close()}diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonindex c0ad9904a..63d40104c 100644--- a/scripts/ci/pinned-vitest-cases/mcp-host.json+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -166,5 +166,45 @@{"suite": "config.toolProgressInterval.test.ts","fullName": "CLERUM_TOOL_PROGRESS_INTERVAL_MS (#1028 P2) P2-b: malformed and sub-second values stop startup with the exact message and do not echo the value"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T1: accepts a v2 fence whose device changed when its inode matches, keeps it byte-for-byte and logs both devices"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T2: refuses a v2 fence naming another database inode as permanent database_identity_changed"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T3: bootstrap writes the v3 fence, a v3 reopen acquires, and reacquisition leaves its hash unchanged"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T4: a second real SQLite connection excludes the writer as verified transient contention while v2 stays on disk"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T5: refuses the empty fence as legacy_fence and leaves it in place"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T5: refuses the unknown schema version fence as legacy_fence and leaves it in place"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T6: operator recovery accepts a v2 fence with another device in place and refuses another inode"+ },+ {+ "suite": "gfsDownloadStoreRecovery.test.ts",+ "fullName": "first boot of a Host locked by a persistent-disk reattach (#1028) T7: recovers the store behind a stale-device v2 fence and keeps every record it must keep"+ },+ {+ "suite": "gfsRuntime.test.ts",+ "fullName": "GFS runtime bootstrap T8: the serialized recovery-required line names the writer refusal branch as writerDetail"+ },+ {+ "suite": "gfsStoreWriterFence.test.ts",+ "fullName": "GFS writer fence across a persistent-disk reattach (#1028) T9: assertInitialized and assertWriterOwnership propagate ownership_lost"}]