Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): fence expired and cancelled file processing

Alfredo Lopeza9844e2305b7
diff --git a/desktop-app/src/__tests__/gfsLargeFileCsvFixture.test.ts b/desktop-app/src/__tests__/gfsLargeFileCsvFixture.test.ts
new file mode 100644
index 000000000..7ec0cc34e
--- /dev/null
+++ b/desktop-app/src/__tests__/gfsLargeFileCsvFixture.test.ts
@@ -0,0 +1,150 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+ GFS_LARGE_CSV_SIZE,
+ GFS_OLD_VISUAL_LIMIT,
+ buildGfsLargeCsvFixture,
+ countMissingCsvColumns,
+ hasCsvDataRecordCount,
+ parseCsvMetadata,
+ resolveGfsLargeCsvFixture,
+} from '../../test/e2e-playwright/helpers/gfsLargeFileCsvFixture'
+
+describe('large-file CSV fixture oracle', () => {
+ afterEach(() => vi.unstubAllEnvs())
+
+ it('derives header fields and counts multiline records with escaped quotes', () => {
+ const buffer = Buffer.from(
+ '\uFEFF"record,id","said ""hello""","multi\nline"\r\n' +
+ '"row,1","value with ""quotes""","line one\r\nline two"\r\n' +
+ 'row-2,,tail'
+ )
+ expect(parseCsvMetadata(buffer)).toEqual({
+ columns: ['record,id', 'said "hello"', 'multi\nline'],
+ dataRecordCount: 2,
+ })
+ })
+
+ it.each(['\n', '\r\n', '\r'])('handles %j record endings without counting blank lines', end => {
+ expect(parseCsvMetadata(Buffer.from(`first,second${end}${end}1,2${end}3,4${end}`))).toEqual({
+ columns: ['first', 'second'],
+ dataRecordCount: 2,
+ })
+ })
+
+ it('counts empty fields and preserves Unicode header names', () => {
+ expect(parseCsvMetadata(Buffer.from('á,列,📁\n,,\n"","",'))).toEqual({
+ columns: ['á', '列', '📁'],
+ dataRecordCount: 2,
+ })
+ })
+
+ it.each([
+ ['a,b\nrow,"unfinished', 'unterminated quoted field'],
+ ['a,b\nrow,"closed"trailer\n', 'content after a quote'],
+ ['a,b\nrow,un"quoted\n', 'Unexpected CSV quote'],
+ ['a,b\nrow\n', 'inconsistent column count'],
+ ['a,b\nrow,value,extra\n', 'inconsistent column count'],
+ ['', 'header record'],
+ ])('rejects malformed CSV %j without echoing record values', (text, message) => {
+ expect(() => parseCsvMetadata(Buffer.from(text))).toThrow(message)
+ })
+
+ it('rejects invalid UTF-8 instead of deriving replacement-character columns', () => {
+ expect(() => parseCsvMetadata(Buffer.from([0xff]))).toThrow()
+ })
+
+ it('accepts a header-only CSV with no data records', () => {
+ expect(parseCsvMetadata(Buffer.from('first,second'))).toEqual({
+ columns: ['first', 'second'],
+ dataRecordCount: 0,
+ })
+ })
+
+ it.each(['size', 'digest'] as const)('keeps the opt-in original %s guard', guard => {
+ const directory = mkdtempSync(join(tmpdir(), 'gfs-large-csv-guard-'))
+ const filePath = join(directory, 'invalid-original.csv')
+ try {
+ // A synthetic invalid file exercises each guard without the original
+ // customer input, and must fail before the CSV parser sees any records.
+ writeFileSync(filePath, guard === 'size' ? 'id\n' : Buffer.alloc(GFS_LARGE_CSV_SIZE, 0x61), {
+ mode: 0o600,
+ })
+ vi.stubEnv('E2E_GFS_LARGE_CSV_PATH', filePath)
+ expect(() => resolveGfsLargeCsvFixture()).toThrow(
+ guard === 'size' ? 'must contain exactly' : 'digest mismatch'
+ )
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+
+ it('independently counts the exact-size synthetic file past the former inline boundary', () => {
+ const csv = buildGfsLargeCsvFixture()
+ const text = csv.buffer.toString('utf8')
+ expect(csv.buffer.byteLength).toBe(GFS_LARGE_CSV_SIZE)
+ expect(csv.buffer.indexOf(csv.sentinel!, 'utf8')).toBeGreaterThan(GFS_OLD_VISUAL_LIMIT)
+ expect(csv.buffer.indexOf(csv.lastRecordId!, 'utf8')).toBeGreaterThan(GFS_OLD_VISUAL_LIMIT)
+ expect(csv.columns).toEqual(['id', 'record name', 'notes,value', 'value'])
+ expect(csv.dataRecordCount).toBe((text.match(/^"record-\d+"/gm) ?? []).length + 1)
+ expect(text.split('\n').length - 1).toBeGreaterThan(csv.dataRecordCount + 1)
+ expect(text).toContain(`"${csv.lastRecordId}","final, sentinel row"`)
+ expect(csv.lastRecordId).toMatch(/^record-final-[0-9a-f-]{36}$/)
+ expect(text).toContain('""""escaped quotes""""')
+ expect(csv.source).toBe('synthetic')
+ })
+})
+
+describe('large-file CSV summary assertions', () => {
+ it.each([
+ ['Hay 923 registros de datos, sin contar la cabecera.', 923, true],
+ ['**Registros de datos (sin cabecera):** 923.', 923, true],
+ ['Rows (excluding header): 923', 923, true],
+ ['data_record_count: 923', 923, true],
+ ['El archivo tiene **923** filas.', 923, true],
+ ['Contiene 38.410 registros.', 38_410, true],
+ ['Records: 38,410', 38_410, true],
+ ['38 410 data rows', 38_410, true],
+ ['Checksum: abc923def. Rows: 924', 923, false],
+ ['Checksum: 923; filas: 924', 923, false],
+ ['Columnas: 923; registros: 924', 923, false],
+ ['Last record ID: 923. There are 924 data records.', 923, false],
+ ['Último registro identificador: 923. Contiene 924 registros.', 923, false],
+ ['Registros IDs: 923; contiene 924 registros.', 923, false],
+ ['Registros: 9,2,3', 923, false],
+ ['Registros: 1923', 923, false],
+ ['Registros: 9230', 923, false],
+ ['Registros: -923', 923, false],
+ ['Registros: 923.4', 923, false],
+ ] as const)('binds the expected count to record meaning in %j', (summary, count, matches) => {
+ expect(hasCsvDataRecordCount(summary, count)).toBe(matches)
+ })
+
+ it('does not let a compound header satisfy a missing shorter column', () => {
+ expect(countMissingCsvColumns('Columnas: notes,value', ['notes,value', 'value'])).toBe(1)
+ expect(
+ countMissingCsvColumns('Columnas: "notes,value", "value"', ['notes,value', 'value'])
+ ).toBe(0)
+ })
+
+ it('requires independent mentions for duplicate column names', () => {
+ expect(countMissingCsvColumns('Columnas: value', ['value', 'value'])).toBe(1)
+ expect(countMissingCsvColumns('Columnas: value, value', ['value', 'value'])).toBe(0)
+ })
+
+ it('matches complete field names rather than substrings or regular expressions', () => {
+ expect(countMissingCsvColumns('Columnas: valid', ['id'])).toBe(1)
+ expect(countMissingCsvColumns('Columnas: notesXvalue', ['notes.value'])).toBe(1)
+ expect(
+ countMissingCsvColumns('Columnas: á, 列, 📁, price (USD), A+B', [
+ 'á',
+ '列',
+ '📁',
+ 'price (USD)',
+ 'A+B',
+ ])
+ ).toBe(0)
+ })
+})
diff --git a/desktop-app/test/e2e-playwright/gfs-agent-large-file-cli.test.ts b/desktop-app/test/e2e-playwright/gfs-agent-large-file-cli.test.ts
index 4d1dbdc9f..84cc0f78a 100644
--- a/desktop-app/test/e2e-playwright/gfs-agent-large-file-cli.test.ts
+++ b/desktop-app/test/e2e-playwright/gfs-agent-large-file-cli.test.ts
@@ -5,10 +5,11 @@
* E2E contract (e2e-test-guardian):
* - Real user journey: login → Files → visible upload of an exact 3,836,961-byte
* CSV → exact managed agent chat → governed download → user-visible
- * attended shell approval → completed tool stepper and bounded response.
- * - Business signals: the download step reports a workspace file; the approved
- * generated program reports the independent data-record count and columns,
- * plus the last record ID beyond the former 3 MiB boundary.
+ * attended shell approvals → completed tool stepper and bounded response.
+ * - Business signals: the download step reports a workspace file; execution
+ * completes after attended review; the final response reports the independent
+ * data-record count and complete header. The synthetic file also proves its
+ * unique last record beyond 3 MiB. Tool output previews are diagnostic only.
* - Setup shortcuts: managed agent/folder fixture seeding and the local CSV
* file are named preconditions. No provider-route mock, storage mutation,
* direct API trigger, or broad network mock is used.
@@ -29,6 +30,8 @@ import {
GFS_LARGE_CSV_SIZE,
GFS_OLD_VISUAL_LIMIT,
type GfsLargeCsvFixture,
+ countMissingCsvColumns,
+ hasCsvDataRecordCount,
resolveGfsLargeCsvFixture,
} from './helpers/gfsLargeFileCsvFixture'
import { openAgentsPage, openResourcesNavItem } from './navigationHelpers'
@@ -37,6 +40,8 @@ import { launchAndLogin } from './workflowUi'
const OWNER_EMAIL = '[email protected]'
const RESPONSE_TIMEOUT_MS = 420_000
const PROGRESS_TIMEOUT_MS = 45_000
+// A 16 KiB ceiling bounds header/count metadata well below the 3.8 MB input.
+const CSV_SUMMARY_MAX_BYTES = 16_384
async function enterAgentChat(page: Page, agentName: string): Promise<void> {
await openAgentsPage(page)
@@ -68,6 +73,9 @@ async function sendTaskAndWaitForReviewedShell(
): Promise<{ response: Locator; expandButton: Locator }> {
const response = page.getByTestId('agent-response')
const approval = page.getByTestId('approval-approve-btn')
+ // This is a new chat. A completed stepper from a prior turn must never
+ // satisfy the completion signal below.
+ await expect(page.getByTestId('progress-stepper')).toHaveCount(0)
await page.getByTestId('chat-input').fill(prompt)
await page.getByTestId('send-button').click()
@@ -85,7 +93,7 @@ async function sendTaskAndWaitForReviewedShell(
await expect(details).toBeVisible()
await details.click()
const downloaded = completedToolStepRow(page, /gfs_download|gfs_read/)
- let receiptPath: string
+ let receiptPath: string | undefined
if (sourceMode === 'path') {
// The user supplied only a human path. An observed discovery result must
// precede the transfer; the harness never injects a resourceId into chat.
@@ -98,39 +106,48 @@ async function sendTaskAndWaitForReviewedShell(
const firstTransfer = steps.findIndex(value => /gfs_download|gfs_read/.test(value))
expect(firstDiscovery).toBeGreaterThanOrEqual(0)
expect(firstTransfer).toBeGreaterThan(firstDiscovery)
- await expect(downloaded).toHaveCount(1)
- const downloadOutput = await stepOutput(downloaded)
- await expect(downloadOutput).toContainText('workspace_file')
- await expect(downloadOutput).toContainText(resourceId.replace(/-/g, ''))
- // The bounded UI preview exposes the receipt ID before source fields. Use
- // that ID to check the complete command and its exact file before approval.
- const receiptId = (await downloadOutput.innerText()).match(/"id":"([A-Za-z0-9_-]+)"/)?.[1]
- expect(receiptId).toBeTruthy()
- receiptPath = `.gfs-downloads/input-${receiptId}/source`
+ await expect.poll(() => downloaded.count()).toBeGreaterThan(0)
+ const commandText = await commandPreview.innerText()
+ for (const row of await downloaded.all()) {
+ const downloadOutput = await stepOutput(row)
+ const outputText = await downloadOutput.innerText()
+ if (!outputText.includes('workspace_file')) continue
+ const sourceId = outputText.match(/"source"\s*:\s*\{[^}]*"resourceId"\s*:\s*"([^"]+)"/)?.[1]
+ if (sourceId?.replace(/-/g, '') !== resourceId.replace(/-/g, '')) continue
+ const receiptId = outputText.match(/"id"\s*:\s*"([A-Za-z0-9_-]+)"/)?.[1]
+ if (!receiptId) continue
+ // The store's receipt ID maps to this local path. The receipt's visible
+ // source binds the transfer to the upload; an ID alone does not prove it.
+ // The bounded preview does not certify fields such as source.version.
+ const candidatePath = `.gfs-downloads/input-${receiptId}/source`
+ if (commandText.includes(candidatePath)) receiptPath = candidatePath
+ }
+ expect(
+ receiptPath,
+ 'The reviewed command must use a receipt for the uploaded source'
+ ).toBeTruthy()
} else {
- // The attached reference is prepared before the first model request. The
- // model must use that local receipt without another GFS content tool call.
- await expect(toolStepRow(page, 'gfs_download')).toHaveCount(0)
- await expect(toolStepRow(page, 'gfs_read')).toHaveCount(0)
+ // An attached reference is prepared before the first model request. The
+ // agent may also revalidate it through the normal GFS tools.
const path = (await commandPreview.innerText()).match(
/\.gfs-downloads\/input-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/source/
)?.[0]
expect(path).toBeTruthy()
receiptPath = path!
}
- await expect(commandPreview).toContainText(receiptPath)
+ await expect(commandPreview).toContainText(receiptPath!)
await expect(completedToolStepRow(page, 'shell_exec')).toHaveCount(0)
// The agent creates its own program from the normal business request. A
- // reviewer inspects the complete visible command and approves once in the
- // UI; keyword matching cannot authorize arbitrary generated code.
+ // reviewer inspects each complete visible command and approves individually
+ // in the UI; keyword matching cannot authorize arbitrary generated code.
await test.info().attach('attended-shell-review-required', {
contentType: 'text/plain',
- body: 'Review the complete command and referenced script, verify read-only access to this receipt and bounded summary output, then approve once in the owned Desktop window.',
+ body: 'Review each complete command and referenced script, verify read-only access to this receipt and bounded summary output, then approve each request individually in the owned Desktop window.',
})
- await expect(approval).toHaveCount(0, { timeout: RESPONSE_TIMEOUT_MS })
- await expect(completedToolStepRow(page, 'shell_exec')).toHaveCount(1, {
+ await expect(page.getByTestId('progress-stepper')).toHaveClass(/\bstatus-completed\b/, {
timeout: RESPONSE_TIMEOUT_MS,
})
+ await expect(approval).toHaveCount(0, { timeout: RESPONSE_TIMEOUT_MS })
await expect(response).toBeVisible({ timeout: RESPONSE_TIMEOUT_MS })
const expandButton = page.getByTestId('progress-expand-btn')
await expect(expandButton).toHaveCount(1, { timeout: PROGRESS_TIMEOUT_MS })
@@ -151,12 +168,12 @@ function completedToolStepRow(page: Page, toolName: string | RegExp): Locator {
async function stepOutput(row: Locator): Promise<Locator> {
const toolCallId = (await row.getAttribute('data-testid'))?.slice('step-row-'.length)
expect(toolCallId).toBeTruthy()
- await row.click()
const output = row
.page()
.getByTestId('step-output-panel')
.and(row.page().locator(`[data-tool-call-id=${JSON.stringify(toolCallId)}]`))
.locator('.stepper-step-output-code')
+ if (!(await output.isVisible())) await row.click()
await expect(output).toBeVisible({ timeout: 10_000 })
return output
}
@@ -308,34 +325,41 @@ test.describe('GFS large-file business analysis with attended approval', () => {
if ((await expandButton.getAttribute('aria-expanded')) === 'false')
await expandButton.click()
const downloadRow = completedToolStepRow(page, /gfs_download|gfs_read/)
- await expect(downloadRow).toHaveCount(sourceMode === 'path' ? 1 : 0, { timeout: 15_000 })
if (sourceMode === 'path') {
- await expect(downloadRow).toBeVisible({ timeout: 15_000 })
+ expect(await downloadRow.count()).toBeGreaterThan(0)
await expect(downloadRow.locator('.stepper-step-duration.state-error')).toHaveCount(0)
- await expect(downloadRow).toContainText(/gfs_download|gfs_read/)
- } else {
- await expect(toolStepRow(page, 'gfs_download')).toHaveCount(0)
- await expect(toolStepRow(page, 'gfs_read')).toHaveCount(0)
}
const shellRow = completedToolStepRow(page, 'shell_exec')
- await expect(shellRow).toHaveCount(1, { timeout: 15_000 })
- await expect(shellRow).toBeVisible({ timeout: 15_000 })
+ expect(await shellRow.count()).toBeGreaterThan(0)
await expect(shellRow.locator('.stepper-step-duration.state-error')).toHaveCount(0)
- const shellOutput = await stepOutput(shellRow)
- const count = String(csv.recordCount - 1)
- const expectedRows = new RegExp(`(?<![0-9])${count.split('').join('[.,\\s]*')}(?![0-9])`)
- await expect(shellOutput).toContainText(expectedRows)
- await expect(response).toContainText(expectedRows, { timeout: RESPONSE_TIMEOUT_MS })
+ for (const row of await shellRow.all()) {
+ await expect(row).toBeVisible()
+ }
+ // The turn is complete. Tool previews cap each line and show only
+ // the tail, so business metadata belongs to the final response.
+ const responseSummary = await response.innerText()
+ expect(
+ Buffer.byteLength(responseSummary, 'utf8'),
+ 'The response must remain a bounded CSV summary'
+ ).toBeLessThanOrEqual(CSV_SUMMARY_MAX_BYTES)
+ expect(
+ hasCsvDataRecordCount(responseSummary, csv.dataRecordCount),
+ 'Response must report the independent count as CSV data records'
+ ).toBe(true)
+ // Original header values remain in fixture memory. Failure messages
+ // report only the number of missing columns, never customer names.
+ expect(
+ countMissingCsvColumns(responseSummary, csv.columns),
+ 'Response must include every independently parsed CSV column'
+ ).toBe(0)
if (csv.source === 'synthetic') {
- for (const column of ['id', 'record name', 'notes,value', 'value']) {
- await expect(shellOutput).toContainText(column)
- await expect(response).toContainText(column)
- }
- await expect(shellOutput).toContainText('record-final')
- await expect(response).toContainText('record-final')
+ await expect(response).toContainText(csv.lastRecordId!)
}
- await expect(response).not.toContainText('zzzzzzzzzz')
+ expect(
+ responseSummary.includes('zzzzzzzzzz'),
+ 'Response must not dump synthetic padding'
+ ).toBe(false)
})
await test.step('source remains visible after read-only processing', async () => {
diff --git a/desktop-app/test/e2e-playwright/helpers/gfsLargeFileCsvFixture.ts b/desktop-app/test/e2e-playwright/helpers/gfsLargeFileCsvFixture.ts
index 6f43122bc..efadc6a3d 100644
--- a/desktop-app/test/e2e-playwright/helpers/gfsLargeFileCsvFixture.ts
+++ b/desktop-app/test/e2e-playwright/helpers/gfsLargeFileCsvFixture.ts
@@ -1,3 +1,4 @@
+import { isUtf8 } from 'node:buffer'
import * as crypto from 'node:crypto'
import * as fs from 'node:fs'
import * as path from 'node:path'
@@ -9,28 +10,75 @@ export const GFS_OLD_VISUAL_LIMIT = 3_145_728
export const GFS_LARGE_CSV_INCIDENT_SHA256 =
'675b72d7ba4c4c6eca3a49a076ed6adcff72165805e2069bc3619bca333c9bd1'
-export interface GfsLargeCsvFixture {
+export interface CsvMetadata {
+ columns: string[]
+ dataRecordCount: number
+}
+
+export interface GfsLargeCsvFixture extends CsvMetadata {
fileName: string
sourcePath?: string
buffer: Buffer
- recordCount: number
sentinel?: string
+ lastRecordId?: string
tailProof: string
source: 'synthetic' | 'actual'
}
+function escapeRegExp(value: string): string {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/** Require a record label, so a receipt ID or checksum cannot supply the count. */
+export function hasCsvDataRecordCount(summary: string, count: number): boolean {
+ const digits = String(count).replace(/\B(?=(\d{3})+(?!\d))/g, '[.,\\s]?')
+ const number = `(?<![\\p{L}\\p{N}_\\-\\u2212])${digits}(?![\\p{L}\\p{N}_]|[.,]\\d)`
+ const countLabel =
+ '\\b(?:registros|filas|(?:data[ _-]*)?(?:records|rows|(?:record|row)[ _-]*count))\\b'
+ const recordLabel = '\\b(?:registros?|filas?|(?:data[ _-]*)?(?:records?|rows?))\\b'
+ const description =
+ '(?:(?!\\b(?:ids?|identificadores?|identifiers?|indices|índices|indexes?|positions?|posiciones)\\b)[^\\d\\n\\r.;]){0,80}'
+ const afterNumber = '[\\s*\\x60_]{1,12}'
+ return new RegExp(
+ `(?:${countLabel}${description}${number}|${number}${afterNumber}${recordLabel})`,
+ 'iu'
+ ).test(summary)
+}
+
+/** Count distinct header mentions without returning private field names. */
+export function countMissingCsvColumns(summary: string, columns: readonly string[]): number {
+ const normalize = (value: string): string => value.replace(/\s+/g, ' ').trim()
+ let remaining = normalize(summary)
+ let missing = 0
+ // Consume longer names first: "notes,value" cannot also prove "value".
+ for (const column of columns.map(normalize).sort((left, right) => right.length - left.length)) {
+ const pattern = new RegExp(
+ `(?<![\\p{L}\\p{N}_])${escapeRegExp(column)}(?![\\p{L}\\p{N}_])`,
+ 'u'
+ )
+ const match = column ? pattern.exec(remaining) : null
+ if (!match) missing += 1
+ else
+ remaining =
+ remaining.slice(0, match.index) +
+ ' '.repeat(match[0].length) +
+ remaining.slice(match.index + match[0].length)
+ }
+ return missing
+}
+
function csvField(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
export function buildGfsLargeCsvFixture(): GfsLargeCsvFixture {
const sentinel = `GFS_LARGE_CSV_SENTINEL_${crypto.randomUUID()}`
+ const lastRecordId = `record-final-${crypto.randomUUID()}`
let content = `${csvField('id')},${csvField('record name')},${csvField(
'notes,value'
)},${csvField('value')}\n`
let contentBytes = Buffer.byteLength(content, 'utf8')
- let recordCount = 1
- const finalPrefix = `${csvField('record-final')},${csvField('final, sentinel row')},"${sentinel}`
+ const finalPrefix = `${csvField(lastRecordId)},${csvField('final, sentinel row')},"${sentinel}`
const closing = `",${csvField('0')}\n`
const minimumFinalLength = Buffer.byteLength(finalPrefix + closing, 'utf8')
@@ -46,14 +94,12 @@ export function buildGfsLargeCsvFixture(): GfsLargeCsvFixture {
if (contentBytes + rowBytes + minimumFinalLength > GFS_LARGE_CSV_SIZE) break
content += row
contentBytes += rowBytes
- recordCount += 1
}
const usedBytes = contentBytes + minimumFinalLength
if (usedBytes > GFS_LARGE_CSV_SIZE) throw new Error('GFS large CSV fixture sizing underflow')
const padding = 'z'.repeat(GFS_LARGE_CSV_SIZE - usedBytes)
content += `${finalPrefix}${padding}${closing}`
- recordCount += 1
const buffer = Buffer.from(content, 'utf8')
if (buffer.byteLength !== GFS_LARGE_CSV_SIZE)
@@ -64,8 +110,9 @@ export function buildGfsLargeCsvFixture(): GfsLargeCsvFixture {
return {
fileName: `gfs-large-file-${crypto.randomUUID()}.csv`,
buffer,
- recordCount,
+ ...parseCsvMetadata(buffer),
sentinel,
+ lastRecordId,
tailProof: crypto
.createHash('sha256')
.update(buffer.subarray(Math.max(0, buffer.byteLength - 4096)))
@@ -75,24 +122,74 @@ export function buildGfsLargeCsvFixture(): GfsLargeCsvFixture {
}
}
-function countCsvRecords(buffer: Buffer): number {
- let records = 0
- let insideQuotes = false
- let sawRecordBytes = false
- for (const byte of buffer) {
- if (byte === 0x22) {
- insideQuotes = !insideQuotes
- sawRecordBytes = true
+/** Independent CSV oracle; retain the header, never the data-record values. */
+export function parseCsvMetadata(buffer: Buffer): CsvMetadata {
+ if (!isUtf8(buffer)) throw new Error('CSV must contain valid UTF-8')
+ const columns: string[] = []
+ let dataRecordCount = 0
+ let header = true
+ let field: number[] = []
+ let fieldCount = 0
+ let state: 'start' | 'unquoted' | 'quoted' | 'closed' = 'start'
+ let recordStarted = false
+
+ const finishField = (): void => {
+ if (header) {
+ columns.push(Buffer.from(field).toString('utf8'))
+ field = []
+ }
+ fieldCount += 1
+ state = 'start'
+ }
+ const finishRecord = (): void => {
+ finishField()
+ if (header) header = false
+ else {
+ if (fieldCount !== columns.length)
+ throw new Error(`CSV data record ${dataRecordCount + 1} has an inconsistent column count`)
+ dataRecordCount += 1
+ }
+ fieldCount = 0
+ recordStarted = false
+ }
+
+ const start = buffer.subarray(0, 3).equals(Buffer.from([0xef, 0xbb, 0xbf])) ? 3 : 0
+ for (let index = start; index < buffer.length; index += 1) {
+ const byte = buffer[index]!
+ if (state === 'quoted') {
+ if (byte === 0x22) {
+ if (buffer[index + 1] === 0x22) {
+ if (header) field.push(0x22)
+ index += 1
+ } else state = 'closed'
+ } else if (header) field.push(byte)
+ continue
+ }
+ if (byte === 0x2c) {
+ finishField()
+ recordStarted = true
continue
}
- if (byte === 0x0a && !insideQuotes) {
- records += 1
- sawRecordBytes = false
+ if (byte === 0x0a || byte === 0x0d) {
+ // Empty physical lines are not data records; quoted empty fields are.
+ if (recordStarted) finishRecord()
+ if (byte === 0x0d && buffer[index + 1] === 0x0a) index += 1
continue
}
- if (byte !== 0x0d) sawRecordBytes = true
+ if (state === 'closed') throw new Error(`Unexpected CSV content after a quote at ${index}`)
+ if (byte === 0x22) {
+ if (state !== 'start') throw new Error(`Unexpected CSV quote at ${index}`)
+ state = 'quoted'
+ } else {
+ state = 'unquoted'
+ if (header) field.push(byte)
+ }
+ recordStarted = true
}
- return records + (sawRecordBytes ? 1 : 0)
+ if (state === 'quoted') throw new Error('CSV contains an unterminated quoted field')
+ if (recordStarted) finishRecord()
+ if (header) throw new Error('CSV must contain a header record')
+ return { columns, dataRecordCount }
}
/**
@@ -120,7 +217,7 @@ export function resolveGfsLargeCsvFixture(): GfsLargeCsvFixture {
fileName: path.basename(explicitPath),
sourcePath: explicitPath,
buffer,
- recordCount: countCsvRecords(buffer),
+ ...parseCsvMetadata(buffer),
tailProof: crypto
.createHash('sha256')
.update(buffer.subarray(Math.max(0, buffer.byteLength - 4096)))
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 3d9a15dd4..d74aa345a 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.487",
+ "version": "0.1.488",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.487",
+ "version": "0.1.488",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index edc364693..cef9cdc9c 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.487",
+ "version": "0.1.488",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/core/tools/__tests__/shellProcessingLeaseCancellation.test.ts b/mcp-host/src/core/tools/__tests__/shellProcessingLeaseCancellation.test.ts
new file mode 100644
index 000000000..356a544bb
--- /dev/null
+++ b/mcp-host/src/core/tools/__tests__/shellProcessingLeaseCancellation.test.ts
@@ -0,0 +1,68 @@
+import { beforeEach, expect, it, vi } from 'vitest'
+import * as os from 'node:os'
+import type {
+ GfsProcessingLease,
+ GfsProcessingLeaseProvider,
+} from '../../../internalTools/gfsProcessingLease'
+import { ShellTool } from '../shell'
+
+const mocks = vi.hoisted(() => ({ spawn: vi.fn() }))
+// The delayed lease is the race under test; no process may start in this suite.
+vi.mock('child_process', () => ({ spawn: mocks.spawn }))
+
+beforeEach(() => vi.clearAllMocks())
+
+function delayedLease() {
+ let finishAcquisition!: (lease: GfsProcessingLease) => void
+ const pending = new Promise<GfsProcessingLease>(resolve => {
+ finishAcquisition = resolve
+ })
+ const provider: GfsProcessingLeaseProvider = {
+ acquireProcessingLease: vi.fn(() => pending),
+ releaseProcessingLease: vi.fn(async () => undefined),
+ }
+ const lease = {
+ leaseId: 'cancelled-admission',
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ return { provider, lease, finishAcquisition }
+}
+
+it('does not spawn and releases a lease acquired after cancellation', async () => {
+ const { provider, lease, finishAcquisition } = delayedLease()
+ const controller = new AbortController()
+ const reason = new Error('cancelled during processing lease acquisition')
+ const tool = new ShellTool(os.tmpdir(), 5_000, [], () => ({}), undefined, provider)
+ const execution = tool.execute(
+ { command: 'printf done' },
+ { signal: controller.signal, onOutput: vi.fn() }
+ )
+ expect(provider.acquireProcessingLease).toHaveBeenCalledOnce()
+
+ controller.abort(reason)
+ finishAcquisition(lease)
+
+ await expect(execution).rejects.toBe(reason)
+ expect(mocks.spawn).not.toHaveBeenCalled()
+ expect(provider.releaseProcessingLease).toHaveBeenCalledExactlyOnceWith(lease)
+})
+
+it('keeps cancellation before spawn when releasing the admitted lease fails', async () => {
+ const { provider, lease, finishAcquisition } = delayedLease()
+ vi.mocked(provider.releaseProcessingLease).mockRejectedValueOnce(new Error('ledger unavailable'))
+ const controller = new AbortController()
+ const tool = new ShellTool(os.tmpdir(), 5_000, [], () => ({}), undefined, provider)
+ const execution = tool.execute(
+ { command: 'printf done' },
+ { signal: controller.signal, onOutput: vi.fn() }
+ )
+
+ controller.abort()
+ finishAcquisition(lease)
+
+ const result = await execution
+ expect(result.is_error).toBe(true)
+ expect(result.content).toContain('processing_lease_release_failed')
+ expect(mocks.spawn).not.toHaveBeenCalled()
+ expect(provider.releaseProcessingLease).toHaveBeenCalledExactlyOnceWith(lease)
+})
diff --git a/mcp-host/src/core/tools/shell.ts b/mcp-host/src/core/tools/shell.ts
index 94691f50a..f57e82c9d 100644
--- a/mcp-host/src/core/tools/shell.ts
+++ b/mcp-host/src/core/tools/shell.ts
@@ -206,6 +206,21 @@ export class ShellTool implements Tool {
}
}
+ if (context?.signal?.aborted) {
+ if (processingLease && this.processingLeases) {
+ try {
+ await this.processingLeases.releaseProcessingLease(processingLease)
+ } catch {
+ return {
+ content: 'Command cancelled before process start [processing_lease_release_failed]',
+ duration_ms: Date.now() - startTime,
+ is_error: true,
+ }
+ }
+ }
+ context.signal.throwIfAborted()
+ }
+
return new Promise<ToolOutput>(resolve => {
// detached: true makes the child a process group leader so we can kill
// the whole group (shell + grandchildren like `sleep`) with -pid signal.
diff --git a/mcp-host/src/internalTools/gfsDownloadStore.test.ts b/mcp-host/src/internalTools/gfsDownloadStore.test.ts
index 4094862da..5b0f41453 100644
--- a/mcp-host/src/internalTools/gfsDownloadStore.test.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStore.test.ts
@@ -217,6 +217,60 @@ describe('GFS download store', () => {
})
})
+ it('releases a failed transfer whose directory is already absent', async () => {
+ const transfer = await store.createTransfer({
+ callerIdentity: 'caller-a',
+ callerWorkspacePath: callerRoot,
+ source,
+ sizeBytes: 1024,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ })
+ // Crash-after-rm analogue: the directory is gone while the durable charge
+ // and its ledger entry remain.
+ await fs.rm(path.join(callerRoot, path.dirname(transfer.path)), {
+ recursive: true,
+ force: true,
+ })
+
+ await store.fail(transfer.id, 'caller-a')
+
+ expect(store.debugRecord(transfer.id)).toBeUndefined()
+ expect(store.debugUsage()).toEqual({ bytes: 0, files: 0, callerBytes: {}, callerFiles: {} })
+ })
+
+ it('keeps the charge when absence cannot be proven at lstat', async () => {
+ const transfer = await store.createTransfer({
+ callerIdentity: 'caller-a',
+ callerWorkspacePath: callerRoot,
+ source,
+ sizeBytes: 7,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ })
+ await fs.writeFile(path.join(callerRoot, `${transfer.path}.partial`), 'fixture')
+ await store.publish(
+ transfer.id,
+ 'caller-a',
+ createHash('sha256').update('fixture').digest('hex')
+ )
+ const record = store.debugRecord(transfer.id)!
+ record.expiresAt = new Date(Date.now() - 1).toISOString()
+ await store.debugPersist()
+ // A regular file where the download directory belongs makes lstat fail with
+ // ENOTDIR. Only ENOENT proves absence; this must stay charged.
+ await fs.rm(path.join(callerRoot, '.gfs-downloads'), { recursive: true, force: true })
+ await fs.writeFile(path.join(callerRoot, '.gfs-downloads'), 'not-a-directory')
+
+ await store.cleanupExpired()
+
+ expect(store.debugRecord(transfer.id)).toMatchObject({ state: 'cleanup_failed' })
+ expect(store.debugUsage()).toEqual({
+ bytes: 7,
+ files: 1,
+ callerBytes: { 'caller-a': 7 },
+ callerFiles: { 'caller-a': 1 },
+ })
+ })
+
it('reserves active-transfer slots atomically per caller and Host', async () => {
const first = await store.createTransfer({
callerIdentity: 'caller-a',
@@ -333,8 +387,10 @@ describe('GFS download store', () => {
expect(store.debugRecord(transfer.id)?.sha256).toBeUndefined()
vi.restoreAllMocks()
- const directory = path.dirname(path.join(callerRoot, transfer.path))
- await fs.rm(directory, { recursive: true, force: true })
+ // Force an unproven absence: a regular file where the download directory
+ // belongs makes lstat fail with ENOTDIR, so the charge must stay reserved.
+ await fs.rm(path.join(callerRoot, '.gfs-downloads'), { recursive: true, force: true })
+ await fs.writeFile(path.join(callerRoot, '.gfs-downloads'), 'not-a-directory')
await expect(store.fail(transfer.id, 'caller-a')).rejects.toMatchObject({
code: 'storage_write_failed',
})
diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.ts
index d0e10885f..af001866c 100644
--- a/mcp-host/src/internalTools/gfsDownloadStore.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts
@@ -527,10 +527,28 @@ export class GfsDownloadStore {
if (record.callerIdentity !== callerIdentity)
throw new GfsDownloadStoreError('caller_mismatch')
const directory = path.join(this.hostRoot, record.directory)
+ let info: Awaited<ReturnType<typeof fs.lstat>>
+ try {
+ info = await fs.lstat(directory)
+ } catch (error) {
+ // A proven ENOENT means the directory was already removed; release the
+ // durable charge instead of stranding an unrecoverable cleanup_failed
+ // entry. Any other failure stays charged.
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+ record.state = 'cleanup_failed'
+ await this.persist()
+ throw new GfsDownloadStoreError('storage_write_failed')
+ }
+ delete this.ledger.records[id]
+ await this.persist()
+ return
+ }
+ if (!info.isDirectory() || info.isSymbolicLink()) {
+ record.state = 'cleanup_failed'
+ await this.persist()
+ throw new GfsDownloadStoreError('storage_write_failed')
+ }
try {
- const info = await fs.lstat(directory)
- if (!info.isDirectory() || info.isSymbolicLink())
- throw new GfsDownloadStoreError('workspace_unavailable')
await fs.rm(directory, { recursive: true, force: true })
} catch {
record.state = 'cleanup_failed'
@@ -709,22 +727,25 @@ export class GfsDownloadStore {
if (!Number.isSafeInteger(requestedMs) || requestedMs <= 0 || requestedMs > 3_600_000)
throw new GfsDownloadStoreError('download_busy')
- const now = Date.now()
- const candidates = Object.values(this.ledger.records).filter(
- record =>
- record.callerIdentity === callerIdentity &&
- record.state === 'completed' &&
- record.sha256 !== undefined &&
- Date.parse(record.expiresAt) > now
- )
- if (candidates.length === 0) {
- return {
- leaseId: randomUUID(),
- expiresAt: new Date(now + requestedMs).toISOString(),
- }
- }
-
return this.serialize(async () => {
+ const now = Date.now()
+ const retained = Object.values(this.ledger.records).filter(
+ record => record.callerIdentity === callerIdentity
+ )
+ // Admission covers this caller's retained files without parsing the
+ // command. Expired bytes can remain while an existing execution holds
+ // them, but they must not authorize another execution before cleanup.
+ if (retained.some(record => Date.parse(record.expiresAt) <= now))
+ throw new GfsDownloadStoreError('download_expired')
+ const candidates = retained.filter(
+ record => record.state === 'completed' && record.sha256 !== undefined
+ )
+ if (candidates.length === 0) {
+ return {
+ leaseId: randomUUID(),
+ expiresAt: new Date(now + requestedMs).toISOString(),
+ }
+ }
for (const record of candidates) await this.inspect(record.path, callerIdentity)
const leaseId = randomUUID()
const expiresAt = new Date(now + requestedMs).toISOString()
@@ -828,8 +849,23 @@ export class GfsDownloadStore {
if (this.activeIds.has(record.id)) continue
if (this.hasProcessingLease(record.id, now)) continue
const directory = path.join(this.hostRoot, record.directory)
+ let info: Awaited<ReturnType<typeof fs.lstat>>
+ try {
+ info = await fs.lstat(directory)
+ } catch (error) {
+ // Only a proven ENOENT establishes that nothing remains to delete.
+ // Permission errors, ENOTDIR and other uncertain outcomes stay charged.
+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+ removed.push(record.id)
+ continue
+ }
+ if (record.state !== 'cleanup_failed') {
+ record.state = 'cleanup_failed'
+ stateChanged = true
+ }
+ continue
+ }
try {
- const info = await fs.lstat(directory)
const realDirectory = await fs.realpath(directory)
if (!info.isDirectory() || info.isSymbolicLink() || realDirectory !== directory) {
if (record.state !== 'cleanup_failed') {
diff --git a/mcp-host/src/internalTools/gfsDownloadStoreProcessingLease.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreProcessingLease.test.ts
index bfbe63c77..5be8e87cf 100644
--- a/mcp-host/src/internalTools/gfsDownloadStoreProcessingLease.test.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStoreProcessingLease.test.ts
@@ -65,6 +65,91 @@ describe('GfsDownloadStore processing leases', () => {
).rejects.toMatchObject({ code: 'caller_mismatch' })
})
+ it('rejects a new lease for an expired retained file before cleanup', async () => {
+ const receipt = await publishFixture()
+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse(receipt.expiresAt))
+
+ await expect(
+ store.processingLeaseProvider('caller-a').acquireProcessingLease()
+ ).rejects.toMatchObject({ code: 'download_expired' })
+ expect(fs.existsSync(path.join(callerRoot, receipt.path))).toBe(true)
+ expect(store.debugRecord(receipt.id)).toBeDefined()
+
+ await store.cleanupExpired()
+ const lease = await store.processingLeaseProvider('caller-a').acquireProcessingLease()
+ await store.processingLeaseProvider('caller-a').releaseProcessingLease(lease)
+ await store.close()
+ })
+
+ it('rejects a new lease after expiry while preserving an admitted execution', async () => {
+ const receipt = await publishFixture()
+ const provider = store.processingLeaseProvider('caller-a')
+ const admitted = await provider.acquireProcessingLease()
+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse(receipt.expiresAt) + 1)
+
+ await expect(provider.acquireProcessingLease()).rejects.toMatchObject({
+ code: 'download_expired',
+ })
+ await store.cleanupExpired()
+ expect(fs.existsSync(path.join(callerRoot, receipt.path))).toBe(true)
+ expect(store.debugRecord(receipt.id)).toBeDefined()
+
+ await provider.releaseProcessingLease(admitted)
+ await store.cleanupExpired()
+ expect(store.debugRecord(receipt.id)).toBeUndefined()
+ await store.close()
+ })
+
+ it('recovers admission when an expired record has no directory left to clean', async () => {
+ const receipt = await publishFixture()
+ // Crash-after-rm analogue: the directory is gone while the ledger entry and
+ // its charge remain. Cleanup must reconcile them, not strand them.
+ fs.rmSync(path.join(callerRoot, path.dirname(receipt.path)), { recursive: true, force: true })
+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse(receipt.expiresAt) + 1)
+
+ await expect(
+ store.processingLeaseProvider('caller-a').acquireProcessingLease()
+ ).rejects.toMatchObject({ code: 'download_expired' })
+
+ await store.cleanupExpired()
+
+ expect(store.debugRecord(receipt.id)).toBeUndefined()
+ expect(store.debugUsage()).toEqual({ bytes: 0, files: 0, callerBytes: {}, callerFiles: {} })
+
+ const lease = await store.processingLeaseProvider('caller-a').acquireProcessingLease()
+ await store.processingLeaseProvider('caller-a').releaseProcessingLease(lease)
+ await store.close()
+ })
+
+ it('keeps another caller admissible while one caller has an expired record', async () => {
+ const expired = await publishFixture()
+ const callerBRoot = path.join(hostRoot, 'users', 'caller-b')
+ fs.mkdirSync(callerBRoot, { recursive: true, mode: 0o700 })
+ const callerBTransfer = await store.createTransfer({
+ callerIdentity: 'caller-b',
+ callerWorkspacePath: callerBRoot,
+ source,
+ sizeBytes: 7,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ })
+ fs.writeFileSync(path.join(callerBRoot, callerBTransfer.partialPath), Buffer.from('fixture'))
+ await store.publish(
+ callerBTransfer.id,
+ 'caller-b',
+ createHash('sha256').update('fixture').digest('hex')
+ )
+ vi.spyOn(Date, 'now').mockReturnValue(Date.parse(expired.expiresAt) + 1)
+
+ await expect(
+ store.processingLeaseProvider('caller-a').acquireProcessingLease()
+ ).rejects.toMatchObject({ code: 'download_expired' })
+
+ const providerB = store.processingLeaseProvider('caller-b')
+ const leaseB = await providerB.acquireProcessingLease()
+ await providerB.releaseProcessingLease(leaseB)
+ await store.close()
+ })
+
it('protects an expired download from cleanup while leased', async () => {
const receipt = await publishFixture()
const lease = await store