Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(desktop): harden the chat-open benchmark

Alfredo Lopez4052914836d0
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index c6e151d98..9397d9bc0 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -180,6 +180,11 @@ jobs:
if: matrix.service == 'desktop-app'
run: node --test scripts/tests/subscription-private-ui-input.test.mjs
+ # Hermetic node:test suite of the chat-open benchmark harness (#1091).
+ - name: Test the chat-open benchmark harness
+ if: matrix.service == 'desktop-app'
+ run: node scripts/tests/run-node-test-files.mjs desktop-app/scripts/bench-chat-open/benchmark.test.mjs
+
# Hermetic subscription-image and Control API memory-calibration node:test
# suites. Their only installed dependency is mcp-host's @napi-rs/canvas,
# which this leg installs for the Desktop fixture checks above.
diff --git a/desktop-app/package.json b/desktop-app/package.json
index ed0582bac..d5b86f643 100644
--- a/desktop-app/package.json
+++ b/desktop-app/package.json
@@ -14,6 +14,7 @@
"icons:generate": "node ./scripts/generate-adaptive-icons.mjs",
"test": "vitest run",
"test:e2e": "vitest run --config test/e2e/vitest.config.ts",
+ "test:bench-harness": "node ../scripts/tests/run-node-test-files.mjs scripts/bench-chat-open/benchmark.test.mjs",
"test:e2e:playwright": "npm run build && playwright test --config test/e2e-playwright/playwright.config.ts",
"test:native:sandbox-find": "npm run build:main --silent && node ./scripts/run-native-sandbox-find.mjs",
"test:native:sandbox-shortcut-focus": "npm run build:main --silent && node ./scripts/run-native-sandbox-shortcut-focus.mjs",
diff --git a/desktop-app/scripts/bench-chat-open.mjs b/desktop-app/scripts/bench-chat-open.mjs
index 660c86e3a..98bcc7ce1 100644
--- a/desktop-app/scripts/bench-chat-open.mjs
+++ b/desktop-app/scripts/bench-chat-open.mjs
@@ -3,8 +3,10 @@
* ChatStore chat-open benchmark (issue #1091, plan sections 7.1 and 11).
*
* Run from desktop-app with Node 24 and installed HEAD dependencies:
- * node scripts/bench-chat-open.mjs --quick --versions v0.10.0,v0.11.0 --out /tmp/chat-open.json
- * node scripts/bench-chat-open.mjs --out /tmp/chat-open-full.json
+ * node scripts/bench-chat-open.mjs --quick --versions v0.10.0,v0.11.0 \
+ * --out <primary-checkout>/.local-notes/issue1091-evidence/chat-open-quick.json
+ * node scripts/bench-chat-open.mjs \
+ * --out <primary-checkout>/.local-notes/issue1091-evidence/chat-open-full.json
*
* --versions selects any comma-separated subset of v0.10.0,v0.11.0,HEAD.
* --quick uses 5 agents x 20 chats; both pending-cleanup values and the permanent
@@ -14,8 +16,9 @@
*
* Historical sources are extracted using git archive -> tar, made read-only and
* compiled with HEAD's node_modules/.bin/tsc into os.tmpdir(). HEAD snapshots the
- * live worktree (including uncommitted changes); its source hashes and
- * commit are recorded. No checkout, dependency installation or Electron launch.
+ * live worktree; its source hashes and commit are recorded. A HEAD with
+ * uncommitted changes is refused unless --allow-dirty is passed, and the JSON
+ * then records dirty: true. No checkout, dependency installation or Electron launch.
* The emitted runtime graph is audited before import. We bypass only Electron's
* binding module: a fresh ChatStore + setAuthorityScope reproduces the completed
* bind; HEAD's actual compiled runChatGetIndex implements the measured handler.
@@ -38,14 +41,15 @@
* Warm stage-p95 sums and paired warm-open p95 are both reported; this Node
* benchmark does not measure the renderer chat-open time.
*
- * --out defaults to a unique JSON path under os.tmpdir(). The selected output is
- * exclusively created; an existing file is an error. JSON is also printed to
- * stdout. Sources/data are removed even on failure; the JSON remains for review.
+ * --out is required: results are evidence and must not land in a directory the
+ * OS clears. The output is exclusively created; an existing file is an error.
+ * JSON is also printed to stdout. Sources/data are removed even on failure; the
+ * JSON remains for review.
*/
import assert from 'node:assert/strict'
import { promises as fs } from 'node:fs'
import { arch, cpus, loadavg, platform, tmpdir } from 'node:os'
-import { basename, dirname, join, resolve } from 'node:path'
+import { dirname, join, resolve } from 'node:path'
import { performance } from 'node:perf_hooks'
import { fileURLToPath, pathToFileURL } from 'node:url'
import {
@@ -71,10 +75,17 @@ const ADAPTER_FILES = new Map([
])
export function parseOptions(args) {
- const options = { versions: [...ADAPTER_FILES.keys()], quick: false, out: undefined, help: false }
+ const options = {
+ versions: [...ADAPTER_FILES.keys()],
+ quick: false,
+ allowDirty: false,
+ out: undefined,
+ help: false,
+ }
for (let index = 0; index < args.length; index += 1) {
const arg = args[index]
if (arg === '--quick') options.quick = true
+ else if (arg === '--allow-dirty') options.allowDirty = true
else if (arg === '--help') options.help = true
else if (arg === '--versions' || arg === '--out') {
const value = args[++index]
@@ -92,6 +103,9 @@ export function parseOptions(args) {
}
} else throw new Error(`Unknown option: ${arg}`)
}
+ if (!options.help && !options.out) {
+ throw Object.assign(new Error('--out is required'), { code: 'OUT_REQUIRED' })
+ }
return options
}
@@ -187,13 +201,13 @@ async function runScenario(runDir, adapter, compiled, dimensions) {
first.ms + active.ms + fullWarm.ms,
]
Object.keys(stageSamples).forEach((stage, index) => stageSamples[stage].push(values[index]))
- if (adapter.ref === 'HEAD') await context.settle()
+ if (context.settle) await context.settle()
const state = await inspectFixture(fixture, dimensions.chatsPerAgent)
const pendingTotal = dimensions.agents * dimensions.pendingCleanupsPerAgent
if (!adapter.handlesPendingCleanups || dimensions.cleanupFilesystem === 'EACCES') {
assert.equal(state.pending, pendingTotal)
assert.equal(state.retainedArtifactDirectories, pendingTotal)
- } else if (adapter.ref === 'v0.11.0') {
+ } else {
assert.equal(state.pending, 0)
assert.equal(state.retainedArtifactDirectories, 0)
}
@@ -269,7 +283,7 @@ export async function main(args = process.argv.slice(2)) {
const options = parseOptions(args)
if (options.help) {
process.stdout.write(
- 'Usage: node scripts/bench-chat-open.mjs [--quick] [--versions v0.10.0,v0.11.0,HEAD] [--out path]\n'
+ 'Usage: node scripts/bench-chat-open.mjs --out path [--quick] [--allow-dirty] [--versions v0.10.0,v0.11.0,HEAD]\n'
)
return
}
@@ -280,13 +294,19 @@ export async function main(args = process.argv.slice(2)) {
const desktopRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const repoRoot = git(desktopRoot, ['rev-parse', '--show-toplevel'])
const runDir = await fs.mkdtemp(join(tmpdir(), 'evenfire-chat-open-'))
- const out = options.out ?? join(tmpdir(), `${basename(runDir)}.json`)
+ const { out } = options
const started = performance.now()
try {
const prepared = []
for (const ref of options.versions) {
const { default: adapter } = await import(`./bench-chat-open/${ADAPTER_FILES.get(ref)}`)
- const compiled = await compileVersion({ adapter, repoRoot, desktopRoot, runDir })
+ const compiled = await compileVersion({
+ adapter,
+ repoRoot,
+ desktopRoot,
+ runDir,
+ allowDirty: options.allowDirty,
+ })
prepared.push({ adapter, compiled })
}
const results = []
diff --git a/desktop-app/scripts/bench-chat-open/benchmark.test.mjs b/desktop-app/scripts/bench-chat-open/benchmark.test.mjs
index 1631e63a2..137cee2ea 100644
--- a/desktop-app/scripts/bench-chat-open/benchmark.test.mjs
+++ b/desktop-app/scripts/bench-chat-open/benchmark.test.mjs
@@ -8,7 +8,7 @@ import { REPETITIONS, distribution, matrix, parseOptions } from '../bench-chat-o
import { captureFilesystemFailures } from './fixtures.mjs'
import { AUTHORITY_SCOPE } from './fixtures.mjs'
import head from './head.mjs'
-import { auditRuntime } from './sources.mjs'
+import { auditRuntime, checkWorktreeClean } from './sources.mjs'
import v011 from './v0.11.0.mjs'
const ts = createRequire(import.meta.url)('typescript')
@@ -24,13 +24,28 @@ test('quick retains both cleanup populations and the failing-filesystem scenario
})
test('version restriction preserves historical refs and rejects duplicates or command-like refs', () => {
- assert.deepEqual(parseOptions(['--versions', 'v0.11.0,v0.10.0', '--quick']).versions, [
- 'v0.11.0',
- 'v0.10.0',
- ])
- assert.throws(() => parseOptions(['--versions', 'v0.10.0,v0.10.0']))
+ assert.deepEqual(
+ parseOptions(['--versions', 'v0.11.0,v0.10.0', '--quick', '--out', 'r.json']).versions,
+ ['v0.11.0', 'v0.10.0']
+ )
+ assert.throws(() => parseOptions(['--versions', 'v0.10.0,v0.10.0', '--out', 'r.json']))
assert.throws(() => parseOptions(['--versions', '--help']))
- assert.throws(() => parseOptions(['--versions', 'origin/dev']))
+ assert.throws(() => parseOptions(['--versions', 'origin/dev', '--out', 'r.json']))
+})
+
+test('--out is required and --allow-dirty is opt-in', () => {
+ assert.throws(() => parseOptions(['--quick']), /--out is required/)
+ const options = parseOptions(['--quick', '--out', 'r.json'])
+ assert.equal(options.out.endsWith('r.json'), true)
+ assert.equal(options.allowDirty, false)
+ assert.equal(parseOptions(['--allow-dirty', '--out', 'r.json']).allowDirty, true)
+ assert.equal(parseOptions(['--help']).help, true)
+})
+
+test('a dirty HEAD tree is refused unless allowed, and then recorded as dirty', () => {
+ assert.equal(checkWorktreeClean('', false), false)
+ assert.throws(() => checkWorktreeClean(' M desktop-app/src/ipc.ts', false), /--allow-dirty/)
+ assert.equal(checkWorktreeClean(' M desktop-app/src/ipc.ts', true), true)
})
test('nearest-rank p95 uses the 29th of 30 observations and rejects missing/nonfinite samples', () => {
@@ -103,29 +118,49 @@ test('v0.11.0 adapter reports a cleanup error and still reads the catalog in han
assert.equal(context.cleanupErrors, 1)
})
-test('HEAD delegates to the supplied compiled export and drains cleanup only after measurement', async () => {
+test('HEAD delegates to the compiled export and settle waits for the cleanups it started', async () => {
const calls = []
+ let release
+ const gate = new Promise(resolve => {
+ release = resolve
+ })
+ const result = { chats: [1] }
const store = {
- async retryPendingDeleteCleanups(scope) {
+ async retryPendingDeleteCleanups(scope, agentRef) {
assert.deepEqual(scope, AUTHORITY_SCOPE)
- calls.push('settle')
+ calls.push(`cleanup:${agentRef}`)
+ await gate
+ calls.push('cleanup:end')
+ },
+ async getIndex(agentRef) {
+ calls.push(`index:${agentRef}`)
+ return result
},
}
- const result = { chats: [1] }
- // This distinct behavior witnesses delegation to the compiled export instead
- // of an adapter-owned copy of a historical store sequence.
+ // A distinct export witnesses delegation to the compiled handler instead of
+ // an adapter-owned copy of the store sequence.
const runChatGetIndex = async (deps, agentRef) => {
calls.push('export')
- assert.equal(deps.store, store)
- assert.equal(agentRef, 'synthetic')
- assert.deepEqual(deps.getChatDeletionFenceAuthority(), { authorityScope: AUTHORITY_SCOPE })
+ const { authorityScope } = deps.getChatDeletionFenceAuthority()
+ void deps.store
+ .retryPendingDeleteCleanups(authorityScope, agentRef)
+ .catch(deps.warnCleanupFailure)
deps.warnCleanupFailure(new Error('Synthetic warning'))
- return result
+ return deps.store.getIndex(agentRef)
}
const context = head.context(store, { runChatGetIndex })
assert.equal(await context.getIndex('synthetic'), result)
- assert.deepEqual(calls, ['export'])
+ assert.deepEqual(calls, ['export', 'cleanup:synthetic', 'index:synthetic'])
assert.equal(context.cleanupErrors, 1)
- await context.settle()
- assert.deepEqual(calls, ['export', 'settle'])
+
+ let settled = false
+ const settling = context.settle().then(() => {
+ settled = true
+ })
+ await new Promise(resolve => setImmediate(resolve))
+ // settle waits for the handler's own cleanup and starts none of its own.
+ assert.equal(settled, false)
+ release()
+ await settling
+ assert.deepEqual(calls, ['export', 'cleanup:synthetic', 'index:synthetic', 'cleanup:end'])
})
diff --git a/desktop-app/scripts/bench-chat-open/head.mjs b/desktop-app/scripts/bench-chat-open/head.mjs
index 0e4f81eba..2d4f446ab 100644
--- a/desktop-app/scripts/bench-chat-open/head.mjs
+++ b/desktop-app/scripts/bench-chat-open/head.mjs
@@ -4,21 +4,26 @@ import { AUTHORITY_SCOPE } from './fixtures.mjs'
export default {
ref: 'HEAD',
handlesPendingCleanups: true,
- // HEAD's chat:getIndex handler in ipc.ts delegates to chatGetIndexHandler.ts;
- // the benchmark calls that compiled export.
- verifyHandler({ source }) {
- assert.ok(source.includes('runChatGetIndex'))
- },
bind(ChatStore, root) {
const store = new ChatStore(root)
store.setAuthorityScope(AUTHORITY_SCOPE)
return store
},
+ // HEAD's chat:getIndex handler in ipc.ts delegates to chatGetIndexHandler.ts;
+ // the benchmark calls that compiled export.
context(store, { runChatGetIndex }) {
assert.equal(typeof runChatGetIndex, 'function')
let cleanupErrors = 0
+ const started = []
const deps = {
- store,
+ store: {
+ retryPendingDeleteCleanups: (...args) => {
+ const cleanup = store.retryPendingDeleteCleanups(...args)
+ started.push(cleanup)
+ return cleanup
+ },
+ getIndex: agentRef => store.getIndex(agentRef),
+ },
getChatDeletionFenceAuthority: () => ({ authorityScope: AUTHORITY_SCOPE }),
warnCleanupFailure: () => {
cleanupErrors += 1
@@ -29,9 +34,11 @@ export default {
return cleanupErrors
},
getIndex: agentRef => runChatGetIndex(deps, agentRef),
- // Drain the real cleanup outside timing before removing its fixture. This
- // also keeps a nonblocking handler from leaking work into another run.
- settle: () => store.retryPendingDeleteCleanups(AUTHORITY_SCOPE),
+ // Waits, outside timing, for exactly the cleanups the handler started, so
+ // the fixture state afterwards is what the handler alone left behind.
+ settle: async () => {
+ await Promise.allSettled(started.splice(0))
+ },
}
},
}
diff --git a/desktop-app/scripts/bench-chat-open/sources.mjs b/desktop-app/scripts/bench-chat-open/sources.mjs
index caf203c9d..0727af490 100644
--- a/desktop-app/scripts/bench-chat-open/sources.mjs
+++ b/desktop-app/scripts/bench-chat-open/sources.mjs
@@ -84,7 +84,22 @@ export function handlerLines(ipcSource) {
return { start: start + 1, end: end + 1, source: lines.slice(start, end + 1).join('\n') }
}
-export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir }) {
+/**
+ * HEAD snapshots the live worktree, so its numbers describe the recorded commit
+ * only when the tree is clean. Returns whether the tree is dirty.
+ */
+export function checkWorktreeClean(porcelainStatus, allowDirty) {
+ const dirty = porcelainStatus !== ''
+ if (dirty && !allowDirty) {
+ throw Object.assign(
+ new Error('HEAD has uncommitted changes; commit them or pass --allow-dirty'),
+ { code: 'DIRTY_WORKTREE' }
+ )
+ }
+ return dirty
+}
+
+export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir, allowDirty }) {
const versionDir = await fs.mkdtemp(join(runDir, 'source-'))
const snapshotRoot = join(versionDir, 'snapshot')
const snapshotDesktop = join(snapshotRoot, 'desktop-app')
@@ -92,7 +107,11 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })
const compiledRoot = join(versionDir, 'compiled')
await fs.mkdir(snapshotDesktop, { recursive: true })
const commit = git(repoRoot, ['rev-parse', `${adapter.ref}^{commit}`])
+ let dirty = false
+ let handler = null
+ let ipcSha256 = null
if (adapter.ref === 'HEAD') {
+ dirty = checkWorktreeClean(git(repoRoot, ['status', '--porcelain']), allowDirty)
// Include the live, possibly uncommitted handler; freeze it before compiling.
await fs.access(join(desktopRoot, 'src/chatGetIndexHandler.ts'))
await fs.cp(join(desktopRoot, 'src'), sourceRoot, { recursive: true })
@@ -121,16 +140,18 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })
})
}
await fs.symlink(join(desktopRoot, 'node_modules'), join(snapshotDesktop, 'node_modules'), 'dir')
- const ipcSource = await fs.readFile(join(sourceRoot, 'ipc.ts'), 'utf8')
- // Verify against git show as well as the archived bytes before trusting an adapter.
- if (
- adapter.ref !== 'HEAD' &&
- ipcSource !== `${git(repoRoot, ['show', `${commit}:desktop-app/src/ipc.ts`])}\n`
- ) {
- throw new Error('Archived ipc.ts does not match git show')
+ // A historical handler is inline in ipc.ts, so its pinned lines are checked
+ // against git show. HEAD's handler is the compiled runChatGetIndex export,
+ // which the HEAD adapter asserts it receives.
+ if (adapter.ref !== 'HEAD') {
+ const ipcSource = await fs.readFile(join(sourceRoot, 'ipc.ts'), 'utf8')
+ if (ipcSource !== `${git(repoRoot, ['show', `${commit}:desktop-app/src/ipc.ts`])}\n`) {
+ throw new Error('Archived ipc.ts does not match git show')
+ }
+ handler = handlerLines(ipcSource)
+ adapter.verifyHandler(handler)
+ ipcSha256 = createHash('sha256').update(ipcSource).digest('hex')
}
- const handler = handlerLines(ipcSource)
- adapter.verifyHandler(handler)
await setSourcePermissions(snapshotRoot, true)
const entries = ['chatStore.ts', ...(adapter.ref === 'HEAD' ? ['chatGetIndexHandler.ts'] : [])]
const config = join(versionDir, 'tsconfig.bench.json')
@@ -185,15 +206,15 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })
return {
ChatStore,
context: store => adapter.context(store, handlerModule),
- snapshotRoot,
metadata: {
version: adapter.ref,
commit,
+ dirty,
sourceMode: adapter.ref === 'HEAD' ? 'live-worktree-snapshot' : 'read-only-git-archive',
compilerVersion: ts.version,
sources,
- ipcSha256: createHash('sha256').update(ipcSource).digest('hex'),
- handlerLines: { start: handler.start, end: handler.end },
+ ipcSha256,
+ handlerLines: handler && { start: handler.start, end: handler.end },
runtimeImports: runtime.imports,
electronIsolation:
'Only the compiled ChatStore/handler runtime dependency graph is loaded; every dependency is audited as a Node builtin or a relative module. Electron bindings and ipc.ts are never loaded.',