Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(desktop): harden the chat-open benchmark
Alfredo Lopez4052914836d0
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex c6e151d98..9397d9bc0 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -180,6 +180,11 @@ jobs:if: matrix.service == 'desktop-app'run: node --test scripts/tests/subscription-private-ui-input.test.mjs+ # Hermetic node:test suite of the chat-open benchmark harness (#1091).+ - name: Test the chat-open benchmark harness+ if: matrix.service == 'desktop-app'+ run: node scripts/tests/run-node-test-files.mjs desktop-app/scripts/bench-chat-open/benchmark.test.mjs+# Hermetic subscription-image and Control API memory-calibration node:test# suites. Their only installed dependency is mcp-host's @napi-rs/canvas,# which this leg installs for the Desktop fixture checks above.diff --git a/desktop-app/package.json b/desktop-app/package.jsonindex ed0582bac..d5b86f643 100644--- a/desktop-app/package.json+++ b/desktop-app/package.json@@ -14,6 +14,7 @@"icons:generate": "node ./scripts/generate-adaptive-icons.mjs","test": "vitest run","test:e2e": "vitest run --config test/e2e/vitest.config.ts",+ "test:bench-harness": "node ../scripts/tests/run-node-test-files.mjs scripts/bench-chat-open/benchmark.test.mjs","test:e2e:playwright": "npm run build && playwright test --config test/e2e-playwright/playwright.config.ts","test:native:sandbox-find": "npm run build:main --silent && node ./scripts/run-native-sandbox-find.mjs","test:native:sandbox-shortcut-focus": "npm run build:main --silent && node ./scripts/run-native-sandbox-shortcut-focus.mjs",diff --git a/desktop-app/scripts/bench-chat-open.mjs b/desktop-app/scripts/bench-chat-open.mjsindex 660c86e3a..98bcc7ce1 100644--- a/desktop-app/scripts/bench-chat-open.mjs+++ b/desktop-app/scripts/bench-chat-open.mjs@@ -3,8 +3,10 @@* ChatStore chat-open benchmark (issue #1091, plan sections 7.1 and 11).** Run from desktop-app with Node 24 and installed HEAD dependencies:- * node scripts/bench-chat-open.mjs --quick --versions v0.10.0,v0.11.0 --out /tmp/chat-open.json- * node scripts/bench-chat-open.mjs --out /tmp/chat-open-full.json+ * node scripts/bench-chat-open.mjs --quick --versions v0.10.0,v0.11.0 \+ * --out <primary-checkout>/.local-notes/issue1091-evidence/chat-open-quick.json+ * node scripts/bench-chat-open.mjs \+ * --out <primary-checkout>/.local-notes/issue1091-evidence/chat-open-full.json** --versions selects any comma-separated subset of v0.10.0,v0.11.0,HEAD.* --quick uses 5 agents x 20 chats; both pending-cleanup values and the permanent@@ -14,8 +16,9 @@** Historical sources are extracted using git archive -> tar, made read-only and* compiled with HEAD's node_modules/.bin/tsc into os.tmpdir(). HEAD snapshots the- * live worktree (including uncommitted changes); its source hashes and- * commit are recorded. No checkout, dependency installation or Electron launch.+ * live worktree; its source hashes and commit are recorded. A HEAD with+ * uncommitted changes is refused unless --allow-dirty is passed, and the JSON+ * then records dirty: true. No checkout, dependency installation or Electron launch.* The emitted runtime graph is audited before import. We bypass only Electron's* binding module: a fresh ChatStore + setAuthorityScope reproduces the completed* bind; HEAD's actual compiled runChatGetIndex implements the measured handler.@@ -38,14 +41,15 @@* Warm stage-p95 sums and paired warm-open p95 are both reported; this Node* benchmark does not measure the renderer chat-open time.*- * --out defaults to a unique JSON path under os.tmpdir(). The selected output is- * exclusively created; an existing file is an error. JSON is also printed to- * stdout. Sources/data are removed even on failure; the JSON remains for review.+ * --out is required: results are evidence and must not land in a directory the+ * OS clears. The output is exclusively created; an existing file is an error.+ * JSON is also printed to stdout. Sources/data are removed even on failure; the+ * JSON remains for review.*/import assert from 'node:assert/strict'import { promises as fs } from 'node:fs'import { arch, cpus, loadavg, platform, tmpdir } from 'node:os'-import { basename, dirname, join, resolve } from 'node:path'+import { dirname, join, resolve } from 'node:path'import { performance } from 'node:perf_hooks'import { fileURLToPath, pathToFileURL } from 'node:url'import {@@ -71,10 +75,17 @@ const ADAPTER_FILES = new Map([])export function parseOptions(args) {- const options = { versions: [...ADAPTER_FILES.keys()], quick: false, out: undefined, help: false }+ const options = {+ versions: [...ADAPTER_FILES.keys()],+ quick: false,+ allowDirty: false,+ out: undefined,+ help: false,+ }for (let index = 0; index < args.length; index += 1) {const arg = args[index]if (arg === '--quick') options.quick = true+ else if (arg === '--allow-dirty') options.allowDirty = trueelse if (arg === '--help') options.help = trueelse if (arg === '--versions' || arg === '--out') {const value = args[++index]@@ -92,6 +103,9 @@ export function parseOptions(args) {}} else throw new Error(`Unknown option: ${arg}`)}+ if (!options.help && !options.out) {+ throw Object.assign(new Error('--out is required'), { code: 'OUT_REQUIRED' })+ }return options}@@ -187,13 +201,13 @@ async function runScenario(runDir, adapter, compiled, dimensions) {first.ms + active.ms + fullWarm.ms,]Object.keys(stageSamples).forEach((stage, index) => stageSamples[stage].push(values[index]))- if (adapter.ref === 'HEAD') await context.settle()+ if (context.settle) await context.settle()const state = await inspectFixture(fixture, dimensions.chatsPerAgent)const pendingTotal = dimensions.agents * dimensions.pendingCleanupsPerAgentif (!adapter.handlesPendingCleanups || dimensions.cleanupFilesystem === 'EACCES') {assert.equal(state.pending, pendingTotal)assert.equal(state.retainedArtifactDirectories, pendingTotal)- } else if (adapter.ref === 'v0.11.0') {+ } else {assert.equal(state.pending, 0)assert.equal(state.retainedArtifactDirectories, 0)}@@ -269,7 +283,7 @@ export async function main(args = process.argv.slice(2)) {const options = parseOptions(args)if (options.help) {process.stdout.write(- 'Usage: node scripts/bench-chat-open.mjs [--quick] [--versions v0.10.0,v0.11.0,HEAD] [--out path]\n'+ 'Usage: node scripts/bench-chat-open.mjs --out path [--quick] [--allow-dirty] [--versions v0.10.0,v0.11.0,HEAD]\n')return}@@ -280,13 +294,19 @@ export async function main(args = process.argv.slice(2)) {const desktopRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')const repoRoot = git(desktopRoot, ['rev-parse', '--show-toplevel'])const runDir = await fs.mkdtemp(join(tmpdir(), 'evenfire-chat-open-'))- const out = options.out ?? join(tmpdir(), `${basename(runDir)}.json`)+ const { out } = optionsconst started = performance.now()try {const prepared = []for (const ref of options.versions) {const { default: adapter } = await import(`./bench-chat-open/${ADAPTER_FILES.get(ref)}`)- const compiled = await compileVersion({ adapter, repoRoot, desktopRoot, runDir })+ const compiled = await compileVersion({+ adapter,+ repoRoot,+ desktopRoot,+ runDir,+ allowDirty: options.allowDirty,+ })prepared.push({ adapter, compiled })}const results = []diff --git a/desktop-app/scripts/bench-chat-open/benchmark.test.mjs b/desktop-app/scripts/bench-chat-open/benchmark.test.mjsindex 1631e63a2..137cee2ea 100644--- a/desktop-app/scripts/bench-chat-open/benchmark.test.mjs+++ b/desktop-app/scripts/bench-chat-open/benchmark.test.mjs@@ -8,7 +8,7 @@ import { REPETITIONS, distribution, matrix, parseOptions } from '../bench-chat-oimport { captureFilesystemFailures } from './fixtures.mjs'import { AUTHORITY_SCOPE } from './fixtures.mjs'import head from './head.mjs'-import { auditRuntime } from './sources.mjs'+import { auditRuntime, checkWorktreeClean } from './sources.mjs'import v011 from './v0.11.0.mjs'const ts = createRequire(import.meta.url)('typescript')@@ -24,13 +24,28 @@ test('quick retains both cleanup populations and the failing-filesystem scenario})test('version restriction preserves historical refs and rejects duplicates or command-like refs', () => {- assert.deepEqual(parseOptions(['--versions', 'v0.11.0,v0.10.0', '--quick']).versions, [- 'v0.11.0',- 'v0.10.0',- ])- assert.throws(() => parseOptions(['--versions', 'v0.10.0,v0.10.0']))+ assert.deepEqual(+ parseOptions(['--versions', 'v0.11.0,v0.10.0', '--quick', '--out', 'r.json']).versions,+ ['v0.11.0', 'v0.10.0']+ )+ assert.throws(() => parseOptions(['--versions', 'v0.10.0,v0.10.0', '--out', 'r.json']))assert.throws(() => parseOptions(['--versions', '--help']))- assert.throws(() => parseOptions(['--versions', 'origin/dev']))+ assert.throws(() => parseOptions(['--versions', 'origin/dev', '--out', 'r.json']))+})++test('--out is required and --allow-dirty is opt-in', () => {+ assert.throws(() => parseOptions(['--quick']), /--out is required/)+ const options = parseOptions(['--quick', '--out', 'r.json'])+ assert.equal(options.out.endsWith('r.json'), true)+ assert.equal(options.allowDirty, false)+ assert.equal(parseOptions(['--allow-dirty', '--out', 'r.json']).allowDirty, true)+ assert.equal(parseOptions(['--help']).help, true)+})++test('a dirty HEAD tree is refused unless allowed, and then recorded as dirty', () => {+ assert.equal(checkWorktreeClean('', false), false)+ assert.throws(() => checkWorktreeClean(' M desktop-app/src/ipc.ts', false), /--allow-dirty/)+ assert.equal(checkWorktreeClean(' M desktop-app/src/ipc.ts', true), true)})test('nearest-rank p95 uses the 29th of 30 observations and rejects missing/nonfinite samples', () => {@@ -103,29 +118,49 @@ test('v0.11.0 adapter reports a cleanup error and still reads the catalog in hanassert.equal(context.cleanupErrors, 1)})-test('HEAD delegates to the supplied compiled export and drains cleanup only after measurement', async () => {+test('HEAD delegates to the compiled export and settle waits for the cleanups it started', async () => {const calls = []+ let release+ const gate = new Promise(resolve => {+ release = resolve+ })+ const result = { chats: [1] }const store = {- async retryPendingDeleteCleanups(scope) {+ async retryPendingDeleteCleanups(scope, agentRef) {assert.deepEqual(scope, AUTHORITY_SCOPE)- calls.push('settle')+ calls.push(`cleanup:${agentRef}`)+ await gate+ calls.push('cleanup:end')+ },+ async getIndex(agentRef) {+ calls.push(`index:${agentRef}`)+ return result},}- const result = { chats: [1] }- // This distinct behavior witnesses delegation to the compiled export instead- // of an adapter-owned copy of a historical store sequence.+ // A distinct export witnesses delegation to the compiled handler instead of+ // an adapter-owned copy of the store sequence.const runChatGetIndex = async (deps, agentRef) => {calls.push('export')- assert.equal(deps.store, store)- assert.equal(agentRef, 'synthetic')- assert.deepEqual(deps.getChatDeletionFenceAuthority(), { authorityScope: AUTHORITY_SCOPE })+ const { authorityScope } = deps.getChatDeletionFenceAuthority()+ void deps.store+ .retryPendingDeleteCleanups(authorityScope, agentRef)+ .catch(deps.warnCleanupFailure)deps.warnCleanupFailure(new Error('Synthetic warning'))- return result+ return deps.store.getIndex(agentRef)}const context = head.context(store, { runChatGetIndex })assert.equal(await context.getIndex('synthetic'), result)- assert.deepEqual(calls, ['export'])+ assert.deepEqual(calls, ['export', 'cleanup:synthetic', 'index:synthetic'])assert.equal(context.cleanupErrors, 1)- await context.settle()- assert.deepEqual(calls, ['export', 'settle'])++ let settled = false+ const settling = context.settle().then(() => {+ settled = true+ })+ await new Promise(resolve => setImmediate(resolve))+ // settle waits for the handler's own cleanup and starts none of its own.+ assert.equal(settled, false)+ release()+ await settling+ assert.deepEqual(calls, ['export', 'cleanup:synthetic', 'index:synthetic', 'cleanup:end'])})diff --git a/desktop-app/scripts/bench-chat-open/head.mjs b/desktop-app/scripts/bench-chat-open/head.mjsindex 0e4f81eba..2d4f446ab 100644--- a/desktop-app/scripts/bench-chat-open/head.mjs+++ b/desktop-app/scripts/bench-chat-open/head.mjs@@ -4,21 +4,26 @@ import { AUTHORITY_SCOPE } from './fixtures.mjs'export default {ref: 'HEAD',handlesPendingCleanups: true,- // HEAD's chat:getIndex handler in ipc.ts delegates to chatGetIndexHandler.ts;- // the benchmark calls that compiled export.- verifyHandler({ source }) {- assert.ok(source.includes('runChatGetIndex'))- },bind(ChatStore, root) {const store = new ChatStore(root)store.setAuthorityScope(AUTHORITY_SCOPE)return store},+ // HEAD's chat:getIndex handler in ipc.ts delegates to chatGetIndexHandler.ts;+ // the benchmark calls that compiled export.context(store, { runChatGetIndex }) {assert.equal(typeof runChatGetIndex, 'function')let cleanupErrors = 0+ const started = []const deps = {- store,+ store: {+ retryPendingDeleteCleanups: (...args) => {+ const cleanup = store.retryPendingDeleteCleanups(...args)+ started.push(cleanup)+ return cleanup+ },+ getIndex: agentRef => store.getIndex(agentRef),+ },getChatDeletionFenceAuthority: () => ({ authorityScope: AUTHORITY_SCOPE }),warnCleanupFailure: () => {cleanupErrors += 1@@ -29,9 +34,11 @@ export default {return cleanupErrors},getIndex: agentRef => runChatGetIndex(deps, agentRef),- // Drain the real cleanup outside timing before removing its fixture. This- // also keeps a nonblocking handler from leaking work into another run.- settle: () => store.retryPendingDeleteCleanups(AUTHORITY_SCOPE),+ // Waits, outside timing, for exactly the cleanups the handler started, so+ // the fixture state afterwards is what the handler alone left behind.+ settle: async () => {+ await Promise.allSettled(started.splice(0))+ },}},}diff --git a/desktop-app/scripts/bench-chat-open/sources.mjs b/desktop-app/scripts/bench-chat-open/sources.mjsindex caf203c9d..0727af490 100644--- a/desktop-app/scripts/bench-chat-open/sources.mjs+++ b/desktop-app/scripts/bench-chat-open/sources.mjs@@ -84,7 +84,22 @@ export function handlerLines(ipcSource) {return { start: start + 1, end: end + 1, source: lines.slice(start, end + 1).join('\n') }}-export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir }) {+/**+ * HEAD snapshots the live worktree, so its numbers describe the recorded commit+ * only when the tree is clean. Returns whether the tree is dirty.+ */+export function checkWorktreeClean(porcelainStatus, allowDirty) {+ const dirty = porcelainStatus !== ''+ if (dirty && !allowDirty) {+ throw Object.assign(+ new Error('HEAD has uncommitted changes; commit them or pass --allow-dirty'),+ { code: 'DIRTY_WORKTREE' }+ )+ }+ return dirty+}++export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir, allowDirty }) {const versionDir = await fs.mkdtemp(join(runDir, 'source-'))const snapshotRoot = join(versionDir, 'snapshot')const snapshotDesktop = join(snapshotRoot, 'desktop-app')@@ -92,7 +107,11 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })const compiledRoot = join(versionDir, 'compiled')await fs.mkdir(snapshotDesktop, { recursive: true })const commit = git(repoRoot, ['rev-parse', `${adapter.ref}^{commit}`])+ let dirty = false+ let handler = null+ let ipcSha256 = nullif (adapter.ref === 'HEAD') {+ dirty = checkWorktreeClean(git(repoRoot, ['status', '--porcelain']), allowDirty)// Include the live, possibly uncommitted handler; freeze it before compiling.await fs.access(join(desktopRoot, 'src/chatGetIndexHandler.ts'))await fs.cp(join(desktopRoot, 'src'), sourceRoot, { recursive: true })@@ -121,16 +140,18 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })})}await fs.symlink(join(desktopRoot, 'node_modules'), join(snapshotDesktop, 'node_modules'), 'dir')- const ipcSource = await fs.readFile(join(sourceRoot, 'ipc.ts'), 'utf8')- // Verify against git show as well as the archived bytes before trusting an adapter.- if (- adapter.ref !== 'HEAD' &&- ipcSource !== `${git(repoRoot, ['show', `${commit}:desktop-app/src/ipc.ts`])}\n`- ) {- throw new Error('Archived ipc.ts does not match git show')+ // A historical handler is inline in ipc.ts, so its pinned lines are checked+ // against git show. HEAD's handler is the compiled runChatGetIndex export,+ // which the HEAD adapter asserts it receives.+ if (adapter.ref !== 'HEAD') {+ const ipcSource = await fs.readFile(join(sourceRoot, 'ipc.ts'), 'utf8')+ if (ipcSource !== `${git(repoRoot, ['show', `${commit}:desktop-app/src/ipc.ts`])}\n`) {+ throw new Error('Archived ipc.ts does not match git show')+ }+ handler = handlerLines(ipcSource)+ adapter.verifyHandler(handler)+ ipcSha256 = createHash('sha256').update(ipcSource).digest('hex')}- const handler = handlerLines(ipcSource)- adapter.verifyHandler(handler)await setSourcePermissions(snapshotRoot, true)const entries = ['chatStore.ts', ...(adapter.ref === 'HEAD' ? ['chatGetIndexHandler.ts'] : [])]const config = join(versionDir, 'tsconfig.bench.json')@@ -185,15 +206,15 @@ export async function compileVersion({ adapter, repoRoot, desktopRoot, runDir })return {ChatStore,context: store => adapter.context(store, handlerModule),- snapshotRoot,metadata: {version: adapter.ref,commit,+ dirty,sourceMode: adapter.ref === 'HEAD' ? 'live-worktree-snapshot' : 'read-only-git-archive',compilerVersion: ts.version,sources,- ipcSha256: createHash('sha256').update(ipcSource).digest('hex'),- handlerLines: { start: handler.start, end: handler.end },+ ipcSha256,+ handlerLines: handler && { start: handler.start, end: handler.end },runtimeImports: runtime.imports,electronIsolation:'Only the compiled ChatStore/handler runtime dependency graph is loaded; every dependency is audited as a Node builtin or a relative module. Electron bindings and ipc.ts are never loaded.',