Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): register the tool-call boundary case and cross-check the registry
Alfredo Lopezd18583af7950
diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjsindex aaa2e930d..9b1144ea8 100644--- a/scripts/e2e/run-codex-approved-tools.mjs+++ b/scripts/e2e/run-codex-approved-tools.mjs@@ -18,9 +18,14 @@ const playwright = path.join(repo, 'tests/e2e/playwright')const specFile = 'codex-subscription-approved-tools.spec.ts'export const toolCallLimitTitle ='tool call limit: Desktop shows Too Many Tool Calls without retry or connector call'+export const toolCallLimitBoundaryTitle =+ 'tool call limit boundary: Desktop completes a turn of exactly 256 tool calls'-// The tool-call limit case exists only in deterministic mode: only the-// deterministic upstream can emit more function calls than the contract allows.+// The tool-call limit cases exist only in deterministic mode: only the+// deterministic upstream can emit more function calls than the contract allows,+// or exactly that many. validateReport compares this registry against the report+// by count as well as by title, so a case the spec declares and this list omits+// fails the entire lane without ever naming it.export function expectedTitles(mode) {if (!['deterministic', 'real'].includes(mode))throw new Error('Select deterministic or real upstream mode explicitly')@@ -31,7 +36,7 @@ export function expectedTitles(mode) {...[83, 150, 250].map(size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`),- ...(mode === 'deterministic' ? [toolCallLimitTitle] : []),+ ...(mode === 'deterministic' ? [toolCallLimitTitle, toolCallLimitBoundaryTitle] : []),]}diff --git a/scripts/e2e/run-codex-approved-tools.test.mjs b/scripts/e2e/run-codex-approved-tools.test.mjsindex ae32877f8..bd01fe82f 100644--- a/scripts/e2e/run-codex-approved-tools.test.mjs+++ b/scripts/e2e/run-codex-approved-tools.test.mjs@@ -3,8 +3,14 @@ import fs from 'node:fs'import os from 'node:os'import path from 'node:path'import { test } from 'node:test'+import { fileURLToPath } from 'node:url'import { openOwnedFile, readOwnedDescriptor } from './prepare-codex-approved-tools.mjs'-import { expectedTitles, toolCallLimitTitle, validateReport } from './run-codex-approved-tools.mjs'+import {+ expectedTitles,+ toolCallLimitBoundaryTitle,+ toolCallLimitTitle,+ validateReport,+} from './run-codex-approved-tools.mjs'// Synthetic reporter records exercise false-green rejection, not product data.function greenReport(mode = 'deterministic') {@@ -41,10 +47,12 @@ for (const mode of ['deterministic', 'real']) {})}test('the tool-call limit case is mandatory only in deterministic mode', () => {- assert.equal(expectedTitles('deterministic').length, 7)+ assert.equal(expectedTitles('deterministic').length, 8)assert.equal(expectedTitles('real').length, 6)- assert.ok(expectedTitles('deterministic').includes(toolCallLimitTitle))- assert.ok(!expectedTitles('real').includes(toolCallLimitTitle))+ for (const title of [toolCallLimitTitle, toolCallLimitBoundaryTitle]) {+ assert.ok(expectedTitles('deterministic').includes(title))+ assert.ok(!expectedTitles('real').includes(title))+ }assert.throws(() => expectedTitles('mock'),/Select deterministic or real upstream mode explicitly/@@ -125,3 +133,40 @@ test('report verification reads only the reserved reporter inode, never a replacfs.rmSync(root, { recursive: true, force: true })}})++// A registry that is only ever compared against itself cannot notice a case the+// spec added and it does not list, and validateReport's count check turns that+// omission into an unnamed lane failure. Read the two real sources instead.+test('every deterministic spec case is registered, and every registered case exists', () => {+ const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')+ const spec = fs.readFileSync(+ path.join(repo, 'tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts'),+ 'utf8'+ )+ const helpers = fs.readFileSync(+ path.join(repo, 'tests/e2e/playwright/helpers/approved-tools-scenarios.ts'),+ 'utf8'+ )+ const sizesSource = helpers.match(/export const catalogSizes = \[([^\]]*)\]/)+ assert.ok(sizesSource, 'catalogSizes is no longer a literal array; update this cross-check')+ const sizes = sizesSource[1].split(',').map(entry => Number(entry.trim()))+ assert.ok(sizes.length > 0 && sizes.every(Number.isInteger), 'catalogSizes must be integers')++ const declared = []+ for (const match of spec.matchAll(/^[ \t]*test\(\s*(['"`])([\s\S]*?)\1\s*,/gm)) {+ const title = match[2]+ const placeholders = [...title.matchAll(/\$\{([^}]*)\}/g)].map(entry => entry[1].trim())+ if (placeholders.length === 0) {+ declared.push(title)+ continue+ }+ assert.deepEqual(+ placeholders,+ ['scenario.catalogSize'],+ `unsupported interpolation in a spec title: ${title}`+ )+ declared.push(...sizes.map(size => title.replace('${scenario.catalogSize}', String(size))))+ }+ assert.equal(declared.length, new Set(declared).size, 'the spec declares a duplicate title')+ assert.deepEqual([...declared].sort(), [...expectedTitles('deterministic')].sort())+})