Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): register the tool-call boundary case and cross-check the registry

Alfredo Lopezd18583af7950
diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjs
index aaa2e930d..9b1144ea8 100644
--- a/scripts/e2e/run-codex-approved-tools.mjs
+++ b/scripts/e2e/run-codex-approved-tools.mjs
@@ -18,9 +18,14 @@ const playwright = path.join(repo, 'tests/e2e/playwright')
const specFile = 'codex-subscription-approved-tools.spec.ts'
export const toolCallLimitTitle =
'tool call limit: Desktop shows Too Many Tool Calls without retry or connector call'
+export const toolCallLimitBoundaryTitle =
+ 'tool call limit boundary: Desktop completes a turn of exactly 256 tool calls'
-// The tool-call limit case exists only in deterministic mode: only the
-// deterministic upstream can emit more function calls than the contract allows.
+// The tool-call limit cases exist only in deterministic mode: only the
+// deterministic upstream can emit more function calls than the contract allows,
+// or exactly that many. validateReport compares this registry against the report
+// by count as well as by title, so a case the spec declares and this list omits
+// fails the entire lane without ever naming it.
export function expectedTitles(mode) {
if (!['deterministic', 'real'].includes(mode))
throw new Error('Select deterministic or real upstream mode explicitly')
@@ -31,7 +36,7 @@ export function expectedTitles(mode) {
...[83, 150, 250].map(
size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`
),
- ...(mode === 'deterministic' ? [toolCallLimitTitle] : []),
+ ...(mode === 'deterministic' ? [toolCallLimitTitle, toolCallLimitBoundaryTitle] : []),
]
}
diff --git a/scripts/e2e/run-codex-approved-tools.test.mjs b/scripts/e2e/run-codex-approved-tools.test.mjs
index ae32877f8..bd01fe82f 100644
--- a/scripts/e2e/run-codex-approved-tools.test.mjs
+++ b/scripts/e2e/run-codex-approved-tools.test.mjs
@@ -3,8 +3,14 @@ import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
import { openOwnedFile, readOwnedDescriptor } from './prepare-codex-approved-tools.mjs'
-import { expectedTitles, toolCallLimitTitle, validateReport } from './run-codex-approved-tools.mjs'
+import {
+ expectedTitles,
+ toolCallLimitBoundaryTitle,
+ toolCallLimitTitle,
+ validateReport,
+} from './run-codex-approved-tools.mjs'
// Synthetic reporter records exercise false-green rejection, not product data.
function greenReport(mode = 'deterministic') {
@@ -41,10 +47,12 @@ for (const mode of ['deterministic', 'real']) {
})
}
test('the tool-call limit case is mandatory only in deterministic mode', () => {
- assert.equal(expectedTitles('deterministic').length, 7)
+ assert.equal(expectedTitles('deterministic').length, 8)
assert.equal(expectedTitles('real').length, 6)
- assert.ok(expectedTitles('deterministic').includes(toolCallLimitTitle))
- assert.ok(!expectedTitles('real').includes(toolCallLimitTitle))
+ for (const title of [toolCallLimitTitle, toolCallLimitBoundaryTitle]) {
+ assert.ok(expectedTitles('deterministic').includes(title))
+ assert.ok(!expectedTitles('real').includes(title))
+ }
assert.throws(
() => expectedTitles('mock'),
/Select deterministic or real upstream mode explicitly/
@@ -125,3 +133,40 @@ test('report verification reads only the reserved reporter inode, never a replac
fs.rmSync(root, { recursive: true, force: true })
}
})
+
+// A registry that is only ever compared against itself cannot notice a case the
+// spec added and it does not list, and validateReport's count check turns that
+// omission into an unnamed lane failure. Read the two real sources instead.
+test('every deterministic spec case is registered, and every registered case exists', () => {
+ const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
+ const spec = fs.readFileSync(
+ path.join(repo, 'tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts'),
+ 'utf8'
+ )
+ const helpers = fs.readFileSync(
+ path.join(repo, 'tests/e2e/playwright/helpers/approved-tools-scenarios.ts'),
+ 'utf8'
+ )
+ const sizesSource = helpers.match(/export const catalogSizes = \[([^\]]*)\]/)
+ assert.ok(sizesSource, 'catalogSizes is no longer a literal array; update this cross-check')
+ const sizes = sizesSource[1].split(',').map(entry => Number(entry.trim()))
+ assert.ok(sizes.length > 0 && sizes.every(Number.isInteger), 'catalogSizes must be integers')
+
+ const declared = []
+ for (const match of spec.matchAll(/^[ \t]*test\(\s*(['"`])([\s\S]*?)\1\s*,/gm)) {
+ const title = match[2]
+ const placeholders = [...title.matchAll(/\$\{([^}]*)\}/g)].map(entry => entry[1].trim())
+ if (placeholders.length === 0) {
+ declared.push(title)
+ continue
+ }
+ assert.deepEqual(
+ placeholders,
+ ['scenario.catalogSize'],
+ `unsupported interpolation in a spec title: ${title}`
+ )
+ declared.push(...sizes.map(size => title.replace('${scenario.catalogSize}', String(size))))
+ }
+ assert.equal(declared.length, new Set(declared).size, 'the spec declares a duplicate title')
+ assert.deepEqual([...declared].sort(), [...expectedTitles('deterministic')].sort())
+})