Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): cover the Codex tool-call limit in the Desktop approved-tools journey
Alfredo Lopezed442902be69
diff --git a/codex-llm-proxy/test/approvedToolsUpstream.test.ts b/codex-llm-proxy/test/approvedToolsUpstream.test.tsindex 28a53cef4..233813bbe 100644--- a/codex-llm-proxy/test/approvedToolsUpstream.test.ts+++ b/codex-llm-proxy/test/approvedToolsUpstream.test.ts@@ -1,5 +1,9 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import { randomUUID } from 'node:crypto'+import { hashCodexCompletionRequestV1 } from '@clerum/llm-provider-attempt-contract'+import { streamCodexCompletion } from '../src/codexTransport.js'+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'+import { CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'import { createApprovedToolsUpstream } from './approvedToolsUpstream'const URL = 'https://chatgpt.com/backend-api/codex/responses'@@ -370,3 +374,160 @@ describe('approved-tools isolated upstream boundary', () => {expect(JSON.stringify(simulator.evidence())).not.toContain('workitem_read_001')})})++describe('approved-tools tool-call limit probe', () => {+ const probe = { role: 'user', content: 'Run the tool call limit probe now.' }++ function events(body: string): Entry[] {+ return body+ .split('\n\n')+ .filter(Boolean)+ .map(frame => JSON.parse(frame.slice('data: '.length)) as Entry)+ }++ it('answers a probe turn with 65 distinct search calls in one completion', async () => {+ const simulator = createApprovedToolsUpstream()+ const { response, body } = await request(simulator, [probe])+ expect(response.status).toBe(200)+ const rows = events(body)+ const calls = rows.slice(0, -1).map(row => row.item as Entry)+ expect(rows[rows.length - 1]).toEqual({ type: 'response.completed' })+ expect(calls).toHaveLength(65)+ expect(new Set(calls.map(call => call.call_id)).size).toBe(65)+ for (const call of calls) {+ expect(call).toMatchObject({+ type: 'function_call',+ name: 'clerum__tool_search',+ arguments: JSON.stringify({ query: 'verification receipt', limit: 5 }),+ })+ }+ expect(simulator.evidence()).toMatchObject({+ limitProbe: { turns: 1, completions: 1, unexpectedRetries: 0 },+ completions: 1,+ searchCalls: 0,+ rejected: 0,+ })+ expect(simulator.evidence().requests).toMatchObject([{ stage: 'limit_probe' }])+ })++ it('rejects a second completion of the same probe turn as unexpected_retry', async () => {+ const simulator = createApprovedToolsUpstream()+ const first = await request(simulator, [probe])+ expect(first.response.status).toBe(200)++ const retry = await request(simulator, [probe])+ const call = events(first.body)[0]!.item as Entry+ const continuation = await request(simulator, [+ probe,+ call,+ { type: 'function_call_output', call_id: call.call_id, output: '{}' },+ ])++ expect(retry.response.status).toBe(422)+ expect(continuation.response.status).toBe(422)+ expect(simulator.evidence()).toMatchObject({+ limitProbe: { turns: 1, completions: 3, unexpectedRetries: 2 },+ rejected: 2,+ })+ })++ it('answers a new probe turn later in the same conversation', async () => {+ const simulator = createApprovedToolsUpstream()+ const first = await request(simulator, [probe])+ const assistant = { role: 'assistant', content: 'The agent stopped.' }+ const second = await request(simulator, [probe, assistant, probe])+ expect([first.response.status, second.response.status]).toEqual([200, 200])+ expect(simulator.evidence().limitProbe).toEqual({+ turns: 2,+ completions: 2,+ unexpectedRetries: 0,+ })+ })++ it('makes the real transport fail the probe with tool_call_limit_exceeded', async () => {+ const simulator = createApprovedToolsUpstream()+ const probeRequest = {+ schemaVersion: 'codex-completion-request.v1' as const,+ requestId: 'req-limit-probe',+ idempotencyKey: 'idem-limit-probe',+ provider: 'codex-subscription' as const,+ model: 'gpt-5.3-codex',+ messages: [{ role: 'user' as const, content: probe.content }],+ tools: bridges.map(name => ({+ name,+ description: `${name} bridge`,+ parameters: { type: 'object' },+ })),+ }+ const requestHash = hashCodexCompletionRequestV1(probeRequest)+ const emitted: Array<{ type: string }> = []+ const finalize = vi.fn(async () => ({+ providerAttemptId: 'att-limit-probe',+ outcome: 'error' as const,+ duplicate: false,+ }))+ const pending = streamCodexCompletion({+ executionTicket: 'ticket-limit-probe',+ requestHash,+ request: probeRequest,+ ticket: {+ jti: 'jti-limit-probe',+ hostRef: 'approved-tools-host',+ model: probeRequest.model,+ requestHash,+ providerAttemptId: 'att-limit-probe',+ },+ redeem: vi.fn(+ async (): Promise<RedeemAttemptSuccess> => ({+ accessToken: `hdr.${Buffer.from(+ JSON.stringify({ 'https://api.openai.com/auth': { chatgpt_account_id: 'acct_probe' } })+ ).toString('base64url')}.sig`,+ transport: {+ protocolVersion: 'codex-subscription-transport.v1',+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,+ catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',+ operation: 'completion_stream',+ servedModel: probeRequest.model,+ maxStreamDurationMs: 300_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'b'.repeat(64),+ })+ ),+ finalize,+ fetchFn: simulator.fetchFn,+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],+ onFrame: frame => {+ emitted.push(frame)+ },+ })++ await expect(pending).rejects.toMatchObject({+ name: 'CodexTransportError',+ code: 'tool_call_limit_exceeded',+ details: { limit: 64, observed: 65 },+ })+ // Liveness witness: the upstream served exactly one probe completion.+ expect(simulator.evidence().limitProbe).toEqual({+ turns: 1,+ completions: 1,+ unexpectedRetries: 0,+ })+ expect(emitted.filter(frame => frame.type === 'tool_call')).toEqual([])+ expect(finalize).toHaveBeenCalledTimes(1)+ })++ it('leaves the receipt journey unchanged after a probe turn (ordinary path witness)', async () => {+ const simulator = createApprovedToolsUpstream()+ await request(simulator, [probe])+ await complete(simulator, [{ role: 'user', content: 'verification receipt' }], 'after-probe')+ expect(simulator.evidence()).toMatchObject({+ limitProbe: { turns: 1, completions: 1, unexpectedRetries: 0 },+ searchCalls: 1,+ describeCalls: 1,+ businessCalls: 1,+ finalResponses: 1,+ rejected: 0,+ })+ })+})diff --git a/codex-llm-proxy/test/approvedToolsUpstream.ts b/codex-llm-proxy/test/approvedToolsUpstream.tsindex 26cc7a601..ff6235e6f 100644--- a/codex-llm-proxy/test/approvedToolsUpstream.ts+++ b/codex-llm-proxy/test/approvedToolsUpstream.ts@@ -1,5 +1,5 @@/** External model boundary for the isolated approved-tools E2E image only. */-import { randomUUID } from 'node:crypto'+import { createHash, randomUUID } from 'node:crypto'import { isDeepStrictEqual } from 'node:util'type Row = Record<string, unknown>@@ -8,6 +8,10 @@ const CATALOG = 'https://chatgpt.com/backend-api/codex/models?client_version=1.0const BRIDGES = ['clerum__tool_search', 'clerum__tool_describe', 'clerum__tool_call']// The deterministic external model is scoped to this fixture's read journey.const FIXTURE_QUERY = 'verification receipt'+// A user turn with this text gets one response carrying more tool calls than+// the Codex contract allows (maxToolCalls 64), so the proxy must reject it.+const LIMIT_PROBE = 'tool call limit probe'+const LIMIT_PROBE_CALLS = 65function record(value: unknown): Row {if (!value || typeof value !== 'object' || Array.isArray(value))@@ -49,10 +53,7 @@ type Exchange = { call: Row; args: Row; result: Row }type Decision = { stage: string; args?: Row; result?: Row }function expectedCall(exchange: Exchange, decision: Decision): void {- if (- exchange.call.name !== decision.stage ||- !isDeepStrictEqual(exchange.args, decision.args)- ) {+ if (exchange.call.name !== decision.stage || !isDeepStrictEqual(exchange.args, decision.args)) {throw new Error('unexpected_fixture_call')}}@@ -268,9 +269,11 @@ function workflowDecision(current: Exchange[], prompt: string): Decision {throw new Error('unsupported_workflow_transition')}-function stream(event: Row): Response {+function stream(...events: Row[]): Response {return new Response(- [event, { type: 'response.completed' }].map(row => `data: ${JSON.stringify(row)}\n\n`).join(''),+ [...events, { type: 'response.completed' }]+ .map(row => `data: ${JSON.stringify(row)}\n\n`)+ .join(''),{headers: { 'content-type': 'text/event-stream' },}@@ -287,6 +290,9 @@ export function createApprovedToolsUpstream() {businessCalls: 0,finalResponses: 0,deniedResponses: 0,+ // Probe turns answered with an over-limit batch, every completion served+ // for them, and completions that repeated an already answered probe turn.+ limitProbe: { turns: 0, completions: 0, unexpectedRetries: 0 },// Only bounded measurements, never request or result contents.requests: [] as Array<{definitionCount: number@@ -298,6 +304,9 @@ export function createApprovedToolsUpstream() {stage: string}>,}+ // Digests of the history up to each answered probe user turn. A retry of the+ // same turn resends that history unchanged.+ const answeredProbeTurns = new Set<string>()const fetchFn: typeof fetch = async (input, init) => {try {@@ -325,6 +334,44 @@ export function createApprovedToolsUpstream() {const user = history[lastUser]if (!user || typeof user.content !== 'string' || !user.content.trim())throw new Error('missing_user_request')+ if (user.content.toLowerCase().includes(LIMIT_PROBE)) {+ if (evidence.requests.length >= 512) throw new Error('evidence_capacity_exceeded')+ evidence.limitProbe.completions+++ const turn = createHash('sha256')+ .update(JSON.stringify(history.slice(0, lastUser + 1)))+ .digest('hex')+ if (answeredProbeTurns.has(turn)) {+ evidence.limitProbe.unexpectedRetries+++ throw new Error('unexpected_retry')+ }+ // An over-limit batch never yields tool results, so a continuation+ // means the limit was not enforced.+ if (history.length !== lastUser + 1) throw new Error('unexpected_probe_continuation')+ answeredProbeTurns.add(turn)+ evidence.limitProbe.turns+++ evidence.completions+++ evidence.requests.push({+ definitionCount: tools.length,+ explicitNonStrictCount: tools.filter(tool => tool.strict === false).length,+ definitionBytes: Buffer.byteLength(JSON.stringify(tools)),+ inputBytes: Buffer.byteLength(JSON.stringify(payload.input)),+ connectorDefinitionCount: 0,+ leakedSchema: false,+ stage: 'limit_probe',+ })+ return stream(+ ...Array.from({ length: LIMIT_PROBE_CALLS }, () => ({+ type: 'response.output_item.done',+ item: {+ type: 'function_call',+ id: randomUUID(),+ call_id: randomUUID(),+ name: BRIDGES[0],+ arguments: JSON.stringify({ query: FIXTURE_QUERY, limit: 5 }),+ },+ }))+ )+ }const isWorkflow = /\bworkflow\b/i.test(user.content)if (!isWorkflow && !user.content.toLowerCase().includes(FIXTURE_QUERY))throw new Error('unsupported_fixture_task')diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjsindex b885d18ee..aaa2e930d 100644--- a/scripts/e2e/run-codex-approved-tools.mjs+++ b/scripts/e2e/run-codex-approved-tools.mjs@@ -16,14 +16,24 @@ import {const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')const playwright = path.join(repo, 'tests/e2e/playwright')const specFile = 'codex-subscription-approved-tools.spec.ts'-export const expectedTitles = [- 'unauthenticated agent route guard prevents connector use',- 'authenticated user without agent access cannot select the protected agents',- 'native workflow: visible approval, trigger, status and result artifact',- ...[83, 150, 250].map(- size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`- ),-]+export const toolCallLimitTitle =+ 'tool call limit: Desktop shows Too Many Tool Calls without retry or connector call'++// The tool-call limit case exists only in deterministic mode: only the+// deterministic upstream can emit more function calls than the contract allows.+export function expectedTitles(mode) {+ if (!['deterministic', 'real'].includes(mode))+ throw new Error('Select deterministic or real upstream mode explicitly')+ return [+ 'unauthenticated agent route guard prevents connector use',+ 'authenticated user without agent access cannot select the protected agents',+ 'native workflow: visible approval, trigger, status and result artifact',+ ...[83, 150, 250].map(+ size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`+ ),+ ...(mode === 'deterministic' ? [toolCallLimitTitle] : []),+ ]+}export function runPhaseTimeouts() {return {@@ -34,7 +44,8 @@ export function runPhaseTimeouts() {}}-export function validateReport(report) {+export function validateReport(report, mode) {+ const titles = expectedTitles(mode)const specs = []function collect(suites) {if (!Array.isArray(suites)) throw new Error('Incomplete Playwright suites')@@ -49,12 +60,12 @@ export function validateReport(report) {report.stats?.unexpected !== 0 ||report.stats?.skipped !== 0 ||report.stats?.flaky !== 0 ||- report.stats?.expected !== expectedTitles.length ||- specs.length !== expectedTitles.length+ report.stats?.expected !== titles.length ||+ specs.length !== titles.length) {throw new Error('Incomplete, failed, flaky or skipped approved-tools report')}- for (const title of expectedTitles) {+ for (const title of titles) {const matches = specs.filter(spec =>spec.title === title &&@@ -194,7 +205,7 @@ function main() {env,APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS)- const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)))+ const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)), mode)process.stdout.write(`${JSON.stringify({ lane: 'Playwright', upstream: mode, ...verified, report })}\n`)diff --git a/scripts/e2e/run-codex-approved-tools.test.mjs b/scripts/e2e/run-codex-approved-tools.test.mjsindex bb32b67a7..e90bee05c 100644--- a/scripts/e2e/run-codex-approved-tools.test.mjs+++ b/scripts/e2e/run-codex-approved-tools.test.mjs@@ -4,16 +4,17 @@ import os from 'node:os'import path from 'node:path'import { test } from 'node:test'import { openOwnedFile, readOwnedDescriptor } from './prepare-codex-approved-tools.mjs'-import { expectedTitles, validateReport } from './run-codex-approved-tools.mjs'+import { expectedTitles, toolCallLimitTitle, validateReport } from './run-codex-approved-tools.mjs'// Synthetic reporter records exercise false-green rejection, not product data.-function greenReport() {+function greenReport(mode = 'deterministic') {+ const titles = expectedTitles(mode)return {errors: [],- stats: { expected: expectedTitles.length, unexpected: 0, skipped: 0, flaky: 0 },+ stats: { expected: titles.length, unexpected: 0, skipped: 0, flaky: 0 },suites: [{- specs: expectedTitles.map(title => ({+ specs: titles.map(title => ({title,file: 'desktop/codex-subscription-approved-tools.spec.ts',ok: true,@@ -30,17 +31,32 @@ function greenReport() {],}}-test('accepts exactly all mandatory green cases', () => {- assert.deepEqual(validateReport(greenReport()), {- tests: expectedTitles.length,- skipped: 0,- failed: 0,+for (const mode of ['deterministic', 'real']) {+ test(`accepts exactly all mandatory green cases in ${mode} mode`, () => {+ assert.deepEqual(validateReport(greenReport(mode), mode), {+ tests: expectedTitles(mode).length,+ skipped: 0,+ failed: 0,+ })})+}+test('the tool-call limit case is mandatory only in deterministic mode', () => {+ assert.equal(expectedTitles('deterministic').length, 7)+ assert.equal(expectedTitles('real').length, 6)+ assert.ok(expectedTitles('deterministic').includes(toolCallLimitTitle))+ assert.ok(!expectedTitles('real').includes(toolCallLimitTitle))+ assert.throws(() => expectedTitles('mock'))+})+test('rejects a deterministic report without the tool-call limit case', () => {+ assert.throws(() => validateReport(greenReport('real'), 'deterministic'))+})+test('rejects a real report that carries the deterministic-only case', () => {+ assert.throws(() => validateReport(greenReport('deterministic'), 'real'))})const corruptions = {'missing case': report => report.suites[0].specs.pop(),'duplicate case': report => {- report.suites[0].specs[0].title = expectedTitles[1]+ report.suites[0].specs[0].title = expectedTitles('deterministic')[1]},'different physical file': report => {report.suites[0].specs[0].file = 'other.spec.ts'@@ -71,7 +87,7 @@ for (const [name, corrupt] of Object.entries(corruptions)) {test(`rejects ${name}`, () => {const report = greenReport()corrupt(report)- assert.throws(() => validateReport(report))+ assert.throws(() => validateReport(report, 'deterministic'))})}@@ -81,9 +97,12 @@ test('report verification reads only the reserved reporter inode, never a replactry {fs.renameSync(path.join(root, 'report.json'), path.join(root, 'reserved.json'))fs.writeFileSync(path.join(root, 'report.json'), JSON.stringify(greenReport()), { mode: 0o600 })- assert.throws(() => validateReport(JSON.parse(readOwnedDescriptor(file))))+ assert.throws(() => validateReport(JSON.parse(readOwnedDescriptor(file)), 'deterministic'))fs.writeSync(file.fd, JSON.stringify(greenReport()))- assert.equal(validateReport(JSON.parse(readOwnedDescriptor(file))).tests, expectedTitles.length)+ assert.equal(+ validateReport(JSON.parse(readOwnedDescriptor(file)), 'deterministic').tests,+ expectedTitles('deterministic').length+ )} finally {fs.closeSync(file.fd)fs.rmSync(root, { recursive: true, force: true })diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.tsindex 2432baed7..4a588ff6c 100644--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts@@ -101,6 +101,30 @@ async function newAgentResponses(page: Page) {return responses.and(page.locator(`[data-chat-message-id]${exclusions}`))}+async function openAgentChat(desktop: Page, scenario: Scenario) {+ await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()+ await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))+ await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))+ await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()+ await expect(desktop.getByTestId('nav-chat')).toBeVisible()+ await desktop.getByTestId('nav-chat').click()+ await desktop.getByTestId('nav-new-chat').click()+ await expect(desktop.getByRole('heading', { name: 'New chat with', exact: true })).toBeVisible()+ await desktop.getByRole('button', { name: 'Switch chat agent', exact: true }).click()+ await desktop.getByRole('menuitem', { name: scenario.agentDisplayName, exact: true }).click()+ await expect(+ desktop.getByRole('button', { name: 'Switch chat agent', exact: true })+ ).toContainText(scenario.agentDisplayName)+ await desktop.getByRole('button', { name: 'Model — Select model', exact: true }).click()+ const modelOption = desktop.getByRole('menuitemradio').filter({ hasText: scenario.modelLabel })+ await expect(modelOption).toHaveCount(1)+ await modelOption.click()+ await expect(+ desktop.getByRole('button', { name: `Model — ${scenario.modelLabel}`, exact: true })+ ).toBeVisible()+ await expect(desktop.getByTestId('agent-response')).toHaveCount(0)+}+async function receiptApproval(page: Page, scenario: Scenario, previousCalls: number) {const pending = page.getByTestId('progress-stepper')@@ -219,33 +243,7 @@ for (const scenario of cases) {try {const desktop = await app.firstWindow()await test.step('Sign in visibly to Desktop and start a chat with the same agent', async () => {- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()- await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))- await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))- await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()- await expect(desktop.getByTestId('nav-chat')).toBeVisible()- await desktop.getByTestId('nav-chat').click()- await desktop.getByTestId('nav-new-chat').click()- await expect(- desktop.getByRole('heading', { name: 'New chat with', exact: true })- ).toBeVisible()- await desktop.getByRole('button', { name: 'Switch chat agent', exact: true }).click()- await desktop- .getByRole('menuitem', { name: scenario.agentDisplayName, exact: true })- .click()- await expect(- desktop.getByRole('button', { name: 'Switch chat agent', exact: true })- ).toContainText(scenario.agentDisplayName)- await desktop.getByRole('button', { name: 'Model — Select model', exact: true }).click()- const modelOption = desktop- .getByRole('menuitemradio')- .filter({ hasText: scenario.modelLabel })- await expect(modelOption).toHaveCount(1)- await modelOption.click()- await expect(- desktop.getByRole('button', { name: `Model — ${scenario.modelLabel}`, exact: true })- ).toBeVisible()- await expect(desktop.getByTestId('agent-response')).toHaveCount(0)+ await openAgentChat(desktop, scenario)})await test.step('Find only the verification receipt tool and return its real business ID', async () => {const upstreamBefore =@@ -522,3 +520,63 @@ for (const scenario of cases) {}})}++// Only the deterministic upstream can answer with more function calls than the+// contract allows, so this case is registered in deterministic mode only. It+// runs after the approved-tools journeys and reuses the first prepared agent,+// whose subscription model binding those journeys persisted.+if (mode === 'deterministic') {+ test('tool call limit: Desktop shows Too Many Tool Calls without retry or connector call', async () => {+ const scenario = cases[0]!+ const before = await readEvidence(scenario)+ // Witness for the negative connector assertion below: the receipt journey+ // of this run executed the connector through the same fixture.+ expect(before.calls.map(call => call.tool)).toEqual([+ 'workitem_read_receipt',+ 'workitem_read_receipt',+ ])+ const upstreamBefore = await readUpstreamEvidence(scenario)+ const app = await launchDesktopApp()+ let journeyPassed = false+ try {+ const desktop = await app.firstWindow()+ await test.step('Sign in visibly to Desktop and start a chat with the prepared agent', async () => {+ await openAgentChat(desktop, scenario)+ })+ const responses = await newAgentResponses(desktop)+ await test.step('Send the tool call limit probe and wait for the completed turn', async () => {+ await send(desktop, 'tool call limit probe')+ await expect(responses).toHaveCount(1, { timeout: 120_000 })+ await expect(desktop.getByTestId('send-button')).toHaveAttribute(+ 'aria-label',+ 'Send message'+ )+ })+ await test.step('The assistant message is in the error state', async () => {+ await expect(responses).toHaveClass(/(^|\s)chat-bubble--error(\s|$)/)+ })+ await test.step('The error names the tool-call limit, not an overloaded model', async () => {+ await expect(responses.getByText('Too Many Tool Calls', { exact: true })).toBeVisible()+ await expect(responses.getByText('Model Overloaded')).toHaveCount(0)+ })+ await test.step('One upstream completion, no retry, no connector call', async () => {+ const upstreamAfter = await readUpstreamEvidence(scenario)+ expect(upstreamAfter.limitProbe.turns - upstreamBefore.limitProbe.turns).toBe(1)+ expect(upstreamAfter.limitProbe.completions - upstreamBefore.limitProbe.completions).toBe(1)+ expect(upstreamAfter.limitProbe.unexpectedRetries).toBe(0)+ expect(upstreamAfter.rejected).toBe(upstreamBefore.rejected)+ const requests = upstreamAfter.requests.slice(upstreamBefore.requests.length)+ expect(requests.map(request => request.stage)).toEqual(['limit_probe'])+ expect((await readEvidence(scenario)).calls).toEqual(before.calls)+ })+ journeyPassed = true+ } finally {+ try {+ await app.close()+ } catch {+ if (journeyPassed) throw new Error('Desktop cleanup failed')+ // Preserve the original assertion failure when cleanup also fails.+ }+ }+ })+}diff --git a/tests/e2e/playwright/helpers/approved-tools-scenarios.ts b/tests/e2e/playwright/helpers/approved-tools-scenarios.tsindex 493d07738..5f896bff9 100644--- a/tests/e2e/playwright/helpers/approved-tools-scenarios.ts+++ b/tests/e2e/playwright/helpers/approved-tools-scenarios.ts@@ -26,6 +26,7 @@ export type UpstreamEvidence = {businessCalls: numberfinalResponses: numberdeniedResponses: number+ limitProbe: { turns: number; completions: number; unexpectedRetries: number }requests: Array<{definitionCount: numberexplicitNonStrictCount: number@@ -134,6 +135,11 @@ export async function readUpstreamEvidence(scenario: Scenario): Promise<Upstreamif (!Number.isSafeInteger(evidence[field]) || evidence[field] < 0)throw new Error(`Missing upstream evidence ${field}`)}+ for (const field of ['turns', 'completions', 'unexpectedRetries'] as const) {+ const value = evidence.limitProbe?.[field]+ if (!Number.isSafeInteger(value) || value < 0)+ throw new Error(`Missing upstream limit probe evidence ${field}`)+ }if (!Array.isArray(evidence.requests)) throw new Error('Missing upstream request evidence')return evidence}diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex eeb73b776..7dc91500c 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.49",+ "version": "0.1.50","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.49",+ "version": "0.1.50","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0",diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 0c6f0421d..843289c64 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.49",+ "version": "0.1.50","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {