Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): cover the Codex tool-call limit in the Desktop approved-tools journey

Alfredo Lopezed442902be69
diff --git a/codex-llm-proxy/test/approvedToolsUpstream.test.ts b/codex-llm-proxy/test/approvedToolsUpstream.test.ts
index 28a53cef4..233813bbe 100644
--- a/codex-llm-proxy/test/approvedToolsUpstream.test.ts
+++ b/codex-llm-proxy/test/approvedToolsUpstream.test.ts
@@ -1,5 +1,9 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
+import { hashCodexCompletionRequestV1 } from '@clerum/llm-provider-attempt-contract'
+import { streamCodexCompletion } from '../src/codexTransport.js'
+import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'
+import { CODEX_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
import { createApprovedToolsUpstream } from './approvedToolsUpstream'
const URL = 'https://chatgpt.com/backend-api/codex/responses'
@@ -370,3 +374,160 @@ describe('approved-tools isolated upstream boundary', () => {
expect(JSON.stringify(simulator.evidence())).not.toContain('workitem_read_001')
})
})
+
+describe('approved-tools tool-call limit probe', () => {
+ const probe = { role: 'user', content: 'Run the tool call limit probe now.' }
+
+ function events(body: string): Entry[] {
+ return body
+ .split('\n\n')
+ .filter(Boolean)
+ .map(frame => JSON.parse(frame.slice('data: '.length)) as Entry)
+ }
+
+ it('answers a probe turn with 65 distinct search calls in one completion', async () => {
+ const simulator = createApprovedToolsUpstream()
+ const { response, body } = await request(simulator, [probe])
+ expect(response.status).toBe(200)
+ const rows = events(body)
+ const calls = rows.slice(0, -1).map(row => row.item as Entry)
+ expect(rows[rows.length - 1]).toEqual({ type: 'response.completed' })
+ expect(calls).toHaveLength(65)
+ expect(new Set(calls.map(call => call.call_id)).size).toBe(65)
+ for (const call of calls) {
+ expect(call).toMatchObject({
+ type: 'function_call',
+ name: 'clerum__tool_search',
+ arguments: JSON.stringify({ query: 'verification receipt', limit: 5 }),
+ })
+ }
+ expect(simulator.evidence()).toMatchObject({
+ limitProbe: { turns: 1, completions: 1, unexpectedRetries: 0 },
+ completions: 1,
+ searchCalls: 0,
+ rejected: 0,
+ })
+ expect(simulator.evidence().requests).toMatchObject([{ stage: 'limit_probe' }])
+ })
+
+ it('rejects a second completion of the same probe turn as unexpected_retry', async () => {
+ const simulator = createApprovedToolsUpstream()
+ const first = await request(simulator, [probe])
+ expect(first.response.status).toBe(200)
+
+ const retry = await request(simulator, [probe])
+ const call = events(first.body)[0]!.item as Entry
+ const continuation = await request(simulator, [
+ probe,
+ call,
+ { type: 'function_call_output', call_id: call.call_id, output: '{}' },
+ ])
+
+ expect(retry.response.status).toBe(422)
+ expect(continuation.response.status).toBe(422)
+ expect(simulator.evidence()).toMatchObject({
+ limitProbe: { turns: 1, completions: 3, unexpectedRetries: 2 },
+ rejected: 2,
+ })
+ })
+
+ it('answers a new probe turn later in the same conversation', async () => {
+ const simulator = createApprovedToolsUpstream()
+ const first = await request(simulator, [probe])
+ const assistant = { role: 'assistant', content: 'The agent stopped.' }
+ const second = await request(simulator, [probe, assistant, probe])
+ expect([first.response.status, second.response.status]).toEqual([200, 200])
+ expect(simulator.evidence().limitProbe).toEqual({
+ turns: 2,
+ completions: 2,
+ unexpectedRetries: 0,
+ })
+ })
+
+ it('makes the real transport fail the probe with tool_call_limit_exceeded', async () => {
+ const simulator = createApprovedToolsUpstream()
+ const probeRequest = {
+ schemaVersion: 'codex-completion-request.v1' as const,
+ requestId: 'req-limit-probe',
+ idempotencyKey: 'idem-limit-probe',
+ provider: 'codex-subscription' as const,
+ model: 'gpt-5.3-codex',
+ messages: [{ role: 'user' as const, content: probe.content }],
+ tools: bridges.map(name => ({
+ name,
+ description: `${name} bridge`,
+ parameters: { type: 'object' },
+ })),
+ }
+ const requestHash = hashCodexCompletionRequestV1(probeRequest)
+ const emitted: Array<{ type: string }> = []
+ const finalize = vi.fn(async () => ({
+ providerAttemptId: 'att-limit-probe',
+ outcome: 'error' as const,
+ duplicate: false,
+ }))
+ const pending = streamCodexCompletion({
+ executionTicket: 'ticket-limit-probe',
+ requestHash,
+ request: probeRequest,
+ ticket: {
+ jti: 'jti-limit-probe',
+ hostRef: 'approved-tools-host',
+ model: probeRequest.model,
+ requestHash,
+ providerAttemptId: 'att-limit-probe',
+ },
+ redeem: vi.fn(
+ async (): Promise<RedeemAttemptSuccess> => ({
+ accessToken: `hdr.${Buffer.from(
+ JSON.stringify({ 'https://api.openai.com/auth': { chatgpt_account_id: 'acct_probe' } })
+ ).toString('base64url')}.sig`,
+ transport: {
+ protocolVersion: 'codex-subscription-transport.v1',
+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,
+ catalogOrigin: 'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0',
+ operation: 'completion_stream',
+ servedModel: probeRequest.model,
+ maxStreamDurationMs: 300_000,
+ },
+ expiryClass: 'short_lived',
+ attemptReceipt: 'b'.repeat(64),
+ })
+ ),
+ finalize,
+ fetchFn: simulator.fetchFn,
+ lookup: async () => [{ address: '104.18.32.47', family: 4 }],
+ onFrame: frame => {
+ emitted.push(frame)
+ },
+ })
+
+ await expect(pending).rejects.toMatchObject({
+ name: 'CodexTransportError',
+ code: 'tool_call_limit_exceeded',
+ details: { limit: 64, observed: 65 },
+ })
+ // Liveness witness: the upstream served exactly one probe completion.
+ expect(simulator.evidence().limitProbe).toEqual({
+ turns: 1,
+ completions: 1,
+ unexpectedRetries: 0,
+ })
+ expect(emitted.filter(frame => frame.type === 'tool_call')).toEqual([])
+ expect(finalize).toHaveBeenCalledTimes(1)
+ })
+
+ it('leaves the receipt journey unchanged after a probe turn (ordinary path witness)', async () => {
+ const simulator = createApprovedToolsUpstream()
+ await request(simulator, [probe])
+ await complete(simulator, [{ role: 'user', content: 'verification receipt' }], 'after-probe')
+ expect(simulator.evidence()).toMatchObject({
+ limitProbe: { turns: 1, completions: 1, unexpectedRetries: 0 },
+ searchCalls: 1,
+ describeCalls: 1,
+ businessCalls: 1,
+ finalResponses: 1,
+ rejected: 0,
+ })
+ })
+})
diff --git a/codex-llm-proxy/test/approvedToolsUpstream.ts b/codex-llm-proxy/test/approvedToolsUpstream.ts
index 26cc7a601..ff6235e6f 100644
--- a/codex-llm-proxy/test/approvedToolsUpstream.ts
+++ b/codex-llm-proxy/test/approvedToolsUpstream.ts
@@ -1,5 +1,5 @@
/** External model boundary for the isolated approved-tools E2E image only. */
-import { randomUUID } from 'node:crypto'
+import { createHash, randomUUID } from 'node:crypto'
import { isDeepStrictEqual } from 'node:util'
type Row = Record<string, unknown>
@@ -8,6 +8,10 @@ const CATALOG = 'https://chatgpt.com/backend-api/codex/models?client_version=1.0
const BRIDGES = ['clerum__tool_search', 'clerum__tool_describe', 'clerum__tool_call']
// The deterministic external model is scoped to this fixture's read journey.
const FIXTURE_QUERY = 'verification receipt'
+// A user turn with this text gets one response carrying more tool calls than
+// the Codex contract allows (maxToolCalls 64), so the proxy must reject it.
+const LIMIT_PROBE = 'tool call limit probe'
+const LIMIT_PROBE_CALLS = 65
function record(value: unknown): Row {
if (!value || typeof value !== 'object' || Array.isArray(value))
@@ -49,10 +53,7 @@ type Exchange = { call: Row; args: Row; result: Row }
type Decision = { stage: string; args?: Row; result?: Row }
function expectedCall(exchange: Exchange, decision: Decision): void {
- if (
- exchange.call.name !== decision.stage ||
- !isDeepStrictEqual(exchange.args, decision.args)
- ) {
+ if (exchange.call.name !== decision.stage || !isDeepStrictEqual(exchange.args, decision.args)) {
throw new Error('unexpected_fixture_call')
}
}
@@ -268,9 +269,11 @@ function workflowDecision(current: Exchange[], prompt: string): Decision {
throw new Error('unsupported_workflow_transition')
}
-function stream(event: Row): Response {
+function stream(...events: Row[]): Response {
return new Response(
- [event, { type: 'response.completed' }].map(row => `data: ${JSON.stringify(row)}\n\n`).join(''),
+ [...events, { type: 'response.completed' }]
+ .map(row => `data: ${JSON.stringify(row)}\n\n`)
+ .join(''),
{
headers: { 'content-type': 'text/event-stream' },
}
@@ -287,6 +290,9 @@ export function createApprovedToolsUpstream() {
businessCalls: 0,
finalResponses: 0,
deniedResponses: 0,
+ // Probe turns answered with an over-limit batch, every completion served
+ // for them, and completions that repeated an already answered probe turn.
+ limitProbe: { turns: 0, completions: 0, unexpectedRetries: 0 },
// Only bounded measurements, never request or result contents.
requests: [] as Array<{
definitionCount: number
@@ -298,6 +304,9 @@ export function createApprovedToolsUpstream() {
stage: string
}>,
}
+ // Digests of the history up to each answered probe user turn. A retry of the
+ // same turn resends that history unchanged.
+ const answeredProbeTurns = new Set<string>()
const fetchFn: typeof fetch = async (input, init) => {
try {
@@ -325,6 +334,44 @@ export function createApprovedToolsUpstream() {
const user = history[lastUser]
if (!user || typeof user.content !== 'string' || !user.content.trim())
throw new Error('missing_user_request')
+ if (user.content.toLowerCase().includes(LIMIT_PROBE)) {
+ if (evidence.requests.length >= 512) throw new Error('evidence_capacity_exceeded')
+ evidence.limitProbe.completions++
+ const turn = createHash('sha256')
+ .update(JSON.stringify(history.slice(0, lastUser + 1)))
+ .digest('hex')
+ if (answeredProbeTurns.has(turn)) {
+ evidence.limitProbe.unexpectedRetries++
+ throw new Error('unexpected_retry')
+ }
+ // An over-limit batch never yields tool results, so a continuation
+ // means the limit was not enforced.
+ if (history.length !== lastUser + 1) throw new Error('unexpected_probe_continuation')
+ answeredProbeTurns.add(turn)
+ evidence.limitProbe.turns++
+ evidence.completions++
+ evidence.requests.push({
+ definitionCount: tools.length,
+ explicitNonStrictCount: tools.filter(tool => tool.strict === false).length,
+ definitionBytes: Buffer.byteLength(JSON.stringify(tools)),
+ inputBytes: Buffer.byteLength(JSON.stringify(payload.input)),
+ connectorDefinitionCount: 0,
+ leakedSchema: false,
+ stage: 'limit_probe',
+ })
+ return stream(
+ ...Array.from({ length: LIMIT_PROBE_CALLS }, () => ({
+ type: 'response.output_item.done',
+ item: {
+ type: 'function_call',
+ id: randomUUID(),
+ call_id: randomUUID(),
+ name: BRIDGES[0],
+ arguments: JSON.stringify({ query: FIXTURE_QUERY, limit: 5 }),
+ },
+ }))
+ )
+ }
const isWorkflow = /\bworkflow\b/i.test(user.content)
if (!isWorkflow && !user.content.toLowerCase().includes(FIXTURE_QUERY))
throw new Error('unsupported_fixture_task')
diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjs
index b885d18ee..aaa2e930d 100644
--- a/scripts/e2e/run-codex-approved-tools.mjs
+++ b/scripts/e2e/run-codex-approved-tools.mjs
@@ -16,14 +16,24 @@ import {
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
const playwright = path.join(repo, 'tests/e2e/playwright')
const specFile = 'codex-subscription-approved-tools.spec.ts'
-export const expectedTitles = [
- 'unauthenticated agent route guard prevents connector use',
- 'authenticated user without agent access cannot select the protected agents',
- 'native workflow: visible approval, trigger, status and result artifact',
- ...[83, 150, 250].map(
- size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`
- ),
-]
+export const toolCallLimitTitle =
+ 'tool call limit: Desktop shows Too Many Tool Calls without retry or connector call'
+
+// The tool-call limit case exists only in deterministic mode: only the
+// deterministic upstream can emit more function calls than the contract allows.
+export function expectedTitles(mode) {
+ if (!['deterministic', 'real'].includes(mode))
+ throw new Error('Select deterministic or real upstream mode explicitly')
+ return [
+ 'unauthenticated agent route guard prevents connector use',
+ 'authenticated user without agent access cannot select the protected agents',
+ 'native workflow: visible approval, trigger, status and result artifact',
+ ...[83, 150, 250].map(
+ size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`
+ ),
+ ...(mode === 'deterministic' ? [toolCallLimitTitle] : []),
+ ]
+}
export function runPhaseTimeouts() {
return {
@@ -34,7 +44,8 @@ export function runPhaseTimeouts() {
}
}
-export function validateReport(report) {
+export function validateReport(report, mode) {
+ const titles = expectedTitles(mode)
const specs = []
function collect(suites) {
if (!Array.isArray(suites)) throw new Error('Incomplete Playwright suites')
@@ -49,12 +60,12 @@ export function validateReport(report) {
report.stats?.unexpected !== 0 ||
report.stats?.skipped !== 0 ||
report.stats?.flaky !== 0 ||
- report.stats?.expected !== expectedTitles.length ||
- specs.length !== expectedTitles.length
+ report.stats?.expected !== titles.length ||
+ specs.length !== titles.length
) {
throw new Error('Incomplete, failed, flaky or skipped approved-tools report')
}
- for (const title of expectedTitles) {
+ for (const title of titles) {
const matches = specs.filter(
spec =>
spec.title === title &&
@@ -194,7 +205,7 @@ function main() {
env,
APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS
)
- const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)))
+ const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)), mode)
process.stdout.write(
`${JSON.stringify({ lane: 'Playwright', upstream: mode, ...verified, report })}\n`
)
diff --git a/scripts/e2e/run-codex-approved-tools.test.mjs b/scripts/e2e/run-codex-approved-tools.test.mjs
index bb32b67a7..e90bee05c 100644
--- a/scripts/e2e/run-codex-approved-tools.test.mjs
+++ b/scripts/e2e/run-codex-approved-tools.test.mjs
@@ -4,16 +4,17 @@ import os from 'node:os'
import path from 'node:path'
import { test } from 'node:test'
import { openOwnedFile, readOwnedDescriptor } from './prepare-codex-approved-tools.mjs'
-import { expectedTitles, validateReport } from './run-codex-approved-tools.mjs'
+import { expectedTitles, toolCallLimitTitle, validateReport } from './run-codex-approved-tools.mjs'
// Synthetic reporter records exercise false-green rejection, not product data.
-function greenReport() {
+function greenReport(mode = 'deterministic') {
+ const titles = expectedTitles(mode)
return {
errors: [],
- stats: { expected: expectedTitles.length, unexpected: 0, skipped: 0, flaky: 0 },
+ stats: { expected: titles.length, unexpected: 0, skipped: 0, flaky: 0 },
suites: [
{
- specs: expectedTitles.map(title => ({
+ specs: titles.map(title => ({
title,
file: 'desktop/codex-subscription-approved-tools.spec.ts',
ok: true,
@@ -30,17 +31,32 @@ function greenReport() {
],
}
}
-test('accepts exactly all mandatory green cases', () => {
- assert.deepEqual(validateReport(greenReport()), {
- tests: expectedTitles.length,
- skipped: 0,
- failed: 0,
+for (const mode of ['deterministic', 'real']) {
+ test(`accepts exactly all mandatory green cases in ${mode} mode`, () => {
+ assert.deepEqual(validateReport(greenReport(mode), mode), {
+ tests: expectedTitles(mode).length,
+ skipped: 0,
+ failed: 0,
+ })
})
+}
+test('the tool-call limit case is mandatory only in deterministic mode', () => {
+ assert.equal(expectedTitles('deterministic').length, 7)
+ assert.equal(expectedTitles('real').length, 6)
+ assert.ok(expectedTitles('deterministic').includes(toolCallLimitTitle))
+ assert.ok(!expectedTitles('real').includes(toolCallLimitTitle))
+ assert.throws(() => expectedTitles('mock'))
+})
+test('rejects a deterministic report without the tool-call limit case', () => {
+ assert.throws(() => validateReport(greenReport('real'), 'deterministic'))
+})
+test('rejects a real report that carries the deterministic-only case', () => {
+ assert.throws(() => validateReport(greenReport('deterministic'), 'real'))
})
const corruptions = {
'missing case': report => report.suites[0].specs.pop(),
'duplicate case': report => {
- report.suites[0].specs[0].title = expectedTitles[1]
+ report.suites[0].specs[0].title = expectedTitles('deterministic')[1]
},
'different physical file': report => {
report.suites[0].specs[0].file = 'other.spec.ts'
@@ -71,7 +87,7 @@ for (const [name, corrupt] of Object.entries(corruptions)) {
test(`rejects ${name}`, () => {
const report = greenReport()
corrupt(report)
- assert.throws(() => validateReport(report))
+ assert.throws(() => validateReport(report, 'deterministic'))
})
}
@@ -81,9 +97,12 @@ test('report verification reads only the reserved reporter inode, never a replac
try {
fs.renameSync(path.join(root, 'report.json'), path.join(root, 'reserved.json'))
fs.writeFileSync(path.join(root, 'report.json'), JSON.stringify(greenReport()), { mode: 0o600 })
- assert.throws(() => validateReport(JSON.parse(readOwnedDescriptor(file))))
+ assert.throws(() => validateReport(JSON.parse(readOwnedDescriptor(file)), 'deterministic'))
fs.writeSync(file.fd, JSON.stringify(greenReport()))
- assert.equal(validateReport(JSON.parse(readOwnedDescriptor(file))).tests, expectedTitles.length)
+ assert.equal(
+ validateReport(JSON.parse(readOwnedDescriptor(file)), 'deterministic').tests,
+ expectedTitles('deterministic').length
+ )
} finally {
fs.closeSync(file.fd)
fs.rmSync(root, { recursive: true, force: true })
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
index 2432baed7..4a588ff6c 100644
--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
@@ -101,6 +101,30 @@ async function newAgentResponses(page: Page) {
return responses.and(page.locator(`[data-chat-message-id]${exclusions}`))
}
+async function openAgentChat(desktop: Page, scenario: Scenario) {
+ await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
+ await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))
+ await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))
+ await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()
+ await expect(desktop.getByTestId('nav-chat')).toBeVisible()
+ await desktop.getByTestId('nav-chat').click()
+ await desktop.getByTestId('nav-new-chat').click()
+ await expect(desktop.getByRole('heading', { name: 'New chat with', exact: true })).toBeVisible()
+ await desktop.getByRole('button', { name: 'Switch chat agent', exact: true }).click()
+ await desktop.getByRole('menuitem', { name: scenario.agentDisplayName, exact: true }).click()
+ await expect(
+ desktop.getByRole('button', { name: 'Switch chat agent', exact: true })
+ ).toContainText(scenario.agentDisplayName)
+ await desktop.getByRole('button', { name: 'Model — Select model', exact: true }).click()
+ const modelOption = desktop.getByRole('menuitemradio').filter({ hasText: scenario.modelLabel })
+ await expect(modelOption).toHaveCount(1)
+ await modelOption.click()
+ await expect(
+ desktop.getByRole('button', { name: `Model — ${scenario.modelLabel}`, exact: true })
+ ).toBeVisible()
+ await expect(desktop.getByTestId('agent-response')).toHaveCount(0)
+}
+
async function receiptApproval(page: Page, scenario: Scenario, previousCalls: number) {
const pending = page
.getByTestId('progress-stepper')
@@ -219,33 +243,7 @@ for (const scenario of cases) {
try {
const desktop = await app.firstWindow()
await test.step('Sign in visibly to Desktop and start a chat with the same agent', async () => {
- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
- await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))
- await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))
- await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()
- await expect(desktop.getByTestId('nav-chat')).toBeVisible()
- await desktop.getByTestId('nav-chat').click()
- await desktop.getByTestId('nav-new-chat').click()
- await expect(
- desktop.getByRole('heading', { name: 'New chat with', exact: true })
- ).toBeVisible()
- await desktop.getByRole('button', { name: 'Switch chat agent', exact: true }).click()
- await desktop
- .getByRole('menuitem', { name: scenario.agentDisplayName, exact: true })
- .click()
- await expect(
- desktop.getByRole('button', { name: 'Switch chat agent', exact: true })
- ).toContainText(scenario.agentDisplayName)
- await desktop.getByRole('button', { name: 'Model — Select model', exact: true }).click()
- const modelOption = desktop
- .getByRole('menuitemradio')
- .filter({ hasText: scenario.modelLabel })
- await expect(modelOption).toHaveCount(1)
- await modelOption.click()
- await expect(
- desktop.getByRole('button', { name: `Model — ${scenario.modelLabel}`, exact: true })
- ).toBeVisible()
- await expect(desktop.getByTestId('agent-response')).toHaveCount(0)
+ await openAgentChat(desktop, scenario)
})
await test.step('Find only the verification receipt tool and return its real business ID', async () => {
const upstreamBefore =
@@ -522,3 +520,63 @@ for (const scenario of cases) {
}
})
}
+
+// Only the deterministic upstream can answer with more function calls than the
+// contract allows, so this case is registered in deterministic mode only. It
+// runs after the approved-tools journeys and reuses the first prepared agent,
+// whose subscription model binding those journeys persisted.
+if (mode === 'deterministic') {
+ test('tool call limit: Desktop shows Too Many Tool Calls without retry or connector call', async () => {
+ const scenario = cases[0]!
+ const before = await readEvidence(scenario)
+ // Witness for the negative connector assertion below: the receipt journey
+ // of this run executed the connector through the same fixture.
+ expect(before.calls.map(call => call.tool)).toEqual([
+ 'workitem_read_receipt',
+ 'workitem_read_receipt',
+ ])
+ const upstreamBefore = await readUpstreamEvidence(scenario)
+ const app = await launchDesktopApp()
+ let journeyPassed = false
+ try {
+ const desktop = await app.firstWindow()
+ await test.step('Sign in visibly to Desktop and start a chat with the prepared agent', async () => {
+ await openAgentChat(desktop, scenario)
+ })
+ const responses = await newAgentResponses(desktop)
+ await test.step('Send the tool call limit probe and wait for the completed turn', async () => {
+ await send(desktop, 'tool call limit probe')
+ await expect(responses).toHaveCount(1, { timeout: 120_000 })
+ await expect(desktop.getByTestId('send-button')).toHaveAttribute(
+ 'aria-label',
+ 'Send message'
+ )
+ })
+ await test.step('The assistant message is in the error state', async () => {
+ await expect(responses).toHaveClass(/(^|\s)chat-bubble--error(\s|$)/)
+ })
+ await test.step('The error names the tool-call limit, not an overloaded model', async () => {
+ await expect(responses.getByText('Too Many Tool Calls', { exact: true })).toBeVisible()
+ await expect(responses.getByText('Model Overloaded')).toHaveCount(0)
+ })
+ await test.step('One upstream completion, no retry, no connector call', async () => {
+ const upstreamAfter = await readUpstreamEvidence(scenario)
+ expect(upstreamAfter.limitProbe.turns - upstreamBefore.limitProbe.turns).toBe(1)
+ expect(upstreamAfter.limitProbe.completions - upstreamBefore.limitProbe.completions).toBe(1)
+ expect(upstreamAfter.limitProbe.unexpectedRetries).toBe(0)
+ expect(upstreamAfter.rejected).toBe(upstreamBefore.rejected)
+ const requests = upstreamAfter.requests.slice(upstreamBefore.requests.length)
+ expect(requests.map(request => request.stage)).toEqual(['limit_probe'])
+ expect((await readEvidence(scenario)).calls).toEqual(before.calls)
+ })
+ journeyPassed = true
+ } finally {
+ try {
+ await app.close()
+ } catch {
+ if (journeyPassed) throw new Error('Desktop cleanup failed')
+ // Preserve the original assertion failure when cleanup also fails.
+ }
+ }
+ })
+}
diff --git a/tests/e2e/playwright/helpers/approved-tools-scenarios.ts b/tests/e2e/playwright/helpers/approved-tools-scenarios.ts
index 493d07738..5f896bff9 100644
--- a/tests/e2e/playwright/helpers/approved-tools-scenarios.ts
+++ b/tests/e2e/playwright/helpers/approved-tools-scenarios.ts
@@ -26,6 +26,7 @@ export type UpstreamEvidence = {
businessCalls: number
finalResponses: number
deniedResponses: number
+ limitProbe: { turns: number; completions: number; unexpectedRetries: number }
requests: Array<{
definitionCount: number
explicitNonStrictCount: number
@@ -134,6 +135,11 @@ export async function readUpstreamEvidence(scenario: Scenario): Promise<Upstream
if (!Number.isSafeInteger(evidence[field]) || evidence[field] < 0)
throw new Error(`Missing upstream evidence ${field}`)
}
+ for (const field of ['turns', 'completions', 'unexpectedRetries'] as const) {
+ const value = evidence.limitProbe?.[field]
+ if (!Number.isSafeInteger(value) || value < 0)
+ throw new Error(`Missing upstream limit probe evidence ${field}`)
+ }
if (!Array.isArray(evidence.requests)) throw new Error('Missing upstream request evidence')
return evidence
}
diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.json
index eeb73b776..7dc91500c 100644
--- a/tests/e2e/playwright/package-lock.json
+++ b/tests/e2e/playwright/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.49",
+ "version": "0.1.50",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-playwright-tests",
- "version": "0.1.49",
+ "version": "0.1.50",
"devDependencies": {
"@playwright/test": "^1.50.0",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.json
index 0c6f0421d..843289c64 100644
--- a/tests/e2e/playwright/package.json
+++ b/tests/e2e/playwright/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.49",
+ "version": "0.1.50",
"private": true,
"description": "Playwright E2E tests for Control UI and Desktop App",
"scripts": {