Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(codex): cover ordinary tool use and approval journeys

Alfredo Lopezdc1e9af9c88f
diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjs
index e3b13634d..49f822633 100644
--- a/scripts/e2e/run-codex-approved-tools.mjs
+++ b/scripts/e2e/run-codex-approved-tools.mjs
@@ -13,7 +13,11 @@ const playwright = path.join(repo, 'tests/e2e/playwright')
const specFile = 'codex-subscription-approved-tools.spec.ts'
export const expectedTitles = [
'unauthenticated agent route guard prevents connector use',
- ...[83, 150, 250].map(size => `approved tools ${size}: selective receipt and revoked connector`),
+ 'authenticated user without agent access cannot select the protected agents',
+ 'native workflow: visible approval, trigger, status and result artifact',
+ ...[83, 150, 250].map(
+ size => `approved tools ${size}: ordinary discovery, reuse, approval decisions and revocation`
+ ),
]
export function validateReport(report) {
@@ -68,7 +72,7 @@ function required(name) {
}
function runNode(args, cwd, env, timeout) {
- if (!Number.isSafeInteger(timeout) || timeout < 1000 || timeout > 22 * 60_000)
+ if (!Number.isSafeInteger(timeout) || timeout < 1000 || timeout > 35 * 60_000)
throw new Error('Invalid browser deadline')
const result = spawnSync(
process.execPath,
@@ -106,6 +110,9 @@ function main() {
'EXTERNAL_REST_API_BASE_URL',
'RPC_PROXY_BASE_URL',
'APPROVED_TOOLS_SCENARIOS',
+ 'APPROVED_TOOLS_WORKFLOW_SCENARIO',
+ 'APPROVED_TOOLS_UNAUTHORIZED_EMAIL',
+ 'APPROVED_TOOLS_UNAUTHORIZED_PASSWORD',
])
required(key)
const mode = required('APPROVED_TOOLS_UPSTREAM_MODE')
@@ -152,6 +159,7 @@ function main() {
'tools/e2e_static_audit.py',
`tests/e2e/playwright/desktop/${specFile}`,
'tests/e2e/playwright/helpers/approved-tools-scenarios.ts',
+ 'tests/e2e/playwright/helpers/approved-tools-workflow.ts',
],
{ cwd: repo, env, stdio: 'inherit', timeout: 30_000 }
)
@@ -165,7 +173,7 @@ function main() {
],
playwright,
env,
- 22 * 60_000
+ 35 * 60_000
)
const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)))
process.stdout.write(
diff --git a/scripts/e2e/run-codex-approved-tools.test.mjs b/scripts/e2e/run-codex-approved-tools.test.mjs
index f1404d1c1..bb32b67a7 100644
--- a/scripts/e2e/run-codex-approved-tools.test.mjs
+++ b/scripts/e2e/run-codex-approved-tools.test.mjs
@@ -10,7 +10,7 @@ import { expectedTitles, validateReport } from './run-codex-approved-tools.mjs'
function greenReport() {
return {
errors: [],
- stats: { expected: 4, unexpected: 0, skipped: 0, flaky: 0 },
+ stats: { expected: expectedTitles.length, unexpected: 0, skipped: 0, flaky: 0 },
suites: [
{
specs: expectedTitles.map(title => ({
@@ -30,8 +30,12 @@ function greenReport() {
],
}
}
-test('accepts exactly the required four green cases', () => {
- assert.deepEqual(validateReport(greenReport()), { tests: 4, skipped: 0, failed: 0 })
+test('accepts exactly all mandatory green cases', () => {
+ assert.deepEqual(validateReport(greenReport()), {
+ tests: expectedTitles.length,
+ skipped: 0,
+ failed: 0,
+ })
})
const corruptions = {
'missing case': report => report.suites[0].specs.pop(),
@@ -79,7 +83,7 @@ test('report verification reads only the reserved reporter inode, never a replac
fs.writeFileSync(path.join(root, 'report.json'), JSON.stringify(greenReport()), { mode: 0o600 })
assert.throws(() => validateReport(JSON.parse(readOwnedDescriptor(file))))
fs.writeSync(file.fd, JSON.stringify(greenReport()))
- assert.equal(validateReport(JSON.parse(readOwnedDescriptor(file))).tests, 4)
+ assert.equal(validateReport(JSON.parse(readOwnedDescriptor(file))).tests, expectedTitles.length)
} finally {
fs.closeSync(file.fd)
fs.rmSync(root, { recursive: true, force: true })
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
index be90e6b09..830efe8ed 100644
--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
@@ -17,6 +17,7 @@ import {
required,
scenarios,
} from '../helpers/approved-tools-scenarios'
+import { workflowJourney } from '../helpers/approved-tools-workflow'
import { launchDesktopApp } from '../helpers/launch-desktop'
import { loginControlUiVisible } from '../helpers/visible-login'
import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'
@@ -70,6 +71,12 @@ async function saveConnector(page: Page, scenario: Scenario, remove: boolean) {
else await expect(row).toBeVisible()
}
+test('native workflow: visible approval, trigger, status and result artifact', async ({
+ page,
+}, testInfo) => {
+ await workflowJourney(page, testInfo)
+})
+
async function send(page: Page, prompt: string) {
await expect(page.getByTestId('chat-input')).toBeVisible()
await page.getByTestId('chat-input').fill(prompt)
@@ -78,6 +85,18 @@ async function send(page: Page, prompt: string) {
await expect(page.getByTestId('message-list').getByText(prompt, { exact: true })).toBeVisible()
}
+async function receiptApproval(page: Page, scenario: Scenario, previousCalls: number) {
+ const pending = page
+ .getByTestId('progress-stepper')
+ .filter({ has: page.getByTestId('approval-approve-btn') })
+ await expect(pending).toHaveCount(1)
+ await expect(pending).toContainText('workitem_read_receipt')
+ await expect(pending).toContainText('requires approval')
+ await expect(pending).not.toContainText('clerum__tool_call requires approval')
+ expect((await readEvidence(scenario)).calls).toHaveLength(previousCalls)
+ return pending
+}
+
test('unauthenticated agent route guard prevents connector use', async ({ page }) => {
const scenario = cases[0]!
const before = await readEvidence(scenario)
@@ -90,8 +109,37 @@ test('unauthenticated agent route guard prevents connector use', async ({ page }
expect((await readEvidence(scenario)).calls).toEqual(before.calls)
})
+test('authenticated user without agent access cannot select the protected agents', async () => {
+ const before = await Promise.all(cases.map(readEvidence))
+ const app = await launchDesktopApp()
+ try {
+ const page = await app.firstWindow()
+ await expect(page.getByLabel('Email', { exact: true })).toBeVisible()
+ await page
+ .getByLabel('Email', { exact: true })
+ .fill(required('APPROVED_TOOLS_UNAUTHORIZED_EMAIL'))
+ await page
+ .getByLabel('Password', { exact: true })
+ .fill(required('APPROVED_TOOLS_UNAUTHORIZED_PASSWORD'))
+ await page.getByRole('button', { name: 'Sign in', exact: true }).click()
+ await expect(page.getByTestId('nav-agents')).toBeVisible()
+ await page.getByTestId('nav-agents').click()
+ await expect(page.getByRole('heading', { name: 'Agents', exact: true })).toBeVisible()
+ await expect(page.getByRole('heading', { name: 'No agents', exact: true })).toBeVisible()
+ for (const scenario of cases) {
+ await expect(page.getByText(scenario.agentDisplayName, { exact: true })).toHaveCount(0)
+ }
+ await page.getByTestId('nav-chat').click()
+ await expect(page.getByTestId('chat-input')).toHaveCount(0)
+ await expect(page.getByTestId('send-button')).toHaveCount(0)
+ expect(await Promise.all(cases.map(readEvidence))).toEqual(before)
+ } finally {
+ await app.close()
+ }
+})
+
for (const scenario of cases) {
- test(`approved tools ${scenario.catalogSize}: selective receipt and revoked connector`, async ({
+ test(`approved tools ${scenario.catalogSize}: ordinary discovery, reuse, approval decisions and revocation`, async ({
page,
}, testInfo) => {
testInfo.annotations.push({ type: 'upstream', description: mode })
@@ -135,6 +183,9 @@ for (const scenario of cases) {
await saveConnector(page, scenario, false)
})
const app = await launchDesktopApp()
+ const corpusStarted = Date.now()
+ const completedTasks: string[] = []
+ let corpusOutcome = 'failed'
try {
const desktop = await app.firstWindow()
await test.step('Sign in visibly to Desktop and start a chat with the same agent', async () => {
@@ -170,10 +221,10 @@ for (const scenario of cases) {
const upstreamBefore =
mode === 'deterministic' ? await readUpstreamEvidence(scenario) : undefined
const started = Date.now()
- await send(
- desktop,
- 'Find the verification receipt tool in my approved connectors. Describe only that tool, call it once, and report its exact businessId. Do not call other business tools or invent a receipt.'
- )
+ await send(desktop, 'Show my verification receipt and its business ID.')
+ const pending = await receiptApproval(desktop, scenario, 0)
+ await pending.getByTestId('approval-approve-btn').click()
+ await expect(pending).toHaveCount(0)
await expect
.poll(async () => (await readEvidence(scenario)).calls.length, { timeout: 120_000 })
.toBe(1)
@@ -235,7 +286,88 @@ for (const scenario of cases) {
body: JSON.stringify(evidence),
contentType: 'application/json',
})
+ completedTasks.push('ordinary-receipt')
})
+ await test.step('Repeat an ordinary request using the current schema without rediscovering it', async () => {
+ const before = await readEvidence(scenario)
+ const upstreamBefore =
+ mode === 'deterministic' ? await readUpstreamEvidence(scenario) : undefined
+ const started = Date.now()
+ await send(desktop, 'Show my verification receipt again.')
+ const pending = await receiptApproval(desktop, scenario, 1)
+ await pending.getByTestId('approval-approve-btn').click()
+ await expect(pending).toHaveCount(0)
+ await expect
+ .poll(async () => (await readEvidence(scenario)).calls.length, { timeout: 120_000 })
+ .toBe(2)
+ const after = await readEvidence(scenario)
+ expect(after.calls.map(call => call.tool)).toEqual([
+ 'workitem_read_receipt',
+ 'workitem_read_receipt',
+ ])
+ // MCP JSON-RPC ids are connection-local and may be reused after reconnect;
+ // the two separately completed turns and exact call count prove reuse.
+ await expect(
+ desktop.getByTestId('agent-response').filter({ hasText: before.calls[0]!.businessId })
+ ).toHaveCount(2)
+ await expect(desktop.getByTestId('send-button')).toHaveAttribute(
+ 'aria-label',
+ 'Send message'
+ )
+ let requests = null
+ if (upstreamBefore) {
+ const after = await readUpstreamEvidence(scenario)
+ expect(after.rejected).toBe(upstreamBefore.rejected)
+ expect(after.describeCalls).toBe(upstreamBefore.describeCalls)
+ expect(after.searchCalls).toBe(upstreamBefore.searchCalls)
+ expect(after.businessCalls - upstreamBefore.businessCalls).toBe(1)
+ requests = after.requests.slice(upstreamBefore.requests.length)
+ expect(requests.map(request => request.stage)).toEqual(['clerum__tool_call', 'final'])
+ for (const request of requests) {
+ expect(request.connectorDefinitionCount).toBe(0)
+ expect(request.leakedSchema).toBe(false)
+ }
+ }
+ await testInfo.attach('repeat-task-metrics', {
+ body: JSON.stringify({
+ mode,
+ catalogSize: scenario.catalogSize,
+ outcome: 'receipt-read',
+ businessCalls: 1,
+ requests,
+ elapsedMs: Date.now() - started,
+ inputTokens: null,
+ outputTokens: null,
+ }),
+ contentType: 'application/json',
+ })
+ completedTasks.push('repeat-receipt')
+ })
+ for (const decision of ['deny', 'cancel'] as const) {
+ await test.step(`${decision} the real tool approval without executing the connector`, async () => {
+ const before = await readEvidence(scenario)
+ const cancelled = desktop.getByTestId('progress-stepper').filter({ hasText: 'Cancelled' })
+ const cancelledBefore = await cancelled.count()
+ await send(
+ desktop,
+ decision === 'deny'
+ ? 'Please read my verification receipt once more.'
+ : 'Please fetch my verification receipt now.'
+ )
+ const pending = await receiptApproval(desktop, scenario, 2)
+ await pending
+ .getByTestId(decision === 'deny' ? 'approval-deny-btn' : 'progress-cancel-btn')
+ .click()
+ await expect(pending).toHaveCount(0)
+ await expect(cancelled).toHaveCount(cancelledBefore + 1)
+ await expect(desktop.getByTestId('send-button')).toHaveAttribute(
+ 'aria-label',
+ 'Send message'
+ )
+ expect((await readEvidence(scenario)).calls).toEqual(before.calls)
+ completedTasks.push(decision)
+ })
+ }
await test.step('Revoke through Control UI and deny the next call in the existing chat', async () => {
const before = await readEvidence(scenario)
const upstreamBefore =
@@ -251,11 +383,11 @@ for (const scenario of cases) {
page.getByRole('alertdialog', { name: 'Remove connector from this agent?' })
).toBeVisible()
await saveConnector(page, scenario, true)
- await send(
- desktop,
- 'Try reading the verification receipt again using the same tool. If access has been revoked or the tool is unavailable, reply exactly CONNECTOR_UNAVAILABLE. Never reuse the earlier receipt as a new result.'
- )
- const denied = mode === 'real' ? 'CONNECTOR_UNAVAILABLE' : '"found":0'
+ await send(desktop, 'Show my verification receipt again, please.')
+ const denied =
+ mode === 'real'
+ ? /unavailable|not available|no longer|cannot access|could not find|not found/i
+ : /"found"\s*:\s*0/
await expect(desktop.getByTestId('agent-response').filter({ hasText: denied })).toBeVisible(
{ timeout: 120_000 }
)
@@ -280,8 +412,28 @@ for (const scenario of cases) {
contentType: 'application/json',
})
}
+ completedTasks.push('revoked-receipt')
})
+ corpusOutcome = 'passed'
} finally {
+ const desktop = await app.firstWindow()
+ // Read-only measurement snapshot, not a readiness or correctness assertion.
+ // Preserve exact provider-reported UI figures; never infer tokens from bytes.
+ const reportedTokenLabels = await desktop
+ .getByLabel(/^Turn token usage/)
+ .evaluateAll(elements => elements.map(element => element.getAttribute('aria-label')))
+ await testInfo.attach('provider-parity-corpus', {
+ body: JSON.stringify({
+ mode,
+ catalogSize: scenario.catalogSize,
+ outcome: corpusOutcome,
+ completedTasks,
+ elapsedMs: Date.now() - corpusStarted,
+ reportedTokenLabels,
+ subscriptionUsage: null,
+ }),
+ contentType: 'application/json',
+ })
await app.close()
}
})
diff --git a/tests/e2e/playwright/helpers/approved-tools-workflow.ts b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
new file mode 100644
index 000000000..0076776e1
--- /dev/null
+++ b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
@@ -0,0 +1,221 @@
+/**
+ * E2E_GUARDIAN_ENTRY_POINT: root entry, visible configuration and login.
+ * E2E_GUARDIAN_IPC_FLOW: chat and both approval decisions use genuine Desktop
+ * IPC. The prepared workflow workload produces a receipt and its JSON artifact;
+ * this helper only reads receipt evidence after the real workflow executes.
+ */
+import { type Page, type TestInfo, expect, test } from '@playwright/test'
+import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'
+import {
+ type Scenario,
+ localUrl,
+ readEvidence,
+ readUpstreamEvidence,
+ required,
+ scenarios,
+} from './approved-tools-scenarios'
+import { launchDesktopApp } from './launch-desktop'
+import { loginControlUiVisible } from './visible-login'
+
+type WorkflowScenario = Scenario & { workflowName: string; workflowNamespace: string }
+
+function workflowScenario(): WorkflowScenario {
+ const value = JSON.parse(required('APPROVED_TOOLS_WORKFLOW_SCENARIO')) as WorkflowScenario
+ for (const key of [
+ 'agentName',
+ 'agentDisplayName',
+ 'contextName',
+ 'subscriptionName',
+ 'connectionKey',
+ 'modelName',
+ 'modelLabel',
+ 'workflowName',
+ 'workflowNamespace',
+ 'runId',
+ ] as const) {
+ if (typeof value[key] !== 'string' || !value[key].trim())
+ throw new Error(`Missing workflow precondition: ${key}`)
+ }
+ for (const key of [
+ 'agentName',
+ 'contextName',
+ 'workflowName',
+ 'workflowNamespace',
+ 'runId',
+ ] as const) {
+ if (!/^[a-zA-Z0-9_-]{1,80}$/.test(value[key]))
+ throw new Error(`Invalid workflow precondition: ${key}`)
+ }
+ if (![83, 150, 250].includes(value.catalogSize))
+ throw new Error('Workflow receipt fixture must use a supported catalog')
+ localUrl(value.fixtureUrl)
+ if (required('APPROVED_TOOLS_UPSTREAM_MODE') === 'deterministic')
+ localUrl(value.upstreamEvidenceUrl)
+ if (
+ scenarios().some(
+ row =>
+ row.agentName === value.agentName ||
+ row.runId === value.runId ||
+ row.fixtureUrl === value.fixtureUrl
+ )
+ )
+ throw new Error('Workflow fixture must be isolated from receipt scenarios')
+ return value
+}
+
+export async function workflowJourney(page: Page, testInfo: TestInfo) {
+ const scenario = workflowScenario()
+ const mode = required('APPROVED_TOOLS_UPSTREAM_MODE')
+ expect((await readEvidence(scenario)).calls).toEqual([])
+ await test.step('Select the isolated workflow agent and bind its subscription visibly', async () => {
+ await page.goto('/')
+ await loginControlUiVisible(page)
+ await new ControlUiShell(page).openAgents()
+ await new AgentListPage(page).openNamed(scenario.agentName)
+ const model = new AgentModelPage(page)
+ await model.openEditor()
+ await model.chooseSubscription(scenario.subscriptionName)
+ await page.getByLabel('Current model', { exact: true }).click()
+ await page.getByRole('option', { name: scenario.modelName, exact: true }).click()
+ const saved = await model.saveHost(scenario.agentName)
+ expect(saved.spec?.model).toMatchObject({
+ provider: 'codex-subscription',
+ name: scenario.modelName,
+ connectionRef: scenario.connectionKey,
+ })
+ expect(
+ (saved.spec as { llmPolicy?: { fallbacks?: unknown[] } })?.llmPolicy?.fallbacks ?? []
+ ).toEqual([])
+ await expect(page.getByRole('button', { name: 'Edit', exact: true })).toBeVisible()
+ })
+
+ await test.step('Require approval for the real native workflow trigger through Advanced settings', async () => {
+ await page.getByRole('tab', { name: 'Advanced', exact: true }).click()
+ await expect(page).toHaveURL(new RegExp(`/agents/${scenario.agentName}/advanced$`))
+ await page.getByRole('tab', { name: 'Per-tool approval', exact: true }).click()
+ const advanced = page.getByRole('region', { name: 'Advanced', exact: true })
+ await advanced.getByRole('button', { name: /Advanced: conditional tools/ }).click()
+ await advanced.getByLabel('Tool name', { exact: true }).fill('workflow_trigger')
+ await advanced
+ .getByLabel('Approval state for the new custom tool', { exact: true })
+ .selectOption('required')
+ await advanced.getByRole('button', { name: 'Add', exact: true }).click()
+ const update = page.waitForResponse(
+ response =>
+ new URL(response.url()).pathname === `/api/v1/admin/hosts/${scenario.agentName}` &&
+ response.request().method() === 'PUT'
+ )
+ await advanced.getByRole('button', { name: 'Save', exact: true }).click()
+ const response = await update
+ expect(response.ok()).toBe(true)
+ expect((await response.json()).spec.approval.tools.workflow_trigger).toBe(true)
+ await expect(advanced.getByRole('button', { name: 'Save', exact: true })).toBeDisabled()
+ await expect(advanced.getByText('workflow_trigger', { exact: true })).toBeVisible()
+ })
+
+ const app = await launchDesktopApp()
+ try {
+ const desktop = await app.firstWindow()
+ await test.step('Sign in to Desktop, select the same agent and subscription model', async () => {
+ await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
+ await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))
+ await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))
+ await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()
+ await expect(desktop.getByTestId('nav-chat')).toBeVisible()
+ await desktop.getByTestId('nav-chat').click()
+ await desktop.getByTestId('nav-new-chat').click()
+ await expect(
+ desktop.getByRole('heading', { name: 'New chat with', exact: true })
+ ).toBeVisible()
+ await desktop.getByRole('button', { name: 'Switch chat agent', exact: true }).click()
+ await desktop.getByRole('menuitem', { name: scenario.agentDisplayName, exact: true }).click()
+ await expect(
+ desktop.getByRole('button', { name: 'Switch chat agent', exact: true })
+ ).toContainText(scenario.agentDisplayName)
+ await desktop.getByRole('button', { name: 'Model — Select model', exact: true }).click()
+ const option = desktop.getByRole('menuitemradio').filter({ hasText: scenario.modelLabel })
+ await expect(option).toHaveCount(1)
+ await option.click()
+ await expect(
+ desktop.getByRole('button', { name: `Model — ${scenario.modelLabel}`, exact: true })
+ ).toBeVisible()
+ })
+
+ await test.step('Request the workflow and approve its native invocation and bound recipe approval', async () => {
+ const upstreamBefore =
+ mode === 'deterministic' ? await readUpstreamEvidence(scenario) : undefined
+ const started = Date.now()
+ const prompt = `Run my ${scenario.workflowName} workflow, check that it finishes, and show the business ID from its result artifact.`
+ await desktop.getByTestId('chat-input').fill(prompt)
+ await desktop.getByTestId('send-button').click()
+ await expect(
+ desktop.getByTestId('message-list').getByText(prompt, { exact: true })
+ ).toBeVisible()
+ const approval = desktop
+ .getByTestId('progress-stepper')
+ .filter({ has: desktop.getByTestId('approval-approve-btn') })
+ await expect(approval).toHaveCount(1)
+ await expect(approval).toContainText('workflow_trigger')
+ expect((await readEvidence(scenario)).calls).toEqual([])
+ await approval.getByTestId('approval-approve-btn').click()
+ await expect(approval).toHaveCount(0)
+
+ await desktop.getByTestId('notification-bell').click()
+ const panel = desktop.getByRole('dialog', { name: 'Notifications and approvals' })
+ await expect(panel).toBeVisible()
+ const card = panel
+ .getByTestId('workflow-approval-card')
+ .filter({ hasText: scenario.workflowName })
+ await expect(card).toHaveCount(1)
+ await expect(card).toContainText(scenario.workflowNamespace)
+ await expect(card).toContainText(
+ `Step: workflow_trigger:${scenario.workflowNamespace}/${scenario.workflowName}`
+ )
+ expect((await readEvidence(scenario)).calls).toEqual([])
+ await card.getByTestId('workflow-approval-approve').click()
+ await expect(card).toHaveCount(0)
+ await desktop.getByTestId('notification-bell').click()
+ await expect(panel).toHaveCount(0)
+ await expect
+ .poll(async () => (await readEvidence(scenario)).calls.length, { timeout: 120_000 })
+ .toBe(1)
+ const receipt = (await readEvidence(scenario)).calls[0]!
+ expect(receipt.tool).toBe('workitem_read_receipt')
+ expect(receipt.businessId).toMatch(/^[0-9a-f-]{36}$/)
+ await expect(
+ desktop.getByTestId('agent-response').filter({ hasText: receipt.businessId })
+ ).toBeVisible({ timeout: 120_000 })
+ await expect(desktop.getByTestId('send-button')).toHaveAttribute('aria-label', 'Send message')
+ expect((await readEvidence(scenario)).calls).toHaveLength(1)
+ let requests = null
+ if (upstreamBefore) {
+ const after = await readUpstreamEvidence(scenario)
+ expect(after.rejected).toBe(upstreamBefore.rejected)
+ requests = after.requests.slice(upstreamBefore.requests.length)
+ const stages = requests.map(request => request.stage)
+ expect(stages.slice(0, 2)).toEqual(['workflow_list', 'workflow_trigger'])
+ expect(stages.slice(2, -2).length).toBeGreaterThan(0)
+ expect(stages.slice(2, -2).every(stage => stage === 'workflow_status')).toBe(true)
+ expect(stages.slice(-2)).toEqual(['workflow_result', 'final'])
+ }
+ // Read-only exact provider-reported UI telemetry, never inferred from bytes.
+ const reportedTokenLabels = await desktop
+ .getByLabel(/^Turn token usage/)
+ .evaluateAll(elements => elements.map(element => element.getAttribute('aria-label')))
+ await testInfo.attach('native-workflow-result', {
+ body: JSON.stringify({
+ mode,
+ workflowName: scenario.workflowName,
+ receipt,
+ requests,
+ elapsedMs: Date.now() - started,
+ reportedTokenLabels,
+ subscriptionUsage: null,
+ }),
+ contentType: 'application/json',
+ })
+ })
+ } finally {
+ await app.close()
+ }
+}
diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.json
index 9ea776ee0..41a19ce63 100644
--- a/tests/e2e/playwright/package-lock.json
+++ b/tests/e2e/playwright/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.41",
+ "version": "0.1.43",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-playwright-tests",
- "version": "0.1.41",
+ "version": "0.1.43",
"devDependencies": {
"@playwright/test": "^1.50.0",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.json
index 8ca8667f0..9a95a127b 100644
--- a/tests/e2e/playwright/package.json
+++ b/tests/e2e/playwright/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.41",
+ "version": "0.1.43",
"private": true,
"description": "Playwright E2E tests for Control UI and Desktop App",
"scripts": {
diff --git a/tests/e2e/playwright/playwright.codex-approved-tools.config.ts b/tests/e2e/playwright/playwright.codex-approved-tools.config.ts
index 68f7e4330..ed7912761 100644
--- a/tests/e2e/playwright/playwright.codex-approved-tools.config.ts
+++ b/tests/e2e/playwright/playwright.codex-approved-tools.config.ts
@@ -7,7 +7,7 @@ export default defineConfig({
fullyParallel: false,
workers: 1,
retries: 0,
- timeout: 300_000,
+ timeout: 480_000,
expect: { timeout: 30_000 },
forbidOnly: true,
reporter: [['list'], ['json', { outputFile: required('APPROVED_TOOLS_REPORT') }]],