Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): make the session tool approval spec runnable on a branch profile

Alfredo Lopez1db498165d75
diff --git a/desktop-app/test/e2e-playwright/fixtures.ts b/desktop-app/test/e2e-playwright/fixtures.ts
index 65c4342a9..d2d87628e 100644
--- a/desktop-app/test/e2e-playwright/fixtures.ts
+++ b/desktop-app/test/e2e-playwright/fixtures.ts
@@ -9,7 +9,12 @@ import { execFileSync } from 'node:child_process'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
-import { E2E_DESKTOP_PASSWORD, loginAs as apiLoginAs, seedDesktopPasswordLogin } from './workflowUi'
+import {
+ E2E_DESKTOP_PASSWORD,
+ loginAs as apiLoginAs,
+ awaitPasswordPace,
+ seedDesktopPasswordLogin,
+} from './workflowUi'
const E2E_EMAIL = process.env.E2E_DEV_LOGIN_EMAIL || '[email protected]'
@@ -74,6 +79,7 @@ export const test = base.extend<Fixtures>({
if (await emailInput.isVisible()) {
await emailInput.fill(E2E_EMAIL)
await passwordInput.fill(E2E_DESKTOP_PASSWORD)
+ await awaitPasswordPace()
await page.click('button:has-text("Sign in")')
}
@@ -150,6 +156,7 @@ export async function loginAs(page: Page, email: string): Promise<void> {
if (await emailInput.isVisible()) {
await emailInput.fill(email)
await passwordInput.fill(E2E_DESKTOP_PASSWORD)
+ await awaitPasswordPace()
await page.click('button:has-text("Sign in")')
}
diff --git a/desktop-app/test/e2e-playwright/session-tool-approval-scope.test.ts b/desktop-app/test/e2e-playwright/session-tool-approval-scope.test.ts
index 95c762006..650d8013f 100644
--- a/desktop-app/test/e2e-playwright/session-tool-approval-scope.test.ts
+++ b/desktop-app/test/e2e-playwright/session-tool-approval-scope.test.ts
@@ -43,6 +43,11 @@ import {
startFreshThread,
} from './workflowAgentChatTools.js'
+// The Host default gpt-5.4-mini skipped requested tool calls (it answered a second shell
+// request from earlier outputs and ran the shell but never the requested http_request), so
+// no approval card could appear. The composer menu lists only the Host provider's allowlisted
+// models (openai on the branch profile), so the lane selects the larger gpt-5.4 from it.
+const CHAT_MODEL = 'gpt-5.4'
const RUN_ID = randomUUID().replace(/-/g, '').slice(0, 8)
const TURN_TIMEOUT = 300_000
const CARD_TIMEOUT = 180_000
@@ -59,8 +64,23 @@ function markerFor(testInfo: TestInfo, step: string): string {
return `STA_${RUN_ID}_${testId}_${step}`
}
+// `date` makes the output unpredictable: with a bare `printf <marker>` the model answered the
+// second task from the first task's outputs without calling shell_exec, so no approval card
+// could appear. Reading /proc/sys/kernel/random/uuid instead made the model refuse the tool.
function shellCommand(marker: string): string {
- return `printf ${marker}`
+ return `echo ${marker}; date`
+}
+
+// Same visible path as qa-recorder-image-capabilities.spec.ts: model chip -> menu -> row.
+async function selectChatModel(page: Page, model: string): Promise<void> {
+ const chip = page.getByTestId('selected-chat-model')
+ await expect(chip).toBeVisible({ timeout: 30_000 })
+ await chip.click()
+ await expect(page.getByRole('menu', { name: 'Select model' })).toBeVisible({ timeout: 20_000 })
+ const row = page.getByTestId(`model-option-${model}`)
+ await expect(row, `${model} must be offered by the chatllm catalog`).toHaveCount(1)
+ await row.click()
+ await expect(chip).toHaveAttribute('data-model-id', model)
}
function escapeRegExp(value: string): string {
@@ -315,6 +335,7 @@ test.describe.serial('per-task tool approval scope', () => {
await test.step('end user opens a fresh chatllm thread', async () => {
await enterChatllmChat(appPage)
await startFreshThread(appPage)
+ await selectChatModel(appPage, CHAT_MODEL)
await installCardLedger(appPage)
expect(await readCardLedger(appPage)).toEqual([])
})
@@ -420,6 +441,7 @@ test.describe.serial('per-task tool approval scope', () => {
await test.step('end user opens a fresh chatllm thread', async () => {
await enterChatllmChat(appPage)
await startFreshThread(appPage)
+ await selectChatModel(appPage, CHAT_MODEL)
await installCardLedger(appPage)
expect(await readCardLedger(appPage)).toEqual([])
})
diff --git a/desktop-app/test/e2e-playwright/workflowAgentChatTools.ts b/desktop-app/test/e2e-playwright/workflowAgentChatTools.ts
index 952ec4fa3..fb26a6c72 100644
--- a/desktop-app/test/e2e-playwright/workflowAgentChatTools.ts
+++ b/desktop-app/test/e2e-playwright/workflowAgentChatTools.ts
@@ -97,10 +97,6 @@ export async function captureE2eEvidence(
await testInfo.attach(fileName, { path: screenshotPath, contentType: 'image/png' })
}
-function escapeRegExp(value: string): string {
- return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-}
-
function sqlLiteral(value: string): string {
return `'${value.replace(/'/g, "''")}'`
}
@@ -286,11 +282,12 @@ export async function enterChatllmChat(page: Page): Promise<void> {
export async function startFreshThread(page: Page): Promise<void> {
await openAgentsPage(page)
await humanRecordedPause([320, 780])
+ // The fleet table labels its controls with the agent display name (e.g. "chatLLM",
+ // FleetBoard.tsx), not the host ref, so find the menu inside the row that shows the ref.
const moreActions = page
- .getByRole('button', {
- name: new RegExp(`^More actions for ${escapeRegExp(CHATLLM_HOST_REF)}$`),
- })
+ .locator('.agents-table-row-clickable', { hasText: CHATLLM_HOST_REF })
.first()
+ .getByRole('button', { name: /^More actions for / })
await expect(moreActions).toBeVisible({ timeout: 30_000 })
await humanClick(moreActions, { afterMs: [260, 620] })
const newChat = page.getByRole('button', { name: /^New chat$/ }).first()
diff --git a/desktop-app/test/e2e-playwright/workflowUi.ts b/desktop-app/test/e2e-playwright/workflowUi.ts
index caaab68d9..22f7cc6e8 100644
--- a/desktop-app/test/e2e-playwright/workflowUi.ts
+++ b/desktop-app/test/e2e-playwright/workflowUi.ts
@@ -282,12 +282,7 @@ export async function loginAs(
password = E2E_DESKTOP_PASSWORD
): Promise<{ userId: string; userToken: string }> {
const userId = seedPasswordForEmail(email, password)
- await waitForPasswordLoginReady(email, password)
- const loginRes = await apiRequest(
- 'POST',
- `${EXT_API}/api/v1/auth/password-login`,
- JSON.stringify({ email, password })
- )
+ const loginRes = await pacedPasswordLogin(email, password)
if (loginRes.status !== 200) {
throw new Error(`password-login failed for ${email}: HTTP ${loginRes.status} ${loginRes.body}`)
}
@@ -400,26 +395,49 @@ export function seedDesktopPasswordLogin(
}
}
-async function waitForPasswordLoginReady(
+// control-api admits one password verification per PASSWORD_ADMISSION_POLICY.paceMs
+// (7.5 s) for the whole server, and charges the identity's 5-per-15-minutes budget
+// before the pace check, so a paced 429 still spends an attempt. Space every password
+// attempt from this worker (API and Desktop sign-in) by the pace. The worker loads after
+// global setup's preflight login, so module load is a safe bound for that attempt.
+const PASSWORD_PACE_MS = 8_000
+// One paced retry at most; a longer Retry-After means the identity budget is spent.
+const MAX_PACE_RETRY_SECONDS = 15
+let lastPasswordAttemptAt = Date.now()
+
+export async function awaitPasswordPace(): Promise<void> {
+ const waitMs = lastPasswordAttemptAt + PASSWORD_PACE_MS - Date.now()
+ if (waitMs > 0) await new Promise(resolve => setTimeout(resolve, waitMs))
+ lastPasswordAttemptAt = Date.now()
+}
+
+async function pacedPasswordLogin(
email: string,
- password = E2E_DESKTOP_PASSWORD
-): Promise<void> {
- let lastStatus = 0
- let lastBody = ''
- for (let attempt = 0; attempt < 10; attempt += 1) {
- const res = await apiRequest(
+ password: string
+): Promise<{ status: number; body: string }> {
+ const attempt = async () => {
+ await awaitPasswordPace()
+ return apiRequest(
'POST',
`${EXT_API}/api/v1/auth/password-login`,
JSON.stringify({ email, password })
)
- lastStatus = res.status
- lastBody = res.body
- if (res.status === 200) return
- await new Promise(resolve => setTimeout(resolve, 250))
}
- throw new Error(
- `seeded Desktop password login was not accepted for ${email}: HTTP ${lastStatus} ${lastBody}`
- )
+ const first = await attempt()
+ if (first.status !== 429) return first
+ let retryAfterSeconds = Number.NaN
+ try {
+ retryAfterSeconds = Number(JSON.parse(first.body).retryAfterSeconds)
+ } catch {
+ throw new Error(`password-login for ${email} returned HTTP 429 without JSON: ${first.body}`)
+ }
+ if (!Number.isFinite(retryAfterSeconds) || retryAfterSeconds > MAX_PACE_RETRY_SECONDS) {
+ throw new Error(
+ `password-login for ${email} is rate limited beyond the pace (identity budget spent?): HTTP 429 ${first.body}`
+ )
+ }
+ lastPasswordAttemptAt = Date.now() + retryAfterSeconds * 1000 - PASSWORD_PACE_MS
+ return attempt()
}
export function seedAllowlist(userId: string, recipeName: string): void {