Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): stop the approved-tools lane inheriting sessions and racing rollouts

Alfredo Lopezf4651eea1813
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
index 0cf46f97a..fd8eafd20 100644
--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
@@ -10,6 +10,7 @@
* providers are simulated in deterministic mode.
*/
import { type Page, expect, test } from '@playwright/test'
+import { readAgentDeploymentGeneration, waitForAgentRollout } from '../helpers/agent-rollout'
import {
type Scenario,
type UpstreamEvidence,
@@ -22,6 +23,7 @@ import {
} from '../helpers/approved-tools-scenarios'
import { prepareSubscriptionVisible } from '../helpers/approved-tools-subscription'
import { workflowJourney } from '../helpers/approved-tools-workflow'
+import { expectSignedOutLaunch, signOutDesktop } from '../helpers/desktop-session'
import { launchDesktopApp } from '../helpers/launch-desktop'
import { loginControlUiVisible } from '../helpers/visible-login'
import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'
@@ -105,7 +107,7 @@ async function newAgentResponses(page: Page) {
}
async function openAgentChat(desktop: Page, scenario: Scenario) {
- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
+ await expectSignedOutLaunch(desktop)
await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))
await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))
await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()
@@ -160,7 +162,7 @@ test('authenticated user without agent access cannot select the protected agents
let journeyPassed = false
try {
const page = await app.firstWindow()
- await expect(page.getByLabel('Email', { exact: true })).toBeVisible()
+ await expectSignedOutLaunch(page)
await page
.getByLabel('Email', { exact: true })
.fill(required('APPROVED_TOOLS_UNAUTHORIZED_EMAIL'))
@@ -233,8 +235,12 @@ for (const scenario of cases) {
page.getByRole('heading', { name: `Agent: ${scenario.agentDisplayName}`, exact: true })
).toBeVisible()
})
+ // Recorded before the save so the wait below can prove the rollout it
+ // waits for is the one this test caused, not one that was already done.
+ let rolloutBaseline = 0
await test.step('Select the prepared subscription and persist its model binding', async () => {
const model = new AgentModelPage(page)
+ rolloutBaseline = readAgentDeploymentGeneration(scenario.agentName)
await model.openEditor()
await model.chooseSubscription(scenario.subscriptionName)
await page.getByLabel('Current model', { exact: true }).click()
@@ -264,6 +270,9 @@ for (const scenario of cases) {
await dialog.getByRole('option', { name: scenario.connectorName, exact: true }).click()
await saveConnector(page, scenario, false)
})
+ await test.step('Wait for the agent rollout the model binding triggered', async () => {
+ await waitForAgentRollout(scenario.agentName, rolloutBaseline)
+ })
const app = await launchDesktopApp()
const corpusStarted = Date.now()
const completedTasks: string[] = []
@@ -536,6 +545,11 @@ for (const scenario of cases) {
} catch {
cleanupFailures.push('measurement unavailable')
}
+ try {
+ await signOutDesktop(await app.firstWindow())
+ } catch {
+ cleanupFailures.push('Desktop sign-out failed; the session stays in the Keychain')
+ }
try {
await app.close()
} catch {
@@ -603,12 +617,25 @@ if (mode === 'deterministic') {
})
journeyPassed = true
} finally {
+ // Sign out before closing: the session lives in the host Keychain, so a
+ // window closed while authenticated signs the next launch in and that
+ // launch never renders a login form.
+ let cleanupError: string | undefined
+ try {
+ await signOutDesktop(await app.firstWindow())
+ } catch {
+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'
+ }
try {
await app.close()
} catch {
- if (journeyPassed) throw new Error('Desktop cleanup failed')
- // Preserve the original assertion failure when cleanup also fails.
+ cleanupError = cleanupError
+ ? `${cleanupError}; Desktop cleanup failed`
+ : 'Desktop cleanup failed'
}
+ // Raise cleanup problems only when the journey passed; otherwise the
+ // original assertion failure is the one worth reporting.
+ if (cleanupError && journeyPassed) throw new Error(cleanupError)
}
})
@@ -663,12 +690,25 @@ if (mode === 'deterministic') {
})
journeyPassed = true
} finally {
+ // Sign out before closing: the session lives in the host Keychain, so a
+ // window closed while authenticated signs the next launch in and that
+ // launch never renders a login form.
+ let cleanupError: string | undefined
+ try {
+ await signOutDesktop(await app.firstWindow())
+ } catch {
+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'
+ }
try {
await app.close()
} catch {
- if (journeyPassed) throw new Error('Desktop cleanup failed')
- // Preserve the original assertion failure when cleanup also fails.
+ cleanupError = cleanupError
+ ? `${cleanupError}; Desktop cleanup failed`
+ : 'Desktop cleanup failed'
}
+ // Raise cleanup problems only when the journey passed; otherwise the
+ // original assertion failure is the one worth reporting.
+ if (cleanupError && journeyPassed) throw new Error(cleanupError)
}
})
}
diff --git a/tests/e2e/playwright/helpers/agent-rollout.ts b/tests/e2e/playwright/helpers/agent-rollout.ts
new file mode 100644
index 000000000..b22292e4d
--- /dev/null
+++ b/tests/e2e/playwright/helpers/agent-rollout.ts
@@ -0,0 +1,106 @@
+import { execFileSync } from 'node:child_process'
+import { required } from './approved-tools-scenarios'
+
+// Saving a model binding or an approval map changes the agent's contract, which
+// makes HCC rotate the runtime token and roll the Deployment
+// (`host-context-controller/src/hostReconciler.ts:1402`, `rolloutRequired:
+// true`). While that rollout is in flight the old pod is terminating and
+// rpc-proxy answers `list-models` through its legacy error path
+// (`rpc-proxy/src/routes/rpc.ts:1056-1119`), so the Desktop model store lands in
+// `loadFailed` and renders "Models unavailable" instead of the model menu. A
+// test that saves and immediately launches the Desktop app is racing that
+// rollout.
+//
+// The fixture creates every agent Host in `mcp-host`
+// (`scripts/e2e/prepare-codex-approved-tools.mjs:230`) and HCC names the
+// Deployment after the Host in the Host's own namespace (`hostReconciler.ts:3526`),
+// so the object is `mcp-host/<agentName>` by construction, not by convention.
+const NAMESPACE = 'mcp-host'
+const POLL_INTERVAL_MS = 2_000
+
+type DeploymentSnapshot = {
+ generation: number
+ observedGeneration: number
+ replicas: number
+ updatedReplicas: number
+ readyReplicas: number
+ availableReplicas: number
+ unavailableReplicas: number
+}
+
+function readAgentDeployment(agentName: string): DeploymentSnapshot {
+ const raw = execFileSync(
+ 'kubectl',
+ [
+ '--context',
+ required('MINIKUBE_PROFILE'),
+ '--request-timeout=30s',
+ '-n',
+ NAMESPACE,
+ 'get',
+ `deployment/${agentName}`,
+ '-o',
+ 'json',
+ ],
+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
+ )
+ const deployment = JSON.parse(raw) as {
+ metadata?: { generation?: number }
+ spec?: { replicas?: number }
+ status?: {
+ observedGeneration?: number
+ updatedReplicas?: number
+ readyReplicas?: number
+ availableReplicas?: number
+ unavailableReplicas?: number
+ }
+ }
+ return {
+ generation: deployment.metadata?.generation ?? 0,
+ observedGeneration: deployment.status?.observedGeneration ?? 0,
+ replicas: deployment.spec?.replicas ?? 0,
+ updatedReplicas: deployment.status?.updatedReplicas ?? 0,
+ readyReplicas: deployment.status?.readyReplicas ?? 0,
+ availableReplicas: deployment.status?.availableReplicas ?? 0,
+ unavailableReplicas: deployment.status?.unavailableReplicas ?? 0,
+ }
+}
+
+export function readAgentDeploymentGeneration(agentName: string): number {
+ return readAgentDeployment(agentName).generation
+}
+
+export async function waitForAgentRollout(
+ agentName: string,
+ baselineGeneration: number,
+ timeoutMs = 120_000
+): Promise<void> {
+ const deadline = Date.now() + timeoutMs
+ let convergedGeneration: number | undefined
+ let last: DeploymentSnapshot | undefined
+ while (Date.now() < deadline) {
+ const snapshot = readAgentDeployment(agentName)
+ last = snapshot
+ const converged =
+ // The generation moving past the baseline is the liveness witness: if the
+ // save triggered no rollout at all, this waits and then fails, rather than
+ // returning instantly on a Deployment that was already converged.
+ snapshot.generation > baselineGeneration &&
+ snapshot.observedGeneration >= snapshot.generation &&
+ snapshot.replicas > 0 &&
+ snapshot.updatedReplicas === snapshot.replicas &&
+ snapshot.readyReplicas === snapshot.replicas &&
+ snapshot.availableReplicas === snapshot.replicas &&
+ snapshot.unavailableReplicas === 0
+ // Two consecutive polls on the same generation. A second spec change landing
+ // late bumps the generation again, and a single converged reading would
+ // otherwise describe a rollout that is already obsolete.
+ if (converged && convergedGeneration === snapshot.generation) return
+ convergedGeneration = converged ? snapshot.generation : undefined
+ await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS))
+ }
+ throw new Error(
+ `Agent ${agentName} did not converge within ${timeoutMs} ms of generation ${baselineGeneration}: ` +
+ JSON.stringify(last)
+ )
+}
diff --git a/tests/e2e/playwright/helpers/approved-tools-workflow.ts b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
index c84781de1..c41a3ab4e 100644
--- a/tests/e2e/playwright/helpers/approved-tools-workflow.ts
+++ b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
@@ -6,6 +6,7 @@
*/
import { type Page, type TestInfo, expect, test } from '@playwright/test'
import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'
+import { readAgentDeploymentGeneration, waitForAgentRollout } from './agent-rollout'
import {
type Scenario,
browserApiPath,
@@ -16,6 +17,7 @@ import {
scenarios,
} from './approved-tools-scenarios'
import { prepareSubscriptionVisible } from './approved-tools-subscription'
+import { expectSignedOutLaunch, signOutDesktop } from './desktop-session'
import { launchDesktopApp } from './launch-desktop'
import { loginControlUiVisible } from './visible-login'
@@ -94,6 +96,9 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {
await expect(page.getByRole('button', { name: 'Edit', exact: true })).toBeVisible()
})
+ // The approval map is part of the agent contract, so saving it rolls the
+ // Deployment exactly as the model binding does. Recorded before the save.
+ const rolloutBaseline = readAgentDeploymentGeneration(scenario.agentName)
await test.step('Require approval for the real native workflow trigger through Advanced settings', async () => {
await page.getByRole('tab', { name: 'Advanced', exact: true }).click()
await expect(page).toHaveURL(new RegExp(`/agents/${scenario.agentName}/advanced$`))
@@ -116,14 +121,32 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {
expect(response.ok()).toBe(true)
expect((await response.json()).spec.approval.tools.workflow_trigger).toBe(true)
await expect(advanced.getByRole('button', { name: 'Save', exact: true })).toBeDisabled()
+ // Saving reloads the agent: `persistApprovalTools`
+ // (`control-ui/app/hosts/[name]/page.tsx:1084-1112`) calls `loadData('none')`,
+ // which sets `initialLoading`, so `HostApprovalSection` unmounts and remounts
+ // with its conditional-tools block collapsed
+ // (`HostApprovalSection/index.tsx:187`) and the custom rows unrendered
+ // (`:260`). The row is hidden, not missing, and asserting the collapsed
+ // state first is what tells those two apart.
+ const conditionalTools = advanced.getByRole('button', {
+ name: /Advanced: conditional tools/,
+ })
+ await expect(conditionalTools).toHaveAttribute('aria-expanded', 'false')
+ await conditionalTools.click()
+ await expect(conditionalTools).toHaveAttribute('aria-expanded', 'true')
await expect(advanced.getByText('workflow_trigger', { exact: true })).toBeVisible()
})
+ await test.step('Wait for the agent rollout the approval change triggered', async () => {
+ await waitForAgentRollout(scenario.agentName, rolloutBaseline)
+ })
+
const app = await launchDesktopApp()
+ let journeyPassed = false
try {
const desktop = await app.firstWindow()
await test.step('Sign in to Desktop, select the same agent and subscription model', async () => {
- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
+ await expectSignedOutLaunch(desktop)
await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))
await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))
await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()
@@ -221,7 +244,23 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {
contentType: 'application/json',
})
})
+ journeyPassed = true
} finally {
- await app.close()
+ // Sign out before closing, or this journey's session is inherited by the
+ // next Electron test and its login form never renders.
+ let cleanupError: string | undefined
+ try {
+ await signOutDesktop(await app.firstWindow())
+ } catch {
+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'
+ }
+ try {
+ await app.close()
+ } catch {
+ cleanupError = cleanupError
+ ? `${cleanupError}; Desktop cleanup failed`
+ : 'Desktop cleanup failed'
+ }
+ if (cleanupError && journeyPassed) throw new Error(cleanupError)
}
}
diff --git a/tests/e2e/playwright/helpers/desktop-session.ts b/tests/e2e/playwright/helpers/desktop-session.ts
new file mode 100644
index 000000000..810a2c2fc
--- /dev/null
+++ b/tests/e2e/playwright/helpers/desktop-session.ts
@@ -0,0 +1,39 @@
+import { type Page, expect } from '@playwright/test'
+
+// The Desktop session token lives in the macOS Keychain under the service
+// `Evenfire` and the account `session-token::<envKey>`
+// (`desktop-app/src/tokenStore.ts`), and `envKey` is derived from the REST and
+// RPC origins alone (`desktop-app/src/config.ts:803-823`). Every launch in this
+// lane uses the same origins, so every launch resolves the same account, while
+// `launchDesktopApp` isolates only `--user-data-dir`, which the Keychain
+// ignores. A window closed while signed in therefore hands its session to the
+// next launch, and `restoreSavedSessionOnce`
+// (`desktop-app/src/appService.ts:1369-1440`) starts that launch authenticated,
+// so its login form never mounts. The leak also crosses runs, because the ports
+// are stable.
+export async function signOutDesktop(desktop: Page): Promise<void> {
+ await desktop.getByTestId('nav-settings-menu').click()
+ await desktop.getByTestId('logout-btn').click()
+ // Logout clears the Keychain account (`desktop-app/src/appService.ts:1919`),
+ // and the login form is the witness that it did: the renderer mounts
+ // `AuthPage` only when no session remains.
+ await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()
+}
+
+// A launch that inherits a session is a cleanup failure in an earlier test, not
+// a condition to recover from. Signing out here would make the lane pass while
+// hiding which test leaked, so this names the cause and fails.
+export async function expectSignedOutLaunch(desktop: Page): Promise<void> {
+ const loginForm = desktop.getByLabel('Email', { exact: true })
+ const signedIn = desktop.getByTestId('nav-settings-menu')
+ // Wait for whichever of the two mounts first. The non-retrying read below is
+ // only sound once this retrying assertion has settled.
+ await expect(loginForm.or(signedIn)).toBeVisible()
+ if (await signedIn.isVisible())
+ throw new Error(
+ 'Desktop launched already signed in: an earlier Electron test left its session in the macOS ' +
+ 'Keychain (service Evenfire, account session-token::<envKey>) instead of signing out before ' +
+ 'closing its window.'
+ )
+ await expect(loginForm).toBeVisible()
+}
diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.json
index e36ea3dd5..5ea3f49e9 100644
--- a/tests/e2e/playwright/package-lock.json
+++ b/tests/e2e/playwright/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.58",
+ "version": "0.1.59",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-playwright-tests",
- "version": "0.1.58",
+ "version": "0.1.59",
"devDependencies": {
"@playwright/test": "^1.50.0",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.json
index 0e0c1a9f3..0ce24c66d 100644
--- a/tests/e2e/playwright/package.json
+++ b/tests/e2e/playwright/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.58",
+ "version": "0.1.59",
"private": true,
"description": "Playwright E2E tests for Control UI and Desktop App",
"scripts": {