Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): stop the approved-tools lane inheriting sessions and racing rollouts
Alfredo Lopezf4651eea1813
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.tsindex 0cf46f97a..fd8eafd20 100644--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts@@ -10,6 +10,7 @@* providers are simulated in deterministic mode.*/import { type Page, expect, test } from '@playwright/test'+import { readAgentDeploymentGeneration, waitForAgentRollout } from '../helpers/agent-rollout'import {type Scenario,type UpstreamEvidence,@@ -22,6 +23,7 @@ import {} from '../helpers/approved-tools-scenarios'import { prepareSubscriptionVisible } from '../helpers/approved-tools-subscription'import { workflowJourney } from '../helpers/approved-tools-workflow'+import { expectSignedOutLaunch, signOutDesktop } from '../helpers/desktop-session'import { launchDesktopApp } from '../helpers/launch-desktop'import { loginControlUiVisible } from '../helpers/visible-login'import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'@@ -105,7 +107,7 @@ async function newAgentResponses(page: Page) {}async function openAgentChat(desktop: Page, scenario: Scenario) {- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()+ await expectSignedOutLaunch(desktop)await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()@@ -160,7 +162,7 @@ test('authenticated user without agent access cannot select the protected agentslet journeyPassed = falsetry {const page = await app.firstWindow()- await expect(page.getByLabel('Email', { exact: true })).toBeVisible()+ await expectSignedOutLaunch(page)await page.getByLabel('Email', { exact: true }).fill(required('APPROVED_TOOLS_UNAUTHORIZED_EMAIL'))@@ -233,8 +235,12 @@ for (const scenario of cases) {page.getByRole('heading', { name: `Agent: ${scenario.agentDisplayName}`, exact: true })).toBeVisible()})+ // Recorded before the save so the wait below can prove the rollout it+ // waits for is the one this test caused, not one that was already done.+ let rolloutBaseline = 0await test.step('Select the prepared subscription and persist its model binding', async () => {const model = new AgentModelPage(page)+ rolloutBaseline = readAgentDeploymentGeneration(scenario.agentName)await model.openEditor()await model.chooseSubscription(scenario.subscriptionName)await page.getByLabel('Current model', { exact: true }).click()@@ -264,6 +270,9 @@ for (const scenario of cases) {await dialog.getByRole('option', { name: scenario.connectorName, exact: true }).click()await saveConnector(page, scenario, false)})+ await test.step('Wait for the agent rollout the model binding triggered', async () => {+ await waitForAgentRollout(scenario.agentName, rolloutBaseline)+ })const app = await launchDesktopApp()const corpusStarted = Date.now()const completedTasks: string[] = []@@ -536,6 +545,11 @@ for (const scenario of cases) {} catch {cleanupFailures.push('measurement unavailable')}+ try {+ await signOutDesktop(await app.firstWindow())+ } catch {+ cleanupFailures.push('Desktop sign-out failed; the session stays in the Keychain')+ }try {await app.close()} catch {@@ -603,12 +617,25 @@ if (mode === 'deterministic') {})journeyPassed = true} finally {+ // Sign out before closing: the session lives in the host Keychain, so a+ // window closed while authenticated signs the next launch in and that+ // launch never renders a login form.+ let cleanupError: string | undefined+ try {+ await signOutDesktop(await app.firstWindow())+ } catch {+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'+ }try {await app.close()} catch {- if (journeyPassed) throw new Error('Desktop cleanup failed')- // Preserve the original assertion failure when cleanup also fails.+ cleanupError = cleanupError+ ? `${cleanupError}; Desktop cleanup failed`+ : 'Desktop cleanup failed'}+ // Raise cleanup problems only when the journey passed; otherwise the+ // original assertion failure is the one worth reporting.+ if (cleanupError && journeyPassed) throw new Error(cleanupError)}})@@ -663,12 +690,25 @@ if (mode === 'deterministic') {})journeyPassed = true} finally {+ // Sign out before closing: the session lives in the host Keychain, so a+ // window closed while authenticated signs the next launch in and that+ // launch never renders a login form.+ let cleanupError: string | undefined+ try {+ await signOutDesktop(await app.firstWindow())+ } catch {+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'+ }try {await app.close()} catch {- if (journeyPassed) throw new Error('Desktop cleanup failed')- // Preserve the original assertion failure when cleanup also fails.+ cleanupError = cleanupError+ ? `${cleanupError}; Desktop cleanup failed`+ : 'Desktop cleanup failed'}+ // Raise cleanup problems only when the journey passed; otherwise the+ // original assertion failure is the one worth reporting.+ if (cleanupError && journeyPassed) throw new Error(cleanupError)}})}diff --git a/tests/e2e/playwright/helpers/agent-rollout.ts b/tests/e2e/playwright/helpers/agent-rollout.tsnew file mode 100644index 000000000..b22292e4d--- /dev/null+++ b/tests/e2e/playwright/helpers/agent-rollout.ts@@ -0,0 +1,106 @@+import { execFileSync } from 'node:child_process'+import { required } from './approved-tools-scenarios'++// Saving a model binding or an approval map changes the agent's contract, which+// makes HCC rotate the runtime token and roll the Deployment+// (`host-context-controller/src/hostReconciler.ts:1402`, `rolloutRequired:+// true`). While that rollout is in flight the old pod is terminating and+// rpc-proxy answers `list-models` through its legacy error path+// (`rpc-proxy/src/routes/rpc.ts:1056-1119`), so the Desktop model store lands in+// `loadFailed` and renders "Models unavailable" instead of the model menu. A+// test that saves and immediately launches the Desktop app is racing that+// rollout.+//+// The fixture creates every agent Host in `mcp-host`+// (`scripts/e2e/prepare-codex-approved-tools.mjs:230`) and HCC names the+// Deployment after the Host in the Host's own namespace (`hostReconciler.ts:3526`),+// so the object is `mcp-host/<agentName>` by construction, not by convention.+const NAMESPACE = 'mcp-host'+const POLL_INTERVAL_MS = 2_000++type DeploymentSnapshot = {+ generation: number+ observedGeneration: number+ replicas: number+ updatedReplicas: number+ readyReplicas: number+ availableReplicas: number+ unavailableReplicas: number+}++function readAgentDeployment(agentName: string): DeploymentSnapshot {+ const raw = execFileSync(+ 'kubectl',+ [+ '--context',+ required('MINIKUBE_PROFILE'),+ '--request-timeout=30s',+ '-n',+ NAMESPACE,+ 'get',+ `deployment/${agentName}`,+ '-o',+ 'json',+ ],+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }+ )+ const deployment = JSON.parse(raw) as {+ metadata?: { generation?: number }+ spec?: { replicas?: number }+ status?: {+ observedGeneration?: number+ updatedReplicas?: number+ readyReplicas?: number+ availableReplicas?: number+ unavailableReplicas?: number+ }+ }+ return {+ generation: deployment.metadata?.generation ?? 0,+ observedGeneration: deployment.status?.observedGeneration ?? 0,+ replicas: deployment.spec?.replicas ?? 0,+ updatedReplicas: deployment.status?.updatedReplicas ?? 0,+ readyReplicas: deployment.status?.readyReplicas ?? 0,+ availableReplicas: deployment.status?.availableReplicas ?? 0,+ unavailableReplicas: deployment.status?.unavailableReplicas ?? 0,+ }+}++export function readAgentDeploymentGeneration(agentName: string): number {+ return readAgentDeployment(agentName).generation+}++export async function waitForAgentRollout(+ agentName: string,+ baselineGeneration: number,+ timeoutMs = 120_000+): Promise<void> {+ const deadline = Date.now() + timeoutMs+ let convergedGeneration: number | undefined+ let last: DeploymentSnapshot | undefined+ while (Date.now() < deadline) {+ const snapshot = readAgentDeployment(agentName)+ last = snapshot+ const converged =+ // The generation moving past the baseline is the liveness witness: if the+ // save triggered no rollout at all, this waits and then fails, rather than+ // returning instantly on a Deployment that was already converged.+ snapshot.generation > baselineGeneration &&+ snapshot.observedGeneration >= snapshot.generation &&+ snapshot.replicas > 0 &&+ snapshot.updatedReplicas === snapshot.replicas &&+ snapshot.readyReplicas === snapshot.replicas &&+ snapshot.availableReplicas === snapshot.replicas &&+ snapshot.unavailableReplicas === 0+ // Two consecutive polls on the same generation. A second spec change landing+ // late bumps the generation again, and a single converged reading would+ // otherwise describe a rollout that is already obsolete.+ if (converged && convergedGeneration === snapshot.generation) return+ convergedGeneration = converged ? snapshot.generation : undefined+ await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS))+ }+ throw new Error(+ `Agent ${agentName} did not converge within ${timeoutMs} ms of generation ${baselineGeneration}: ` ++ JSON.stringify(last)+ )+}diff --git a/tests/e2e/playwright/helpers/approved-tools-workflow.ts b/tests/e2e/playwright/helpers/approved-tools-workflow.tsindex c84781de1..c41a3ab4e 100644--- a/tests/e2e/playwright/helpers/approved-tools-workflow.ts+++ b/tests/e2e/playwright/helpers/approved-tools-workflow.ts@@ -6,6 +6,7 @@*/import { type Page, type TestInfo, expect, test } from '@playwright/test'import { AgentListPage, AgentModelPage, ControlUiShell } from '../pages/codex-subscription'+import { readAgentDeploymentGeneration, waitForAgentRollout } from './agent-rollout'import {type Scenario,browserApiPath,@@ -16,6 +17,7 @@ import {scenarios,} from './approved-tools-scenarios'import { prepareSubscriptionVisible } from './approved-tools-subscription'+import { expectSignedOutLaunch, signOutDesktop } from './desktop-session'import { launchDesktopApp } from './launch-desktop'import { loginControlUiVisible } from './visible-login'@@ -94,6 +96,9 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {await expect(page.getByRole('button', { name: 'Edit', exact: true })).toBeVisible()})+ // The approval map is part of the agent contract, so saving it rolls the+ // Deployment exactly as the model binding does. Recorded before the save.+ const rolloutBaseline = readAgentDeploymentGeneration(scenario.agentName)await test.step('Require approval for the real native workflow trigger through Advanced settings', async () => {await page.getByRole('tab', { name: 'Advanced', exact: true }).click()await expect(page).toHaveURL(new RegExp(`/agents/${scenario.agentName}/advanced$`))@@ -116,14 +121,32 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {expect(response.ok()).toBe(true)expect((await response.json()).spec.approval.tools.workflow_trigger).toBe(true)await expect(advanced.getByRole('button', { name: 'Save', exact: true })).toBeDisabled()+ // Saving reloads the agent: `persistApprovalTools`+ // (`control-ui/app/hosts/[name]/page.tsx:1084-1112`) calls `loadData('none')`,+ // which sets `initialLoading`, so `HostApprovalSection` unmounts and remounts+ // with its conditional-tools block collapsed+ // (`HostApprovalSection/index.tsx:187`) and the custom rows unrendered+ // (`:260`). The row is hidden, not missing, and asserting the collapsed+ // state first is what tells those two apart.+ const conditionalTools = advanced.getByRole('button', {+ name: /Advanced: conditional tools/,+ })+ await expect(conditionalTools).toHaveAttribute('aria-expanded', 'false')+ await conditionalTools.click()+ await expect(conditionalTools).toHaveAttribute('aria-expanded', 'true')await expect(advanced.getByText('workflow_trigger', { exact: true })).toBeVisible()})+ await test.step('Wait for the agent rollout the approval change triggered', async () => {+ await waitForAgentRollout(scenario.agentName, rolloutBaseline)+ })+const app = await launchDesktopApp()+ let journeyPassed = falsetry {const desktop = await app.firstWindow()await test.step('Sign in to Desktop, select the same agent and subscription model', async () => {- await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()+ await expectSignedOutLaunch(desktop)await desktop.getByLabel('Email', { exact: true }).fill(required('TEST_USER_EMAIL'))await desktop.getByLabel('Password', { exact: true }).fill(required('TEST_USER_PASSWORD'))await desktop.getByRole('button', { name: 'Sign in', exact: true }).click()@@ -221,7 +244,23 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {contentType: 'application/json',})})+ journeyPassed = true} finally {- await app.close()+ // Sign out before closing, or this journey's session is inherited by the+ // next Electron test and its login form never renders.+ let cleanupError: string | undefined+ try {+ await signOutDesktop(await app.firstWindow())+ } catch {+ cleanupError = 'Desktop sign-out failed; the session stays in the Keychain'+ }+ try {+ await app.close()+ } catch {+ cleanupError = cleanupError+ ? `${cleanupError}; Desktop cleanup failed`+ : 'Desktop cleanup failed'+ }+ if (cleanupError && journeyPassed) throw new Error(cleanupError)}}diff --git a/tests/e2e/playwright/helpers/desktop-session.ts b/tests/e2e/playwright/helpers/desktop-session.tsnew file mode 100644index 000000000..810a2c2fc--- /dev/null+++ b/tests/e2e/playwright/helpers/desktop-session.ts@@ -0,0 +1,39 @@+import { type Page, expect } from '@playwright/test'++// The Desktop session token lives in the macOS Keychain under the service+// `Evenfire` and the account `session-token::<envKey>`+// (`desktop-app/src/tokenStore.ts`), and `envKey` is derived from the REST and+// RPC origins alone (`desktop-app/src/config.ts:803-823`). Every launch in this+// lane uses the same origins, so every launch resolves the same account, while+// `launchDesktopApp` isolates only `--user-data-dir`, which the Keychain+// ignores. A window closed while signed in therefore hands its session to the+// next launch, and `restoreSavedSessionOnce`+// (`desktop-app/src/appService.ts:1369-1440`) starts that launch authenticated,+// so its login form never mounts. The leak also crosses runs, because the ports+// are stable.+export async function signOutDesktop(desktop: Page): Promise<void> {+ await desktop.getByTestId('nav-settings-menu').click()+ await desktop.getByTestId('logout-btn').click()+ // Logout clears the Keychain account (`desktop-app/src/appService.ts:1919`),+ // and the login form is the witness that it did: the renderer mounts+ // `AuthPage` only when no session remains.+ await expect(desktop.getByLabel('Email', { exact: true })).toBeVisible()+}++// A launch that inherits a session is a cleanup failure in an earlier test, not+// a condition to recover from. Signing out here would make the lane pass while+// hiding which test leaked, so this names the cause and fails.+export async function expectSignedOutLaunch(desktop: Page): Promise<void> {+ const loginForm = desktop.getByLabel('Email', { exact: true })+ const signedIn = desktop.getByTestId('nav-settings-menu')+ // Wait for whichever of the two mounts first. The non-retrying read below is+ // only sound once this retrying assertion has settled.+ await expect(loginForm.or(signedIn)).toBeVisible()+ if (await signedIn.isVisible())+ throw new Error(+ 'Desktop launched already signed in: an earlier Electron test left its session in the macOS ' ++ 'Keychain (service Evenfire, account session-token::<envKey>) instead of signing out before ' ++ 'closing its window.'+ )+ await expect(loginForm).toBeVisible()+}diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex e36ea3dd5..5ea3f49e9 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.58",+ "version": "0.1.59","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.58",+ "version": "0.1.59","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0",diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 0e0c1a9f3..0ce24c66d 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.58",+ "version": "0.1.59","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {