Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(desktop): make the chat-open A/B lane fail closed and measure what it claims
Alfredo Lopezd799cd045a7c
diff --git a/desktop-app/scripts/bench-chat-open/fixtures.mjs b/desktop-app/scripts/bench-chat-open/fixtures.mjsindex baee73f2e..dc4b8033e 100644--- a/desktop-app/scripts/bench-chat-open/fixtures.mjs+++ b/desktop-app/scripts/bench-chat-open/fixtures.mjs@@ -1,29 +1,24 @@import assert from 'node:assert/strict'import { promises as fs } from 'node:fs'import { join, sep } from 'node:path'+import {+ AUTHORITY_SCOPE,+ FULL_CHAT,+ PARTIAL_MESSAGES,+ TARGET_AGENT,+} from '../../test/e2e-playwright/helpers/chatCacheFixture.mjs'-export const AUTHORITY_SCOPE = Object.freeze({- environmentKey: 'benchmark-environment',- userId: 'benchmark-user',- teamId: 'benchmark-team',-})-export const TARGET_AGENT = 'agent-000'-export const FULL_CHAT = 'chat-0000'-export const PARTIAL_CHAT = 'chat-0001'-export const PAGE_SIZE = 100-export const PARTIAL_SIZE = 37--function messages(count) {- return Array.from({ length: count }, (_, index) => ({- id: `message-${index}`,- role: index % 2 === 0 ? 'user' : 'assistant',- content: 'Synthetic benchmark message.',- timestamp: 1_700_000_000_000 + index,- }))-}--export const FULL_MESSAGES = messages(PAGE_SIZE)-export const PARTIAL_MESSAGES = messages(PARTIAL_SIZE)+export {+ AUTHORITY_SCOPE,+ FULL_CHAT,+ FULL_MESSAGES,+ PAGE_SIZE,+ PARTIAL_CHAT,+ PARTIAL_MESSAGES,+ PARTIAL_SIZE,+ TARGET_AGENT,+ createFixture,+} from '../../test/e2e-playwright/helpers/chatCacheFixture.mjs'export function captureFilesystemFailures(root, failCleanup) {const originalReaddir = fs.readdir@@ -57,103 +52,6 @@ export function captureFilesystemFailures(root, failCleanup) {}}-export async function createFixture({- runDir,- adapter,- ChatStore,- agents,- chatsPerAgent,- pendingCleanupsPerAgent,-}) {- const root = await fs.mkdtemp(join(runDir, 'data-'))- const agentNames = Array.from(- { length: agents },- (_, index) => `agent-${String(index).padStart(3, '0')}`- )- const tombstones = Array.from({ length: pendingCleanupsPerAgent }, (_, index) => ({- chatId: `deleted-${index}`,- authorityScope: { ...AUTHORITY_SCOPE },- }))- const chats = Array.from({ length: chatsPerAgent }, (_, index) => ({- id: `chat-${String(index).padStart(4, '0')}`,- title: 'Synthetic benchmark chat',- createdAt: '2024-01-01T00:00:00.000Z',- updatedAt: '2024-01-01T00:00:00.000Z',- messageCount: index === 0 ? PAGE_SIZE : index === 1 ? PARTIAL_SIZE : 0,- errorCount: 0,- toolCallCount: 0,- }))- const index = {- version: 2,- lastActiveChatId: PARTIAL_CHAT,- onboardingDismissed: false,- chats,- ...(tombstones.length- ? { deletedChatTombstones: tombstones, pendingChatCleanup: tombstones }- : {}),- }- const catalog = JSON.stringify(index, null, 2)- const emptyChatCatalog = JSON.stringify(- {- ...index,- chats: chats.map(chat => ({ ...chat, messageCount: 0 })),- },- null,- 2- )- for (const agent of agentNames) {- const agentDir = join(root, agent)- await fs.mkdir(join(agentDir, '.corrupt'), { recursive: true, mode: 0o700 })- await fs.writeFile(- join(agentDir, 'index.json'),- agent === TARGET_AGENT ? catalog : emptyChatCatalog,- { mode: 0o600 }- )- for (const tombstone of tombstones) {- const chatDir = join(agentDir, 'chats', tombstone.chatId)- await fs.mkdir(chatDir, { recursive: true, mode: 0o700 })- await fs.writeFile(- join(chatDir, 'meta.json'),- JSON.stringify({- version: 1,- chatId: tombstone.chatId,- pageSize: PAGE_SIZE,- messageCount: 0,- localMessageCount: 0,- pages: [],- }),- { mode: 0o600 }- )- await fs.writeFile(- join(agentDir, `${tombstone.chatId}.json`),- JSON.stringify({- version: 2,- chatId: tombstone.chatId,- messages: [],- }),- { mode: 0o600 }- )- }- }- // Use each version's own writer for valid page/compatibility metadata. Other- // catalog entries are empty chats: their transcripts are not opened by this bench.- const seedStore = adapter.bind(ChatStore, root)- await seedStore.saveMessages(TARGET_AGENT, FULL_CHAT, FULL_MESSAGES)- await seedStore.saveMessages(TARGET_AGENT, PARTIAL_CHAT, PARTIAL_MESSAGES)- // Seeding updates activity/counts. Restore identical catalogs before a fresh bind.- await fs.writeFile(join(root, TARGET_AGENT, 'index.json'), catalog, { mode: 0o600 })- assert.equal(- JSON.parse(await fs.readFile(join(root, TARGET_AGENT, 'index.json'), 'utf8')).chats.length,- chatsPerAgent- )- if (tombstones.length) {- assert.ok(- (await fs.stat(join(root, TARGET_AGENT, 'chats', tombstones[0].chatId))).isDirectory()- )- }- return { root, agentNames, tombstones, catalogBytes: Buffer.byteLength(catalog) }-}-export async function inspectFixture(fixture, chatsPerAgent) {let pending = 0let retainedArtifactDirectories = 0diff --git a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts b/desktop-app/test/e2e-playwright/chat-open-ab.spec.tsindex 403cd01b0..63807d9a7 100644--- a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts+++ b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts@@ -1,14 +1,15 @@import { type ElectronApplication, type Page, expect, test } from '@playwright/test'-import { execFileSync } from 'node:child_process'+import { execFileSync, spawnSync } from 'node:child_process'import fs from 'node:fs'import os from 'node:os'import path from 'node:path'-import { createFixture } from '../../scripts/bench-chat-open/fixtures.mjs'import { ChatStore } from '../../src/chatStore'import { isPathInside } from '../../src/devIsolation'import { assertSafeFilesystemSegment } from '../../src/pathSafety'import { type ChatAuthorityScope } from '../../src/types'import { SIDEBAR_SESSION_PREVIEW_LIMIT as SIDEBAR_PREVIEW } from '../../ui/src/constants/navigation'+import { assertAllowedKubeContext } from './global-setup'+import { createFixture } from './helpers/chatCacheFixture.mjs'import {type HostSnapshot,readHostSnapshot,@@ -59,7 +60,12 @@ type Sample = {blank?: boolean/** What the observer saw just before the click and on its first callbacks after it. */trace?: OpenObservation[]+ /** Relaunch metrics: observed before the test's first pointer event in the window. */+ before_first_click?: boolean}+/** Runs a measured window; the Host snapshot loop of the round is paused while it runs. */+type Timed = <T>(work: () => Promise<T>) => Promise<T>+const untimed: Timed = work => work()type OpenObservation = {ms: numberlists: number@@ -76,6 +82,10 @@ const LOADING_SKELETON = '[aria-label="Loading conversation"]'const TRACE_LIMIT = 8const LIMIT_WINDOW_MS = 65_000const RATE_LIMITED = /\b429\b|Too Many Requests/g+// Relaunch rounds per build in L and W.+const RELAUNCH_ROUNDS = 3+// A live Desktop from an earlier A/B run would hold the Keychain item this run needs.+const AB_PROCESS = /Evenfire Issue1091Ab/const rows = (host: string) =>`[data-testid="latest-session-row"][data-agent-ref=${JSON.stringify(host)}]`const row = (host: string, id: string) => `${rows(host)}[data-chat-id=${JSON.stringify(id)}]`@@ -114,10 +124,12 @@ async function openChat(page: Page,host: string,label: string,- chat: Chat+ chat: Chat,+ timed: Timed): Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }> {+ // A deadline, not a fixed sleep: wait only for what is left of the pace since the last open.const pace = lastOpenAt + OPEN_PACE_MS - Date.now()- if (pace > 0) await page.waitForTimeout(pace)+ if (pace > 0) await new Promise(resolve => setTimeout(resolve, pace))lastOpenAt = Date.now()await page.getByRole('navigation', { name: 'Chat breadcrumb', exact: true })@@ -151,77 +163,84 @@ async function openChat(}).toBe(true)- return page.evaluate(- ({ selector, chat, messages, skeleton, traceLimit }) =>- new Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }>((resolve, reject) => {- const button = document.querySelector<HTMLButtonElement>(`${selector} button`)- if (!button || button.disabled)- throw new Error(`Chat row has no enabled open button: ${selector}`)- let started = performance.now()- let clicked = false- let blank = false- const trace: OpenObservation[] = []- // Before the click the Chat breadcrumb shows the Host's chat list with no active chat, which- // already has zero bubbles; only the loading skeleton marks a chat that was opened empty.- const observe = () => {- const transcripts = document.querySelectorAll('[data-testid="message-list"]')- const observation = {- ms: clicked ? performance.now() - started : -1,- lists: transcripts.length,- bubbles: transcripts[0]?.querySelectorAll(messages).length ?? 0,- skeleton: document.querySelector(skeleton) !== null,- dedicated: transcripts[0]?.classList.contains('chat-thread-dedicated') ?? false,+ return timed(() =>+ page.evaluate(+ ({ selector, chat, messages, skeleton, traceLimit }) =>+ new Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }>((resolve, reject) => {+ const button = document.querySelector<HTMLButtonElement>(`${selector} button`)+ if (!button || button.disabled)+ throw new Error(`Chat row has no enabled open button: ${selector}`)+ let blank = false+ const trace: OpenObservation[] = []+ // Before the click the Chat breadcrumb shows the Host's chat list with no active chat, which+ // already has zero bubbles; only the loading skeleton marks a chat that was opened empty.+ const observe = (ms: number) => {+ const transcripts = document.querySelectorAll('[data-testid="message-list"]')+ const observation = {+ ms,+ lists: transcripts.length,+ bubbles: transcripts[0]?.querySelectorAll(messages).length ?? 0,+ skeleton: document.querySelector(skeleton) !== null,+ dedicated: transcripts[0]?.classList.contains('chat-thread-dedicated') ?? false,+ }+ if (trace.length < traceLimit) trace.push(observation)+ return observation}- if (trace.length < traceLimit) trace.push(observation)- return observation- }- observe()- // The sidebar previews only the latest sessions, so the active-row witness is checked outside the timing.- const finish = () => {- const transcripts = document.querySelectorAll('[data-testid="message-list"]')- if (transcripts.length > 1) {- observer.disconnect()- clearTimeout(deadline)- reject(new Error(`Expected one transcript, found ${transcripts.length}.`))- return+ observe(-1)+ // The sidebar previews only the latest sessions, so the active-row witness is checked outside the timing.+ let scheduled = false+ const finish = () => {+ scheduled = false+ const transcripts = document.querySelectorAll('[data-testid="message-list"]')+ if (transcripts.length > 1) {+ observer.disconnect()+ clearTimeout(deadline)+ reject(new Error(`Expected one transcript, found ${transcripts.length}.`))+ return+ }+ const transcript = transcripts[0]+ const bubbles = transcript?.querySelectorAll(messages)+ if (observe(performance.now() - started).skeleton) blank = true+ if (+ bubbles?.length === chat.count &&+ bubbles[chat.count - 1]?.textContent?.includes(chat.lastText) &&+ transcript &&+ transcript.getBoundingClientRect().height > 0+ ) {+ observer.disconnect()+ clearTimeout(deadline)+ resolve({ ms: performance.now() - started, blank, trace })+ }}- const transcript = transcripts[0]- const bubbles = transcript?.querySelectorAll(messages)- if (observe().skeleton) blank = true- if (- bubbles?.length === chat.count &&- bubbles[chat.count - 1]?.textContent?.includes(chat.lastText) &&- transcript &&- transcript.getBoundingClientRect().height > 0- ) {+ // One check per animation frame: a mutation only schedules it, so layout is read once+ // per frame instead of once per mutation. Callbacks run after the click below sets `started`.+ const observer = new MutationObserver(() => {+ if (scheduled) return+ scheduled = true+ requestAnimationFrame(finish)+ })+ observer.observe(document.documentElement, {+ childList: true,+ subtree: true,+ characterData: true,+ attributes: true,+ })+ const deadline = setTimeout(() => {observer.disconnect()- clearTimeout(deadline)- resolve({ ms: performance.now() - started, blank, trace })- }- }- const observer = new MutationObserver(finish)- observer.observe(document.documentElement, {- childList: true,- subtree: true,- characterData: true,- attributes: true,- })- const deadline = setTimeout(() => {- observer.disconnect()- reject(new Error(`Transcript did not match server chat ${chat.id}`))- }, 120_000)- started = performance.now()- button.click()- clicked = true- finish()- }),- {- selector: agentRow(host, chat.id),- chat,- messages: MESSAGES,- skeleton: LOADING_SKELETON,- traceLimit: TRACE_LIMIT,- }+ reject(new Error(`Transcript did not match server chat ${chat.id}`))+ }, 120_000)+ const started = performance.now()+ button.click()+ finish()+ }),+ {+ selector: agentRow(host, chat.id),+ chat,+ messages: MESSAGES,+ skeleton: LOADING_SKELETON,+ traceLimit: TRACE_LIMIT,+ }+ ))}async function relaunchPaint(@@ -237,6 +256,7 @@ async function relaunchPaint(ms: numberobserved_at_install: booleanobserver_installed_ms: number+ before_first_click: boolean}>>((resolve, reject) => {const navigation = performance.getEntriesByType('navigation')[0] as@@ -244,12 +264,23 @@ async function relaunchPaint(| undefinedif (!navigation) throw new Error('Window navigation timing is unavailable.')const installed = performance.now()- const observed = new Map<string, { at: number; initial: boolean }>()+ const observed = new Map<string, { at: number; initial: boolean; beforeClick: boolean }>()let initial = true+ // Witness for metrics that must be observed before the test clicks anything.+ let clicked = false+ document.addEventListener(+ 'pointerdown',+ () => {+ clicked = true+ },+ { capture: true, once: true }+ )+ let scheduled = falseconst visible = (element: Element) =>element.getBoundingClientRect().height > 0 &&getComputedStyle(element).visibility !== 'hidden'const finish = () => {+ scheduled = falseconst transcripts = document.querySelectorAll('[data-testid="message-list"]')if (transcripts.length > 1) {observer.disconnect()@@ -269,7 +300,11 @@ async function relaunchPaint(.querySelectorAll(messages)[target.chat.count - 1]?.textContent?.includes(target.chat.lastText)))if (ready && !observed.has(target.metric))- observed.set(target.metric, { at: performance.now(), initial })+ observed.set(target.metric, {+ at: performance.now(),+ initial,+ beforeClick: !clicked,+ })}if (observed.size === targets.length && navigation.loadEventEnd > 0) {observer.disconnect()@@ -283,13 +318,19 @@ async function relaunchPaint(ms: Math.max(0, result.at - navigation.loadEventEnd),observed_at_install: result.initial,observer_installed_ms: installed - navigation.loadEventEnd,+ before_first_click: result.beforeClick,}}))}initial = false}- const observer = new MutationObserver(finish)+ // One check per animation frame, as in openChat.+ const observer = new MutationObserver(() => {+ if (scheduled) return+ scheduled = true+ requestAnimationFrame(finish)+ })observer.observe(document.documentElement, {childList: true,subtree: true,@@ -331,23 +372,28 @@ if (const context = required('CHAT_OPEN_AB_KUBE_CONTEXT')if (context !== context.trim() || context.startsWith('-'))throw new Error('CHAT_OPEN_AB_KUBE_CONTEXT must be the literal Kubernetes context.')+ // The shared global setup is disabled for this config; apply its context guard before any kubectl call.+ assertAllowedKubeContext(context, 'CHAT_OPEN_AB_KUBE_CONTEXT')const output = required('CHAT_OPEN_AB_OUTPUT_JSON')if (!path.isAbsolute(output) || path.extname(output) !== '.json')throw new Error('CHAT_OPEN_AB_OUTPUT_JSON must be an absolute JSON path.')const phase = required('CHAT_OPEN_AB_PHASE')if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')const builds = Object.fromEntries(- Object.entries(entries).map(([build, entry]) => [- build,- {- entry,- sha: execFileSync(- 'git',- ['-C', path.resolve(path.dirname(entry), '../..'), 'rev-parse', 'HEAD'],- { encoding: 'utf8' }- ).trim(),- },- ])+ Object.entries(entries).map(([build, entry]) => {+ const git = (...args: string[]) =>+ execFileSync('git', ['-C', path.resolve(path.dirname(entry), '../..'), ...args], {+ encoding: 'utf8',+ }).trim()+ // A build must be exactly its commit: a clean tree, and dist/main.js built after the commit.+ const dirty = git('status', '--porcelain')+ if (dirty) throw new Error(`precondition: the ${build} worktree has uncommitted changes.`)+ const committedAt = Number(git('log', '-1', '--format=%ct')) * 1000+ const builtAt = fs.statSync(entry).mtimeMs+ if (!(builtAt > committedAt))+ throw new Error(`precondition: the ${build} build is older than its HEAD commit.`)+ return [build, { entry, sha: git('rev-parse', 'HEAD'), committedAt, builtAt }]+ }))const samples: Sample[] = []const rounds: Array<{@@ -393,27 +439,84 @@ if (closedAt = Date.now()}}+ // One app identity per run, so Electron creates its own `<appName> Safe Storage` Keychain item+ // instead of reading one an earlier run left behind. BEFORE and AFTER share it and the user data.+ const isolationTarget = `issue-1091-ab-${Date.now().toString(36)}`+ const electronPath = fs.realpathSync(require('electron') as unknown as string)let launches = 0let mainLog = ''+ // The current launch's main-process output, kept in memory so a round reads all of it.+ let mainOutput: string[] = []let launchedBuild: Build | undefined- const launch = async (build: Build) => {+ // A first window that never opens is usually a Keychain prompt nobody answers. Record what the+ // process was doing and whether macOS shows a prompt; never answer or bypass it.+ const diagnoseFirstWindow = (starting: ElectronApplication): string => {+ const samplePath = testInfo.outputPath(`sample-${launches}.txt`)+ execFileSync('sample', [String(starting.process().pid), '5', '-file', samplePath])+ const agent = spawnSync('pgrep', ['-x', 'SecurityAgent'])+ if (agent.status !== 0 && agent.status !== 1)+ throw new Error(`pgrep SecurityAgent failed with status ${agent.status}.`)+ return agent.status === 0+ ? `KEYCHAIN_PROMPT_PENDING: SecurityAgent is running; sample in ${samplePath}`+ : `FIRST_WINDOW_TIMEOUT: no SecurityAgent process; sample in ${samplePath}`+ }+ // A closed build's background requests can exhaust control-api's per-minute+ // limits; the next build's session restore then fails. Start each launch in+ // a fresh window, outside every timed section.+ const awaitQuietWindow = async () => {await close()- // A closed build's background requests can exhaust control-api's per-minute- // limits; the next build's session restore then fails. Start each launch in- // a fresh window, outside every timed section.const quiet = closedAt + LIMIT_WINDOW_MS - Date.now()if (quiet > 0) await new Promise(resolve => setTimeout(resolve, quiet))- const launched = await launchDesktopApp(testInfo, 'issue-1091-ab', entries[build])- app = launched.app- page = launched.page- launchedBuild = build+ }+ const launch = async (build: Build) => {+ await awaitQuietWindow()+ const live = execFileSync('ps', ['-axo', 'pid=,args='], { encoding: 'utf8' })+ .split('\n')+ .filter(line => AB_PROCESS.test(line))+ .map(line => line.trim().split(/\s+/)[0])+ if (live.length)+ throw new Error(`precondition: an A/B Desktop is still running (pids ${live.join(', ')}).`)// Keep each main process's output so a lost session can be explained after the run.mainLog = testInfo.outputPath(`main-${++launches}-${build}.log`)- const log = fs.createWriteStream(mainLog, {- mode: 0o600,- })- app.process().stdout?.pipe(log)- app.process().stderr?.pipe(log)+ const log = fs.createWriteStream(mainLog, { mode: 0o600 })+ const output: string[] = []+ mainOutput = output+ let starting: ElectronApplication | undefined+ try {+ const launched = await launchDesktopApp(testInfo, isolationTarget, entries[build], {+ executablePath: electronPath,+ beforeFirstWindow: started => {+ starting = started+ for (const stream of [started.process().stdout, started.process().stderr])+ stream?.on('data', (chunk: Buffer) => {+ output.push(chunk.toString('utf8'))+ log.write(chunk)+ })+ },+ })+ app = launched.app+ page = launched.page+ } catch (error) {+ const diagnosis =+ starting && /firstWindow: Timeout/.test(String((error as Error)?.message))+ ? diagnoseFirstWindow(starting)+ : undefined+ await starting?.close()+ if (diagnosis) throw new Error(diagnosis, { cause: error })+ throw error+ }+ launchedBuild = build+ // Public process identities only: the pinned Electron, this build's code and the run's profile.+ const runtime = await app.evaluate(({ app: electronApp }) => ({+ executablePath: process.execPath,+ appPath: electronApp.getAppPath(),+ userData: electronApp.getPath('userData'),+ }))+ expect(fs.realpathSync(runtime.executablePath)).toBe(electronPath)+ expect(fs.realpathSync(runtime.appPath)).toBe(fs.realpathSync(path.dirname(entries[build])))+ expect(fs.realpathSync(runtime.userData)).toBe(+ fs.realpathSync(path.join(runDir, 'user-data'))+ )}const authenticated = async () => {try {@@ -479,7 +582,13 @@ if (),}}- const round = async (scenario: string, build: Build, action: (id: number) => Promise<void>) => {+ // `timed` wraps every measured window; the Host snapshots (each a few kubectl processes)+ // run only between those windows, so they never compete with a measurement.+ const round = async (+ scenario: string,+ build: Build,+ action: (id: number, timed: Timed) => Promise<void>+ ) => {const record: (typeof rounds)[number] = {id: rounds.length + 1,scenario,@@ -491,19 +600,36 @@ if (}rounds.push(record)let stopped = false+ let paused = false+ let sampling: Promise<void> = Promise.resolve()let observationFailure: { error: unknown } | undefinedconst observation = (async () => {while (!stopped) {- record.snapshots.push(await snapshot())+ if (!paused) {+ sampling = snapshot().then(state => {+ record.snapshots.push(state)+ })+ await sampling+ }if (!stopped) await new Promise(resolve => setTimeout(resolve, 250))}})().catch(error => {observationFailure = { error }})+ const timed: Timed = async work => {+ paused = true+ try {+ // A snapshot already running finishes before the measured window opens.+ await sampling+ return await work()+ } finally {+ paused = false+ }+ }try {- await action(record.id)+ await action(record.id, timed)// A rate-limited round runs on stale or empty catalogs, so its timings are not valid samples.- const limited = fs.readFileSync(mainLog, 'utf8').match(RATE_LIMITED)?.length ?? 0+ const limited = mainOutput.join('').match(RATE_LIMITED)?.length ?? 0if (limited)throw new Error(`Round ${record.id} (${scenario} ${build}) logged ${limited} rate-limited responses in ${path.basename(mainLog)}.`@@ -671,7 +797,7 @@ if (for (const build of ['before', 'after', 'after', 'before'] as const) {await ready()await test.step(`O: ${build} cached chat opens`, () =>- round('O', build, async id => {+ round('O', build, async (id, timed) => {await launch(build)await authenticated()await page.getByTestId('nav-chat').click()@@ -721,7 +847,8 @@ if (page,catalog.host,hostLabels[catalog.host]!,- chat+ chat,+ timed)samples.push({round: id,@@ -753,8 +880,8 @@ if (// The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.// An empty preview would satisfy "none is active" without checking anything.await expect(- page.locator('[data-testid="latest-session-row"]').first()- ).toBeVisible()+ page.locator('[data-testid="latest-session-row"]')+ ).not.toHaveCount(0)const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0await expect(page.locator('[data-testid="latest-session-row"][aria-current="true"]')@@ -786,7 +913,7 @@ if (throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)return at > updatedAt.get(latest.id)! ? chat : latest})- await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)+ await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected, untimed)await expect.poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0])).toBe(selected.id)@@ -805,31 +932,39 @@ if (isolatedPath(runDir, path.join(savedCache, agent)),{ recursive: true })+ // L: the large cache is in place and the app relaunches. Desktop does not reopen the last+ // active chat on launch, so the user opens it: relaunch_rows_ms ends at the first stateful+ // sidebar row (before any click), relaunch_ms at the transcript of the explicit open.for (const build of ['before', 'after'] as const)- for (let iteration = 0; iteration < 3; iteration++) {+ for (let iteration = 0; iteration < RELAUNCH_ROUNDS; iteration++) {await ready()await close()fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>- round('L', build, async id => {- await launch(build)- const [paint] = await Promise.all([- relaunchPaint(page, [- {- host: HOSTS[0],- metric: 'relaunch_ms',- selector: row(HOSTS[0], selected.id),- chat: selected,- },- ]),- (async () => {- await authenticated()- await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()- await page.getByTestId('nav-chat').click()- await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)- })(),- ])+ round('L', build, async (id, timed) => {+ await awaitQuietWindow()+ const paint = await timed(async () => {+ await launch(build)+ const [observed] = await Promise.all([+ relaunchPaint(page, [+ { host: HOSTS[0], metric: 'relaunch_rows_ms', selector: rows(HOSTS[0]) },+ {+ host: HOSTS[0],+ metric: 'relaunch_ms',+ selector: row(HOSTS[0], selected.id),+ chat: selected,+ },+ ]),+ (async () => {+ await authenticated()+ await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()+ await page.getByTestId('nav-chat').click()+ await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected, untimed)+ })(),+ ])+ return observed+ })for (const sample of paint) samples.push({ round: id, build, ...sample })await authenticated()}))@@ -837,24 +972,27 @@ if (}if (phase === 'W')for (const build of ['before', 'after'] as const)- for (let iteration = 0; iteration < 3; iteration++) {+ for (let iteration = 0; iteration < RELAUNCH_ROUNDS; iteration++) {await close()await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])// Empty only the isolated chat cache: both observed rows must come from the remote catalog.if (fs.existsSync(cacheDir!))fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>- round('W', build, async id => {- await launch(build)- for (const sample of await relaunchPaint(- page,- HOSTS.map((host, index) => ({- host,- selector: rows(host),- metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',- }))- ))- samples.push({ round: id, build, ...sample })+ round('W', build, async (id, timed) => {+ await awaitQuietWindow()+ const paint = await timed(async () => {+ await launch(build)+ return relaunchPaint(+ page,+ HOSTS.map((host, index) => ({+ host,+ selector: rows(host),+ metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',+ }))+ )+ })+ for (const sample of paint) samples.push({ round: id, build, ...sample })await authenticated()await ready()}))@@ -926,16 +1064,63 @@ if (witness.before_reopen_blank === values('before', host, 'open_memo_ms').length,})}- const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,- after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50- report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })+ // Every relaunch round of both builds contributes one sample per L metric.+ const relaunchSamples = (build: Build, metric: string) => {+ const found = values(build, HOSTS[0], metric)+ if (found.length !== RELAUNCH_ROUNDS)+ throw new Error(`L recorded ${found.length} ${build} ${metric} samples.`)+ return found+ }+ // relaunch_rows_ms: window load to the first stateful sidebar row, observed before the+ // test clicks anything; the non-blocking chat:getIndex is what shortens it.+ const rowsUnclicked = samples+ .filter(sample => sample.metric === 'relaunch_rows_ms')+ .every(sample => sample.before_first_click === true)+ const rowsBefore = quantiles(relaunchSamples('before', 'relaunch_rows_ms')).p50,+ rowsAfter = quantiles(relaunchSamples('after', 'relaunch_rows_ms')).p50+ report.gates.push({+ metric: 'relaunch_rows_ms',+ definition: 'window load -> first stateful sidebar row, before any click',+ before: rowsBefore,+ after: rowsAfter,+ observed_before_click: rowsUnclicked,+ passed: rowsUnclicked && rowsAfter < rowsBefore,+ })+ // relaunch_ms: window load to the transcript of the selected chat, including the explicit+ // open. Desktop never restores the last active chat on launch.+ const before = quantiles(relaunchSamples('before', 'relaunch_ms')).p50,+ after = quantiles(relaunchSamples('after', 'relaunch_ms')).p50+ report.gates.push({+ metric: 'relaunch_ms',+ definition: 'window load -> sidebar row -> explicit open -> transcript with all messages',+ before,+ after,+ passed: after < before,+ })} else {- const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))+ // W needs a control: BEFORE runs the same suspended-Host relaunches as AFTER.+ const wake = (build: Build) => {+ const found = values(build, HOSTS[0], 'first_stateful_row_ms')+ if (found.length !== RELAUNCH_ROUNDS)+ throw new Error(`W recorded ${found.length} ${build} first_stateful_row_ms samples.`)+ return found+ }+ const wakeBefore = wake('before'),+ wakeAfter = wake('after')+ const afterMax = Math.max(...wakeAfter)report.gates.push({metric: 'first_stateful_row_ms',- after_max_ms: wakeAfter,+ after_max_ms: afterMax,budget_ms: 2000,- passed: wakeAfter < 2000,+ passed: afterMax < 2000,+ })+ const beforeP50 = quantiles(wakeBefore).p50,+ afterP50 = quantiles(wakeAfter).p50+ report.gates.push({+ metric: 'first_stateful_row_ms_vs_before',+ before: beforeP50,+ after: afterP50,+ passed: afterP50 < beforeP50,})}report.gates.push({@@ -966,6 +1151,10 @@ if (fs.writeFileSync(output, JSON.stringify({ ...report, catalogs }, null, 2), { mode: 0o600 })}}+ // O: three gates per Host; L: two; W: two; both phases: round validity.+ expect(report.gates.length, 'Desktop A/B gate count').toBe(+ phase === 'OL' ? HOSTS.length * 3 + 3 : 3+ )expect(report.gates.every(gate => gate.passed),`Desktop A/B gates: ${JSON.stringify(report.gates)}; evidence: ${output}`diff --git a/desktop-app/test/e2e-playwright/global-setup.ts b/desktop-app/test/e2e-playwright/global-setup.tsindex 07479223a..74ba4b267 100644--- a/desktop-app/test/e2e-playwright/global-setup.ts+++ b/desktop-app/test/e2e-playwright/global-setup.ts@@ -6,12 +6,12 @@ import { E2E_TEST_EMAIL } from '../../../tests/e2e/testUser'const configuredAllowedContexts = process.env.E2E_ALLOWED_CONTEXTS?.split(',').map(context => context.trim()).filter(Boolean)-const ALLOWED_CONTEXTS = new Set(+export const ALLOWED_CONTEXTS: ReadonlySet<string> = new Set(configuredAllowedContexts?.length ? configuredAllowedContexts : ['clerum-test'])const PROD_CONTEXT = process.env.E2E_PROD_CONTEXT?.trim() || ''-function isProductionContext(context: string): boolean {+export function isProductionContext(context: string): boolean {return ((PROD_CONTEXT !== '' && context === PROD_CONTEXT) ||/(^|[-_])(prod|production)([-_]|$)/i.test(context)@@ -27,7 +27,7 @@ function isBranchProfileContext(context: string): boolean {return /^clerum-.+-[0-9a-f]{7,8}$/.test(context)}-function isLocalPortForwardContext(context: string): boolean {+export function isLocalPortForwardContext(context: string): boolean {return (context === 'clerum-test' ||/^clerum-(codex|detached)-.+-[0-9a-f]{8}$/.test(context) ||@@ -35,6 +35,21 @@ function isLocalPortForwardContext(context: string): boolean {)}+/** Refuses production and any context that is neither a local port-forward profile nor allow-listed. */+export function assertAllowedKubeContext(context: string, source: string): void {+ if (isProductionContext(context)) {+ throw new Error(+ `[E2E-GUARD] Production context "${context}" is hard-blocked. E2E against prod is never allowed.`+ )+ }+ if (!isLocalPortForwardContext(context) && !ALLOWED_CONTEXTS.has(context)) {+ throw new Error(+ `[E2E-GUARD] ${source}="${context}" is not in the allow-list. ` ++ `Allowed: ${[...ALLOWED_CONTEXTS].join(', ')}.`+ )+ }+}+function parseBooleanEnv(value: string | undefined): boolean {// Keep this narrow on purpose so only the documented values ("1"/"true")// enable dev port-forward mode in CI/E2E runs.@@ -174,17 +189,7 @@ async function globalSetup(): Promise<void> {process.env.K8S_CONTEXT ||'clerum-test'- if (isProductionContext(expected)) {- throw new Error(- `[E2E-GUARD] Production context "${expected}" is hard-blocked. E2E against prod is never allowed.`- )- }- if (!isLocalPortForwardContext(expected) && !ALLOWED_CONTEXTS.has(expected)) {- throw new Error(- `[E2E-GUARD] E2E_K8S_CONTEXT="${expected}" is not in the allow-list. ` +- `Allowed: ${[...ALLOWED_CONTEXTS].join(', ')}.`- )- }+ assertAllowedKubeContext(expected, 'E2E_K8S_CONTEXT')let current: stringtry {diff --git a/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjs b/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjsnew file mode 100644index 000000000..34a3c09bc--- /dev/null+++ b/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjs@@ -0,0 +1,125 @@+import assert from 'node:assert/strict'+import { promises as fs } from 'node:fs'+import { join } from 'node:path'++// A synthetic chat cache written through ChatStore: shared by the chat-open benchmark+// (scripts/bench-chat-open) and the Desktop A/B spec, which uses it as the large cache.+export const AUTHORITY_SCOPE = Object.freeze({+ environmentKey: 'benchmark-environment',+ userId: 'benchmark-user',+ teamId: 'benchmark-team',+})+export const TARGET_AGENT = 'agent-000'+export const FULL_CHAT = 'chat-0000'+export const PARTIAL_CHAT = 'chat-0001'+export const PAGE_SIZE = 100+export const PARTIAL_SIZE = 37++function messages(count) {+ return Array.from({ length: count }, (_, index) => ({+ id: `message-${index}`,+ role: index % 2 === 0 ? 'user' : 'assistant',+ content: 'Synthetic benchmark message.',+ timestamp: 1_700_000_000_000 + index,+ }))+}++export const FULL_MESSAGES = messages(PAGE_SIZE)+export const PARTIAL_MESSAGES = messages(PARTIAL_SIZE)++export async function createFixture({+ runDir,+ adapter,+ ChatStore,+ agents,+ chatsPerAgent,+ pendingCleanupsPerAgent,+}) {+ const root = await fs.mkdtemp(join(runDir, 'data-'))+ const agentNames = Array.from(+ { length: agents },+ (_, index) => `agent-${String(index).padStart(3, '0')}`+ )+ const tombstones = Array.from({ length: pendingCleanupsPerAgent }, (_, index) => ({+ chatId: `deleted-${index}`,+ authorityScope: { ...AUTHORITY_SCOPE },+ }))+ const chats = Array.from({ length: chatsPerAgent }, (_, index) => ({+ id: `chat-${String(index).padStart(4, '0')}`,+ title: 'Synthetic benchmark chat',+ createdAt: '2024-01-01T00:00:00.000Z',+ updatedAt: '2024-01-01T00:00:00.000Z',+ messageCount: index === 0 ? PAGE_SIZE : index === 1 ? PARTIAL_SIZE : 0,+ errorCount: 0,+ toolCallCount: 0,+ }))+ const index = {+ version: 2,+ lastActiveChatId: PARTIAL_CHAT,+ onboardingDismissed: false,+ chats,+ ...(tombstones.length+ ? { deletedChatTombstones: tombstones, pendingChatCleanup: tombstones }+ : {}),+ }+ const catalog = JSON.stringify(index, null, 2)+ const emptyChatCatalog = JSON.stringify(+ {+ ...index,+ chats: chats.map(chat => ({ ...chat, messageCount: 0 })),+ },+ null,+ 2+ )+ for (const agent of agentNames) {+ const agentDir = join(root, agent)+ await fs.mkdir(join(agentDir, '.corrupt'), { recursive: true, mode: 0o700 })+ await fs.writeFile(+ join(agentDir, 'index.json'),+ agent === TARGET_AGENT ? catalog : emptyChatCatalog,+ { mode: 0o600 }+ )+ for (const tombstone of tombstones) {+ const chatDir = join(agentDir, 'chats', tombstone.chatId)+ await fs.mkdir(chatDir, { recursive: true, mode: 0o700 })+ await fs.writeFile(+ join(chatDir, 'meta.json'),+ JSON.stringify({+ version: 1,+ chatId: tombstone.chatId,+ pageSize: PAGE_SIZE,+ messageCount: 0,+ localMessageCount: 0,+ pages: [],+ }),+ { mode: 0o600 }+ )+ await fs.writeFile(+ join(agentDir, `${tombstone.chatId}.json`),+ JSON.stringify({+ version: 2,+ chatId: tombstone.chatId,+ messages: [],+ }),+ { mode: 0o600 }+ )+ }+ }+ // Use each version's own writer for valid page/compatibility metadata. Other+ // catalog entries are empty chats: their transcripts are not opened by this bench.+ const seedStore = adapter.bind(ChatStore, root)+ await seedStore.saveMessages(TARGET_AGENT, FULL_CHAT, FULL_MESSAGES)+ await seedStore.saveMessages(TARGET_AGENT, PARTIAL_CHAT, PARTIAL_MESSAGES)+ // Seeding updates activity/counts. Restore identical catalogs before a fresh bind.+ await fs.writeFile(join(root, TARGET_AGENT, 'index.json'), catalog, { mode: 0o600 })+ assert.equal(+ JSON.parse(await fs.readFile(join(root, TARGET_AGENT, 'index.json'), 'utf8')).chats.length,+ chatsPerAgent+ )+ if (tombstones.length) {+ assert.ok(+ (await fs.stat(join(root, TARGET_AGENT, 'chats', tombstones[0].chatId))).isDirectory()+ )+ }+ return { root, agentNames, tombstones, catalogBytes: Buffer.byteLength(catalog) }+}diff --git a/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts b/desktop-app/test/e2e-playwright/helpers/hostSnapshot.tsindex ae076f638..f8e2cfa52 100644--- a/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts+++ b/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts@@ -5,22 +5,43 @@ import { promisify } from 'node:util'// E2E_GUARDIAN_IPC_FLOW: Host lifecycle is observed read-only through Kubernetes;// Desktop RPC runs in the main process, outside renderer HTTP observation.const execFileAsync = promisify(execFile)-const STATELESS_NAMESPACE = 'mcp-host'+const HOST_NAMESPACE = 'mcp-host'-async function kubectl(context: string, args: string[]): Promise<string> {+export async function kubectl(+ context: string,+ args: string[],+ timeoutMs = 20_000+): Promise<string> {if (!context.trim() || context.startsWith('-'))throw new Error('An explicit kube context is required.')const { stdout } = await execFileAsync('kubectl', ['--context', context, ...args], {- timeout: 20_000,+ timeout: timeoutMs,maxBuffer: 8 * 1024 * 1024,})return stdout.toString()}-async function jsonKubectl(context: string, resource: string): Promise<any> {- return JSON.parse(- await kubectl(context, ['-n', STATELESS_NAMESPACE, 'get', resource, '-o', 'json'])- )+export async function jsonKubectl<T>(+ context: string,+ namespace: string,+ resource: string+): Promise<T> {+ return JSON.parse(await kubectl(context, ['-n', namespace, 'get', resource, '-o', 'json'])) as T+}++/** The fields of the Deployment, Host and Pod JSON that a snapshot reads. */+type DeploymentJson = { spec?: { replicas?: number }; status?: { readyReplicas?: number } }+type HostJson = {+ status?: {+ lifecycle?: { state?: string; reason?: string }+ conditions?: Array<{ type?: string; status?: string; reason?: string }>+ }+}+type PodListJson = {+ items?: Array<{+ metadata?: { name?: string }+ status?: { containerStatuses?: Array<{ imageID?: string }> }+ }>}export type HostSnapshot = {@@ -39,39 +60,27 @@ export async function readHostSnapshot(context: string): Promise<HostSnapshot> {const [deploymentJson, hostJson, podsJson] = await Promise.all([- jsonKubectl(context, `deployment.apps/${deployment}`),- jsonKubectl(context, `host/${hostName}`),- kubectl(context, [- '-n',- STATELESS_NAMESPACE,- 'get',- 'pods',- '-l',- `app=${hostName}`,- '-o',- 'json',- ]),+ jsonKubectl<DeploymentJson>(context, HOST_NAMESPACE, `deployment.apps/${deployment}`),+ jsonKubectl<HostJson>(context, HOST_NAMESPACE, `host/${hostName}`),+ kubectl(context, ['-n', HOST_NAMESPACE, 'get', 'pods', '-l', `app=${hostName}`, '-o', 'json']),])- const pods = JSON.parse(podsJson).items ?? []- const spec = deploymentJson.spec ?? {}- const status = deploymentJson.status ?? {}+ const pods = (JSON.parse(podsJson) as PodListJson).items ?? []const lifecycle = hostJson.status?.lifecycle ?? {}const pullPolicyRejection = (hostJson.status?.conditions ?? []).find(- (condition: { type?: string; status?: string }) =>- condition.type === 'StatelessPullPolicyRejected' && condition.status === 'True'+ condition => condition.type === 'StatelessPullPolicyRejected' && condition.status === 'True')return {lifecycleState: String(lifecycle.state ?? ''),lifecycleReason: String(lifecycle.reason ?? ''),- replicas: Number(spec.replicas ?? -1),- readyReplicas: Number(status.readyReplicas ?? 0),+ replicas: Number(deploymentJson.spec?.replicas ?? -1),+ readyReplicas: Number(deploymentJson.status?.readyReplicas ?? 0),podNames: pods- .map((pod: { metadata?: { name?: string } }) => pod.metadata?.name)- .filter(Boolean)+ .map(pod => pod.metadata?.name)+ .filter((name): name is string => Boolean(name)).sort(),imageIds: [...new Set<string>(- pods.flatMap((pod: { status?: { containerStatuses?: Array<{ imageID?: string }> } }) =>+ pods.flatMap(pod =>(pod.status?.containerStatuses ?? []).map(container => container.imageID).filter((id): id is string => typeof id === 'string' && id.length > 0)diff --git a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.tsindex 085dab1a8..a1e44f03a 100644--- a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts+++ b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts@@ -16,10 +16,10 @@* `HUMAN_E2E_STATEFUL_HOST_RESOURCE`, `HUMAN_E2E_STATELESS_HOST_RESOURCE`.*/import { type ElectronApplication, type Locator, type Page, expect, test } from '@playwright/test'-import { type ChildProcess, execFile } from 'node:child_process'+import { type ChildProcess } from 'node:child_process'import { writeFileSync } from 'node:fs'-import { promisify } from 'node:util'import {+ jsonKubectl,readHostSnapshot,waitForStatelessReady,waitForStatelessSuspended,@@ -37,8 +37,6 @@ import {screenshotAndLog,} from './qa-recorder-helpers'-const execFileAsync = promisify(execFile)-const KUBE_CONTEXT = process.env.HUMAN_E2E_KUBE_CONTEXT?.trim() || ''const STATEFUL_HOST = process.env.HUMAN_E2E_STATEFUL_HOST?.trim() || 'chatLLM'const STATELESS_HOST = process.env.HUMAN_E2E_STATELESS_HOST?.trim() || 'chatllm-stateless'@@ -47,7 +45,6 @@ const STATELESS_HOST_RESOURCE =process.env.HUMAN_E2E_STATELESS_HOST_RESOURCE?.trim() || 'chatllm-stateless'const HCC_DEPLOYMENT = 'host-context-controller'const HCC_NAMESPACE = 'control-plane'-const STATELESS_NAMESPACE = 'mcp-host'const STATELESS_DEPLOYMENT = 'chatllm-stateless'const STATEFUL_DEPLOYMENT = 'chatllm'const SUSPEND_TIMEOUT_MS = 240_000@@ -103,28 +100,22 @@ type JourneyMetrics = {}}-async function kubectl(args: string[], timeoutMs = 20_000): Promise<string> {- const { stdout } = await execFileAsync('kubectl', ['--context', KUBE_CONTEXT, ...args], {- timeout: timeoutMs,- maxBuffer: 8 * 1024 * 1024,- })- return stdout.toString()-}--async function jsonKubectl(resource: string, namespace: string): Promise<any> {- return JSON.parse(await kubectl(['-n', namespace, 'get', resource, '-o', 'json']))-}-async function readHccCadences(): Promise<{idleMinutes: numberidleFloorMinutes: numberdrainGraceMs: numberheartbeatPollMs: number}> {- const deployment = await jsonKubectl(`deployment.apps/${HCC_DEPLOYMENT}`, HCC_NAMESPACE)+ const deployment = await jsonKubectl<{+ spec?: {+ template?: {+ spec?: { containers?: Array<{ env?: Array<{ name?: string; value?: string }> }> }+ }+ }+ }>(KUBE_CONTEXT, HCC_NAMESPACE, `deployment.apps/${HCC_DEPLOYMENT}`)const env = deployment.spec?.template?.spec?.containers?.[0]?.env ?? []const value = (key: string, fallback: number): number => {- const found = env.find((item: { name?: string }) => item.name === key)+ const found = env.find(item => item.name === key)return typeof found?.value === 'string' ? Number(found.value) : fallback}return {diff --git a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.tsindex cfb747bcb..b7209ce59 100644--- a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts+++ b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts@@ -17,6 +17,10 @@ if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must beexport default defineConfig({...base,+ // The shared global setup stays off: its preflight login spends an attempt from the password+ // budget, it reads E2E_K8S_CONTEXT instead of CHAT_OPEN_AB_KUBE_CONTEXT, and its seed marker+ // ties the profile to this worktree's HEAD while BEFORE is another build. The spec runs the+ // same context guard (assertAllowedKubeContext) before its first kubectl call.globalSetup: undefined,projects: [{ name: 'chat-open-ab', testMatch: /chat-open-ab\.spec\.ts/ }],// Paced opens, 65 s rate-limit windows between launches and stateless suspension waits.diff --git a/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts b/desktop-app/test/e2e-playwright/qa-recorder-helpers.tsindex ec2b47863..d4a1538a4 100644--- a/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts+++ b/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts@@ -131,11 +131,17 @@ export function configuredHostRef(): string {* 1280x720 WebM. Returns the app handle and its first window page. Caller owns* closing the app (use finalizeRecording in a finally block) so the video file* is finalized on disk.+ * `launch.executablePath` pins the Electron binary; `launch.beforeFirstWindow`+ * receives the app before its first window opens (to attach output streams).*/export async function launchDesktopApp(testInfo: TestInfo,isolationTarget?: string,- mainEntry: string = MAIN_ENTRY+ mainEntry: string = MAIN_ENTRY,+ launch: {+ executablePath?: string+ beforeFirstWindow?: (app: ElectronApplication) => void+ } = {}): Promise<{app: ElectronApplicationpage: Page@@ -153,6 +159,7 @@ export async function launchDesktopApp(const slowMo = Number(process.env.QA_RECORDER_SLOW_MO_MS ?? 75)const app = await electron.launch({+ executablePath: launch.executablePath,args: [`--user-data-dir=${userDataDir}`, mainEntry],env: {...process.env,@@ -188,6 +195,7 @@ export async function launchDesktopApp(slowMo: Number.isFinite(slowMo) ? slowMo : 75,})+ launch.beforeFirstWindow?.(app)const actualUserDataDir = await app.evaluate(({ app: electronApp }) =>electronApp.getPath('userData'))@@ -281,14 +289,8 @@ export async function openAgentsPage(page: Page): Promise<void> {/*** Navigate to the exact agent named by hostRef and land on its chat composer.* Uses exact identity (no first-available fallback), per the recorder contract.- * The UI shows the agent's display name; pass it as `label` when it differs- * from hostRef.*/-export async function openExactAgentChat(- page: Page,- hostRef: string,- label: string = hostRef-): Promise<void> {+export async function openExactAgentChat(page: Page, hostRef: string): Promise<void> {const settingsMenu = page.getByTestId('nav-settings-menu')if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') {await settingsMenu.click()@@ -299,7 +301,7 @@ export async function openExactAgentChat(await expect(agentsItem).toBeVisible({ timeout: 15_000 })await agentsItem.click()const chatInput = page.getByRole('textbox', { name: 'Agent message composer' })- const exactAgent = page.locator('.agents-table-row-clickable', { hasText: label }).first()+ const exactAgent = page.locator('.agents-table-row-clickable', { hasText: hostRef }).first()const emptyState = page.getByText('No agents available')await expect(chatInput.or(exactAgent).or(emptyState)).toBeVisible({ timeout: 20_000 })@@ -317,16 +319,16 @@ export async function openExactAgentChat(if (!(await chatInput.isVisible().catch(() => false)) ||!(await switchAgent- .getByText(label, { exact: true })+ .getByText(hostRef, { exact: true }).isVisible().catch(() => false))) {await switchAgent.click()- await page.getByRole('menuitem', { name: label, exact: true }).click()+ await page.getByRole('menuitem', { name: hostRef, exact: true }).click()}await expect(chatInput).toBeVisible({ timeout: 20_000 })- await expect(switchAgent).toContainText(label)+ await expect(switchAgent).toContainText(hostRef)}/** Send a chat message and wait for a non-empty assistant response. */diff --git a/desktop-app/test/globalSetupGuard.test.ts b/desktop-app/test/globalSetupGuard.test.tsindex 9b4c979a8..687f0001a 100644--- a/desktop-app/test/globalSetupGuard.test.ts+++ b/desktop-app/test/globalSetupGuard.test.ts@@ -1,5 +1,11 @@import { describe, expect, it } from 'vitest'-import { validateBaseUrls } from './e2e-playwright/global-setup'+import {+ ALLOWED_CONTEXTS,+ assertAllowedKubeContext,+ isLocalPortForwardContext,+ isProductionContext,+ validateBaseUrls,+} from './e2e-playwright/global-setup'const DEV_CONTEXT = 'gke_${GCP_PROJECT}_us-central1-a_example-dev'@@ -118,3 +124,27 @@ describe('validateBaseUrls', () => {).toThrow(/requires non-localhost URLs/)})})++describe('assertAllowedKubeContext', () => {+ it('accepts a branch profile and refuses production with the same call', () => {+ // Positive control: the guard returns for the context the A/B benchmark actually uses.+ expect(isLocalPortForwardContext('clerum-fix-issue-1091-chat-open-latency-d7ddc3ab')).toBe(true)+ expect(() =>+ assertAllowedKubeContext(+ 'clerum-fix-issue-1091-chat-open-latency-d7ddc3ab',+ 'CHAT_OPEN_AB_KUBE_CONTEXT'+ )+ ).not.toThrow()+ expect(isProductionContext('gke_example_us-central1-a_clerum-prod')).toBe(true)+ expect(() =>+ assertAllowedKubeContext('gke_example_us-central1-a_clerum-prod', 'CHAT_OPEN_AB_KUBE_CONTEXT')+ ).toThrow(/Production context .* is hard-blocked/)+ })++ it('refuses a context that is neither a local profile nor allow-listed', () => {+ expect(ALLOWED_CONTEXTS.has(DEV_CONTEXT)).toBe(false)+ expect(() => assertAllowedKubeContext(DEV_CONTEXT, 'CHAT_OPEN_AB_KUBE_CONTEXT')).toThrow(+ /CHAT_OPEN_AB_KUBE_CONTEXT=.* is not in the allow-list/+ )+ })+})