Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(desktop): make the chat-open A/B lane fail closed and measure what it claims

Alfredo Lopezd799cd045a7c
diff --git a/desktop-app/scripts/bench-chat-open/fixtures.mjs b/desktop-app/scripts/bench-chat-open/fixtures.mjs
index baee73f2e..dc4b8033e 100644
--- a/desktop-app/scripts/bench-chat-open/fixtures.mjs
+++ b/desktop-app/scripts/bench-chat-open/fixtures.mjs
@@ -1,29 +1,24 @@
import assert from 'node:assert/strict'
import { promises as fs } from 'node:fs'
import { join, sep } from 'node:path'
+import {
+ AUTHORITY_SCOPE,
+ FULL_CHAT,
+ PARTIAL_MESSAGES,
+ TARGET_AGENT,
+} from '../../test/e2e-playwright/helpers/chatCacheFixture.mjs'
-export const AUTHORITY_SCOPE = Object.freeze({
- environmentKey: 'benchmark-environment',
- userId: 'benchmark-user',
- teamId: 'benchmark-team',
-})
-export const TARGET_AGENT = 'agent-000'
-export const FULL_CHAT = 'chat-0000'
-export const PARTIAL_CHAT = 'chat-0001'
-export const PAGE_SIZE = 100
-export const PARTIAL_SIZE = 37
-
-function messages(count) {
- return Array.from({ length: count }, (_, index) => ({
- id: `message-${index}`,
- role: index % 2 === 0 ? 'user' : 'assistant',
- content: 'Synthetic benchmark message.',
- timestamp: 1_700_000_000_000 + index,
- }))
-}
-
-export const FULL_MESSAGES = messages(PAGE_SIZE)
-export const PARTIAL_MESSAGES = messages(PARTIAL_SIZE)
+export {
+ AUTHORITY_SCOPE,
+ FULL_CHAT,
+ FULL_MESSAGES,
+ PAGE_SIZE,
+ PARTIAL_CHAT,
+ PARTIAL_MESSAGES,
+ PARTIAL_SIZE,
+ TARGET_AGENT,
+ createFixture,
+} from '../../test/e2e-playwright/helpers/chatCacheFixture.mjs'
export function captureFilesystemFailures(root, failCleanup) {
const originalReaddir = fs.readdir
@@ -57,103 +52,6 @@ export function captureFilesystemFailures(root, failCleanup) {
}
}
-export async function createFixture({
- runDir,
- adapter,
- ChatStore,
- agents,
- chatsPerAgent,
- pendingCleanupsPerAgent,
-}) {
- const root = await fs.mkdtemp(join(runDir, 'data-'))
- const agentNames = Array.from(
- { length: agents },
- (_, index) => `agent-${String(index).padStart(3, '0')}`
- )
- const tombstones = Array.from({ length: pendingCleanupsPerAgent }, (_, index) => ({
- chatId: `deleted-${index}`,
- authorityScope: { ...AUTHORITY_SCOPE },
- }))
- const chats = Array.from({ length: chatsPerAgent }, (_, index) => ({
- id: `chat-${String(index).padStart(4, '0')}`,
- title: 'Synthetic benchmark chat',
- createdAt: '2024-01-01T00:00:00.000Z',
- updatedAt: '2024-01-01T00:00:00.000Z',
- messageCount: index === 0 ? PAGE_SIZE : index === 1 ? PARTIAL_SIZE : 0,
- errorCount: 0,
- toolCallCount: 0,
- }))
- const index = {
- version: 2,
- lastActiveChatId: PARTIAL_CHAT,
- onboardingDismissed: false,
- chats,
- ...(tombstones.length
- ? { deletedChatTombstones: tombstones, pendingChatCleanup: tombstones }
- : {}),
- }
- const catalog = JSON.stringify(index, null, 2)
- const emptyChatCatalog = JSON.stringify(
- {
- ...index,
- chats: chats.map(chat => ({ ...chat, messageCount: 0 })),
- },
- null,
- 2
- )
- for (const agent of agentNames) {
- const agentDir = join(root, agent)
- await fs.mkdir(join(agentDir, '.corrupt'), { recursive: true, mode: 0o700 })
- await fs.writeFile(
- join(agentDir, 'index.json'),
- agent === TARGET_AGENT ? catalog : emptyChatCatalog,
- { mode: 0o600 }
- )
- for (const tombstone of tombstones) {
- const chatDir = join(agentDir, 'chats', tombstone.chatId)
- await fs.mkdir(chatDir, { recursive: true, mode: 0o700 })
- await fs.writeFile(
- join(chatDir, 'meta.json'),
- JSON.stringify({
- version: 1,
- chatId: tombstone.chatId,
- pageSize: PAGE_SIZE,
- messageCount: 0,
- localMessageCount: 0,
- pages: [],
- }),
- { mode: 0o600 }
- )
- await fs.writeFile(
- join(agentDir, `${tombstone.chatId}.json`),
- JSON.stringify({
- version: 2,
- chatId: tombstone.chatId,
- messages: [],
- }),
- { mode: 0o600 }
- )
- }
- }
- // Use each version's own writer for valid page/compatibility metadata. Other
- // catalog entries are empty chats: their transcripts are not opened by this bench.
- const seedStore = adapter.bind(ChatStore, root)
- await seedStore.saveMessages(TARGET_AGENT, FULL_CHAT, FULL_MESSAGES)
- await seedStore.saveMessages(TARGET_AGENT, PARTIAL_CHAT, PARTIAL_MESSAGES)
- // Seeding updates activity/counts. Restore identical catalogs before a fresh bind.
- await fs.writeFile(join(root, TARGET_AGENT, 'index.json'), catalog, { mode: 0o600 })
- assert.equal(
- JSON.parse(await fs.readFile(join(root, TARGET_AGENT, 'index.json'), 'utf8')).chats.length,
- chatsPerAgent
- )
- if (tombstones.length) {
- assert.ok(
- (await fs.stat(join(root, TARGET_AGENT, 'chats', tombstones[0].chatId))).isDirectory()
- )
- }
- return { root, agentNames, tombstones, catalogBytes: Buffer.byteLength(catalog) }
-}
-
export async function inspectFixture(fixture, chatsPerAgent) {
let pending = 0
let retainedArtifactDirectories = 0
diff --git a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
index 403cd01b0..63807d9a7 100644
--- a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
+++ b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
@@ -1,14 +1,15 @@
import { type ElectronApplication, type Page, expect, test } from '@playwright/test'
-import { execFileSync } from 'node:child_process'
+import { execFileSync, spawnSync } from 'node:child_process'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
-import { createFixture } from '../../scripts/bench-chat-open/fixtures.mjs'
import { ChatStore } from '../../src/chatStore'
import { isPathInside } from '../../src/devIsolation'
import { assertSafeFilesystemSegment } from '../../src/pathSafety'
import { type ChatAuthorityScope } from '../../src/types'
import { SIDEBAR_SESSION_PREVIEW_LIMIT as SIDEBAR_PREVIEW } from '../../ui/src/constants/navigation'
+import { assertAllowedKubeContext } from './global-setup'
+import { createFixture } from './helpers/chatCacheFixture.mjs'
import {
type HostSnapshot,
readHostSnapshot,
@@ -59,7 +60,12 @@ type Sample = {
blank?: boolean
/** What the observer saw just before the click and on its first callbacks after it. */
trace?: OpenObservation[]
+ /** Relaunch metrics: observed before the test's first pointer event in the window. */
+ before_first_click?: boolean
}
+/** Runs a measured window; the Host snapshot loop of the round is paused while it runs. */
+type Timed = <T>(work: () => Promise<T>) => Promise<T>
+const untimed: Timed = work => work()
type OpenObservation = {
ms: number
lists: number
@@ -76,6 +82,10 @@ const LOADING_SKELETON = '[aria-label="Loading conversation"]'
const TRACE_LIMIT = 8
const LIMIT_WINDOW_MS = 65_000
const RATE_LIMITED = /\b429\b|Too Many Requests/g
+// Relaunch rounds per build in L and W.
+const RELAUNCH_ROUNDS = 3
+// A live Desktop from an earlier A/B run would hold the Keychain item this run needs.
+const AB_PROCESS = /Evenfire Issue1091Ab/
const rows = (host: string) =>
`[data-testid="latest-session-row"][data-agent-ref=${JSON.stringify(host)}]`
const row = (host: string, id: string) => `${rows(host)}[data-chat-id=${JSON.stringify(id)}]`
@@ -114,10 +124,12 @@ async function openChat(
page: Page,
host: string,
label: string,
- chat: Chat
+ chat: Chat,
+ timed: Timed
): Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }> {
+ // A deadline, not a fixed sleep: wait only for what is left of the pace since the last open.
const pace = lastOpenAt + OPEN_PACE_MS - Date.now()
- if (pace > 0) await page.waitForTimeout(pace)
+ if (pace > 0) await new Promise(resolve => setTimeout(resolve, pace))
lastOpenAt = Date.now()
await page
.getByRole('navigation', { name: 'Chat breadcrumb', exact: true })
@@ -151,77 +163,84 @@ async function openChat(
}
)
.toBe(true)
- return page.evaluate(
- ({ selector, chat, messages, skeleton, traceLimit }) =>
- new Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }>((resolve, reject) => {
- const button = document.querySelector<HTMLButtonElement>(`${selector} button`)
- if (!button || button.disabled)
- throw new Error(`Chat row has no enabled open button: ${selector}`)
- let started = performance.now()
- let clicked = false
- let blank = false
- const trace: OpenObservation[] = []
- // Before the click the Chat breadcrumb shows the Host's chat list with no active chat, which
- // already has zero bubbles; only the loading skeleton marks a chat that was opened empty.
- const observe = () => {
- const transcripts = document.querySelectorAll('[data-testid="message-list"]')
- const observation = {
- ms: clicked ? performance.now() - started : -1,
- lists: transcripts.length,
- bubbles: transcripts[0]?.querySelectorAll(messages).length ?? 0,
- skeleton: document.querySelector(skeleton) !== null,
- dedicated: transcripts[0]?.classList.contains('chat-thread-dedicated') ?? false,
+ return timed(() =>
+ page.evaluate(
+ ({ selector, chat, messages, skeleton, traceLimit }) =>
+ new Promise<{ ms: number; blank: boolean; trace: OpenObservation[] }>((resolve, reject) => {
+ const button = document.querySelector<HTMLButtonElement>(`${selector} button`)
+ if (!button || button.disabled)
+ throw new Error(`Chat row has no enabled open button: ${selector}`)
+ let blank = false
+ const trace: OpenObservation[] = []
+ // Before the click the Chat breadcrumb shows the Host's chat list with no active chat, which
+ // already has zero bubbles; only the loading skeleton marks a chat that was opened empty.
+ const observe = (ms: number) => {
+ const transcripts = document.querySelectorAll('[data-testid="message-list"]')
+ const observation = {
+ ms,
+ lists: transcripts.length,
+ bubbles: transcripts[0]?.querySelectorAll(messages).length ?? 0,
+ skeleton: document.querySelector(skeleton) !== null,
+ dedicated: transcripts[0]?.classList.contains('chat-thread-dedicated') ?? false,
+ }
+ if (trace.length < traceLimit) trace.push(observation)
+ return observation
}
- if (trace.length < traceLimit) trace.push(observation)
- return observation
- }
- observe()
- // The sidebar previews only the latest sessions, so the active-row witness is checked outside the timing.
- const finish = () => {
- const transcripts = document.querySelectorAll('[data-testid="message-list"]')
- if (transcripts.length > 1) {
- observer.disconnect()
- clearTimeout(deadline)
- reject(new Error(`Expected one transcript, found ${transcripts.length}.`))
- return
+ observe(-1)
+ // The sidebar previews only the latest sessions, so the active-row witness is checked outside the timing.
+ let scheduled = false
+ const finish = () => {
+ scheduled = false
+ const transcripts = document.querySelectorAll('[data-testid="message-list"]')
+ if (transcripts.length > 1) {
+ observer.disconnect()
+ clearTimeout(deadline)
+ reject(new Error(`Expected one transcript, found ${transcripts.length}.`))
+ return
+ }
+ const transcript = transcripts[0]
+ const bubbles = transcript?.querySelectorAll(messages)
+ if (observe(performance.now() - started).skeleton) blank = true
+ if (
+ bubbles?.length === chat.count &&
+ bubbles[chat.count - 1]?.textContent?.includes(chat.lastText) &&
+ transcript &&
+ transcript.getBoundingClientRect().height > 0
+ ) {
+ observer.disconnect()
+ clearTimeout(deadline)
+ resolve({ ms: performance.now() - started, blank, trace })
+ }
}
- const transcript = transcripts[0]
- const bubbles = transcript?.querySelectorAll(messages)
- if (observe().skeleton) blank = true
- if (
- bubbles?.length === chat.count &&
- bubbles[chat.count - 1]?.textContent?.includes(chat.lastText) &&
- transcript &&
- transcript.getBoundingClientRect().height > 0
- ) {
+ // One check per animation frame: a mutation only schedules it, so layout is read once
+ // per frame instead of once per mutation. Callbacks run after the click below sets `started`.
+ const observer = new MutationObserver(() => {
+ if (scheduled) return
+ scheduled = true
+ requestAnimationFrame(finish)
+ })
+ observer.observe(document.documentElement, {
+ childList: true,
+ subtree: true,
+ characterData: true,
+ attributes: true,
+ })
+ const deadline = setTimeout(() => {
observer.disconnect()
- clearTimeout(deadline)
- resolve({ ms: performance.now() - started, blank, trace })
- }
- }
- const observer = new MutationObserver(finish)
- observer.observe(document.documentElement, {
- childList: true,
- subtree: true,
- characterData: true,
- attributes: true,
- })
- const deadline = setTimeout(() => {
- observer.disconnect()
- reject(new Error(`Transcript did not match server chat ${chat.id}`))
- }, 120_000)
- started = performance.now()
- button.click()
- clicked = true
- finish()
- }),
- {
- selector: agentRow(host, chat.id),
- chat,
- messages: MESSAGES,
- skeleton: LOADING_SKELETON,
- traceLimit: TRACE_LIMIT,
- }
+ reject(new Error(`Transcript did not match server chat ${chat.id}`))
+ }, 120_000)
+ const started = performance.now()
+ button.click()
+ finish()
+ }),
+ {
+ selector: agentRow(host, chat.id),
+ chat,
+ messages: MESSAGES,
+ skeleton: LOADING_SKELETON,
+ traceLimit: TRACE_LIMIT,
+ }
+ )
)
}
async function relaunchPaint(
@@ -237,6 +256,7 @@ async function relaunchPaint(
ms: number
observed_at_install: boolean
observer_installed_ms: number
+ before_first_click: boolean
}>
>((resolve, reject) => {
const navigation = performance.getEntriesByType('navigation')[0] as
@@ -244,12 +264,23 @@ async function relaunchPaint(
| undefined
if (!navigation) throw new Error('Window navigation timing is unavailable.')
const installed = performance.now()
- const observed = new Map<string, { at: number; initial: boolean }>()
+ const observed = new Map<string, { at: number; initial: boolean; beforeClick: boolean }>()
let initial = true
+ // Witness for metrics that must be observed before the test clicks anything.
+ let clicked = false
+ document.addEventListener(
+ 'pointerdown',
+ () => {
+ clicked = true
+ },
+ { capture: true, once: true }
+ )
+ let scheduled = false
const visible = (element: Element) =>
element.getBoundingClientRect().height > 0 &&
getComputedStyle(element).visibility !== 'hidden'
const finish = () => {
+ scheduled = false
const transcripts = document.querySelectorAll('[data-testid="message-list"]')
if (transcripts.length > 1) {
observer.disconnect()
@@ -269,7 +300,11 @@ async function relaunchPaint(
.querySelectorAll(messages)
[target.chat.count - 1]?.textContent?.includes(target.chat.lastText)))
if (ready && !observed.has(target.metric))
- observed.set(target.metric, { at: performance.now(), initial })
+ observed.set(target.metric, {
+ at: performance.now(),
+ initial,
+ beforeClick: !clicked,
+ })
}
if (observed.size === targets.length && navigation.loadEventEnd > 0) {
observer.disconnect()
@@ -283,13 +318,19 @@ async function relaunchPaint(
ms: Math.max(0, result.at - navigation.loadEventEnd),
observed_at_install: result.initial,
observer_installed_ms: installed - navigation.loadEventEnd,
+ before_first_click: result.beforeClick,
}
})
)
}
initial = false
}
- const observer = new MutationObserver(finish)
+ // One check per animation frame, as in openChat.
+ const observer = new MutationObserver(() => {
+ if (scheduled) return
+ scheduled = true
+ requestAnimationFrame(finish)
+ })
observer.observe(document.documentElement, {
childList: true,
subtree: true,
@@ -331,23 +372,28 @@ if (
const context = required('CHAT_OPEN_AB_KUBE_CONTEXT')
if (context !== context.trim() || context.startsWith('-'))
throw new Error('CHAT_OPEN_AB_KUBE_CONTEXT must be the literal Kubernetes context.')
+ // The shared global setup is disabled for this config; apply its context guard before any kubectl call.
+ assertAllowedKubeContext(context, 'CHAT_OPEN_AB_KUBE_CONTEXT')
const output = required('CHAT_OPEN_AB_OUTPUT_JSON')
if (!path.isAbsolute(output) || path.extname(output) !== '.json')
throw new Error('CHAT_OPEN_AB_OUTPUT_JSON must be an absolute JSON path.')
const phase = required('CHAT_OPEN_AB_PHASE')
if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')
const builds = Object.fromEntries(
- Object.entries(entries).map(([build, entry]) => [
- build,
- {
- entry,
- sha: execFileSync(
- 'git',
- ['-C', path.resolve(path.dirname(entry), '../..'), 'rev-parse', 'HEAD'],
- { encoding: 'utf8' }
- ).trim(),
- },
- ])
+ Object.entries(entries).map(([build, entry]) => {
+ const git = (...args: string[]) =>
+ execFileSync('git', ['-C', path.resolve(path.dirname(entry), '../..'), ...args], {
+ encoding: 'utf8',
+ }).trim()
+ // A build must be exactly its commit: a clean tree, and dist/main.js built after the commit.
+ const dirty = git('status', '--porcelain')
+ if (dirty) throw new Error(`precondition: the ${build} worktree has uncommitted changes.`)
+ const committedAt = Number(git('log', '-1', '--format=%ct')) * 1000
+ const builtAt = fs.statSync(entry).mtimeMs
+ if (!(builtAt > committedAt))
+ throw new Error(`precondition: the ${build} build is older than its HEAD commit.`)
+ return [build, { entry, sha: git('rev-parse', 'HEAD'), committedAt, builtAt }]
+ })
)
const samples: Sample[] = []
const rounds: Array<{
@@ -393,27 +439,84 @@ if (
closedAt = Date.now()
}
}
+ // One app identity per run, so Electron creates its own `<appName> Safe Storage` Keychain item
+ // instead of reading one an earlier run left behind. BEFORE and AFTER share it and the user data.
+ const isolationTarget = `issue-1091-ab-${Date.now().toString(36)}`
+ const electronPath = fs.realpathSync(require('electron') as unknown as string)
let launches = 0
let mainLog = ''
+ // The current launch's main-process output, kept in memory so a round reads all of it.
+ let mainOutput: string[] = []
let launchedBuild: Build | undefined
- const launch = async (build: Build) => {
+ // A first window that never opens is usually a Keychain prompt nobody answers. Record what the
+ // process was doing and whether macOS shows a prompt; never answer or bypass it.
+ const diagnoseFirstWindow = (starting: ElectronApplication): string => {
+ const samplePath = testInfo.outputPath(`sample-${launches}.txt`)
+ execFileSync('sample', [String(starting.process().pid), '5', '-file', samplePath])
+ const agent = spawnSync('pgrep', ['-x', 'SecurityAgent'])
+ if (agent.status !== 0 && agent.status !== 1)
+ throw new Error(`pgrep SecurityAgent failed with status ${agent.status}.`)
+ return agent.status === 0
+ ? `KEYCHAIN_PROMPT_PENDING: SecurityAgent is running; sample in ${samplePath}`
+ : `FIRST_WINDOW_TIMEOUT: no SecurityAgent process; sample in ${samplePath}`
+ }
+ // A closed build's background requests can exhaust control-api's per-minute
+ // limits; the next build's session restore then fails. Start each launch in
+ // a fresh window, outside every timed section.
+ const awaitQuietWindow = async () => {
await close()
- // A closed build's background requests can exhaust control-api's per-minute
- // limits; the next build's session restore then fails. Start each launch in
- // a fresh window, outside every timed section.
const quiet = closedAt + LIMIT_WINDOW_MS - Date.now()
if (quiet > 0) await new Promise(resolve => setTimeout(resolve, quiet))
- const launched = await launchDesktopApp(testInfo, 'issue-1091-ab', entries[build])
- app = launched.app
- page = launched.page
- launchedBuild = build
+ }
+ const launch = async (build: Build) => {
+ await awaitQuietWindow()
+ const live = execFileSync('ps', ['-axo', 'pid=,args='], { encoding: 'utf8' })
+ .split('\n')
+ .filter(line => AB_PROCESS.test(line))
+ .map(line => line.trim().split(/\s+/)[0])
+ if (live.length)
+ throw new Error(`precondition: an A/B Desktop is still running (pids ${live.join(', ')}).`)
// Keep each main process's output so a lost session can be explained after the run.
mainLog = testInfo.outputPath(`main-${++launches}-${build}.log`)
- const log = fs.createWriteStream(mainLog, {
- mode: 0o600,
- })
- app.process().stdout?.pipe(log)
- app.process().stderr?.pipe(log)
+ const log = fs.createWriteStream(mainLog, { mode: 0o600 })
+ const output: string[] = []
+ mainOutput = output
+ let starting: ElectronApplication | undefined
+ try {
+ const launched = await launchDesktopApp(testInfo, isolationTarget, entries[build], {
+ executablePath: electronPath,
+ beforeFirstWindow: started => {
+ starting = started
+ for (const stream of [started.process().stdout, started.process().stderr])
+ stream?.on('data', (chunk: Buffer) => {
+ output.push(chunk.toString('utf8'))
+ log.write(chunk)
+ })
+ },
+ })
+ app = launched.app
+ page = launched.page
+ } catch (error) {
+ const diagnosis =
+ starting && /firstWindow: Timeout/.test(String((error as Error)?.message))
+ ? diagnoseFirstWindow(starting)
+ : undefined
+ await starting?.close()
+ if (diagnosis) throw new Error(diagnosis, { cause: error })
+ throw error
+ }
+ launchedBuild = build
+ // Public process identities only: the pinned Electron, this build's code and the run's profile.
+ const runtime = await app.evaluate(({ app: electronApp }) => ({
+ executablePath: process.execPath,
+ appPath: electronApp.getAppPath(),
+ userData: electronApp.getPath('userData'),
+ }))
+ expect(fs.realpathSync(runtime.executablePath)).toBe(electronPath)
+ expect(fs.realpathSync(runtime.appPath)).toBe(fs.realpathSync(path.dirname(entries[build])))
+ expect(fs.realpathSync(runtime.userData)).toBe(
+ fs.realpathSync(path.join(runDir, 'user-data'))
+ )
}
const authenticated = async () => {
try {
@@ -479,7 +582,13 @@ if (
),
}
}
- const round = async (scenario: string, build: Build, action: (id: number) => Promise<void>) => {
+ // `timed` wraps every measured window; the Host snapshots (each a few kubectl processes)
+ // run only between those windows, so they never compete with a measurement.
+ const round = async (
+ scenario: string,
+ build: Build,
+ action: (id: number, timed: Timed) => Promise<void>
+ ) => {
const record: (typeof rounds)[number] = {
id: rounds.length + 1,
scenario,
@@ -491,19 +600,36 @@ if (
}
rounds.push(record)
let stopped = false
+ let paused = false
+ let sampling: Promise<void> = Promise.resolve()
let observationFailure: { error: unknown } | undefined
const observation = (async () => {
while (!stopped) {
- record.snapshots.push(await snapshot())
+ if (!paused) {
+ sampling = snapshot().then(state => {
+ record.snapshots.push(state)
+ })
+ await sampling
+ }
if (!stopped) await new Promise(resolve => setTimeout(resolve, 250))
}
})().catch(error => {
observationFailure = { error }
})
+ const timed: Timed = async work => {
+ paused = true
+ try {
+ // A snapshot already running finishes before the measured window opens.
+ await sampling
+ return await work()
+ } finally {
+ paused = false
+ }
+ }
try {
- await action(record.id)
+ await action(record.id, timed)
// A rate-limited round runs on stale or empty catalogs, so its timings are not valid samples.
- const limited = fs.readFileSync(mainLog, 'utf8').match(RATE_LIMITED)?.length ?? 0
+ const limited = mainOutput.join('').match(RATE_LIMITED)?.length ?? 0
if (limited)
throw new Error(
`Round ${record.id} (${scenario} ${build}) logged ${limited} rate-limited responses in ${path.basename(mainLog)}.`
@@ -671,7 +797,7 @@ if (
for (const build of ['before', 'after', 'after', 'before'] as const) {
await ready()
await test.step(`O: ${build} cached chat opens`, () =>
- round('O', build, async id => {
+ round('O', build, async (id, timed) => {
await launch(build)
await authenticated()
await page.getByTestId('nav-chat').click()
@@ -721,7 +847,8 @@ if (
page,
catalog.host,
hostLabels[catalog.host]!,
- chat
+ chat,
+ timed
)
samples.push({
round: id,
@@ -753,8 +880,8 @@ if (
// The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.
// An empty preview would satisfy "none is active" without checking anything.
await expect(
- page.locator('[data-testid="latest-session-row"]').first()
- ).toBeVisible()
+ page.locator('[data-testid="latest-session-row"]')
+ ).not.toHaveCount(0)
const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0
await expect(
page.locator('[data-testid="latest-session-row"][aria-current="true"]')
@@ -786,7 +913,7 @@ if (
throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)
return at > updatedAt.get(latest.id)! ? chat : latest
})
- await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
+ await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected, untimed)
await expect
.poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0]))
.toBe(selected.id)
@@ -805,31 +932,39 @@ if (
isolatedPath(runDir, path.join(savedCache, agent)),
{ recursive: true }
)
+ // L: the large cache is in place and the app relaunches. Desktop does not reopen the last
+ // active chat on launch, so the user opens it: relaunch_rows_ms ends at the first stateful
+ // sidebar row (before any click), relaunch_ms at the transcript of the explicit open.
for (const build of ['before', 'after'] as const)
- for (let iteration = 0; iteration < 3; iteration++) {
+ for (let iteration = 0; iteration < RELAUNCH_ROUNDS; iteration++) {
await ready()
await close()
fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })
fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })
await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>
- round('L', build, async id => {
- await launch(build)
- const [paint] = await Promise.all([
- relaunchPaint(page, [
- {
- host: HOSTS[0],
- metric: 'relaunch_ms',
- selector: row(HOSTS[0], selected.id),
- chat: selected,
- },
- ]),
- (async () => {
- await authenticated()
- await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()
- await page.getByTestId('nav-chat').click()
- await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
- })(),
- ])
+ round('L', build, async (id, timed) => {
+ await awaitQuietWindow()
+ const paint = await timed(async () => {
+ await launch(build)
+ const [observed] = await Promise.all([
+ relaunchPaint(page, [
+ { host: HOSTS[0], metric: 'relaunch_rows_ms', selector: rows(HOSTS[0]) },
+ {
+ host: HOSTS[0],
+ metric: 'relaunch_ms',
+ selector: row(HOSTS[0], selected.id),
+ chat: selected,
+ },
+ ]),
+ (async () => {
+ await authenticated()
+ await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()
+ await page.getByTestId('nav-chat').click()
+ await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected, untimed)
+ })(),
+ ])
+ return observed
+ })
for (const sample of paint) samples.push({ round: id, build, ...sample })
await authenticated()
}))
@@ -837,24 +972,27 @@ if (
}
if (phase === 'W')
for (const build of ['before', 'after'] as const)
- for (let iteration = 0; iteration < 3; iteration++) {
+ for (let iteration = 0; iteration < RELAUNCH_ROUNDS; iteration++) {
await close()
await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])
// Empty only the isolated chat cache: both observed rows must come from the remote catalog.
if (fs.existsSync(cacheDir!))
fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })
await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>
- round('W', build, async id => {
- await launch(build)
- for (const sample of await relaunchPaint(
- page,
- HOSTS.map((host, index) => ({
- host,
- selector: rows(host),
- metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',
- }))
- ))
- samples.push({ round: id, build, ...sample })
+ round('W', build, async (id, timed) => {
+ await awaitQuietWindow()
+ const paint = await timed(async () => {
+ await launch(build)
+ return relaunchPaint(
+ page,
+ HOSTS.map((host, index) => ({
+ host,
+ selector: rows(host),
+ metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',
+ }))
+ )
+ })
+ for (const sample of paint) samples.push({ round: id, build, ...sample })
await authenticated()
await ready()
}))
@@ -926,16 +1064,63 @@ if (
witness.before_reopen_blank === values('before', host, 'open_memo_ms').length,
})
}
- const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,
- after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50
- report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })
+ // Every relaunch round of both builds contributes one sample per L metric.
+ const relaunchSamples = (build: Build, metric: string) => {
+ const found = values(build, HOSTS[0], metric)
+ if (found.length !== RELAUNCH_ROUNDS)
+ throw new Error(`L recorded ${found.length} ${build} ${metric} samples.`)
+ return found
+ }
+ // relaunch_rows_ms: window load to the first stateful sidebar row, observed before the
+ // test clicks anything; the non-blocking chat:getIndex is what shortens it.
+ const rowsUnclicked = samples
+ .filter(sample => sample.metric === 'relaunch_rows_ms')
+ .every(sample => sample.before_first_click === true)
+ const rowsBefore = quantiles(relaunchSamples('before', 'relaunch_rows_ms')).p50,
+ rowsAfter = quantiles(relaunchSamples('after', 'relaunch_rows_ms')).p50
+ report.gates.push({
+ metric: 'relaunch_rows_ms',
+ definition: 'window load -> first stateful sidebar row, before any click',
+ before: rowsBefore,
+ after: rowsAfter,
+ observed_before_click: rowsUnclicked,
+ passed: rowsUnclicked && rowsAfter < rowsBefore,
+ })
+ // relaunch_ms: window load to the transcript of the selected chat, including the explicit
+ // open. Desktop never restores the last active chat on launch.
+ const before = quantiles(relaunchSamples('before', 'relaunch_ms')).p50,
+ after = quantiles(relaunchSamples('after', 'relaunch_ms')).p50
+ report.gates.push({
+ metric: 'relaunch_ms',
+ definition: 'window load -> sidebar row -> explicit open -> transcript with all messages',
+ before,
+ after,
+ passed: after < before,
+ })
} else {
- const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))
+ // W needs a control: BEFORE runs the same suspended-Host relaunches as AFTER.
+ const wake = (build: Build) => {
+ const found = values(build, HOSTS[0], 'first_stateful_row_ms')
+ if (found.length !== RELAUNCH_ROUNDS)
+ throw new Error(`W recorded ${found.length} ${build} first_stateful_row_ms samples.`)
+ return found
+ }
+ const wakeBefore = wake('before'),
+ wakeAfter = wake('after')
+ const afterMax = Math.max(...wakeAfter)
report.gates.push({
metric: 'first_stateful_row_ms',
- after_max_ms: wakeAfter,
+ after_max_ms: afterMax,
budget_ms: 2000,
- passed: wakeAfter < 2000,
+ passed: afterMax < 2000,
+ })
+ const beforeP50 = quantiles(wakeBefore).p50,
+ afterP50 = quantiles(wakeAfter).p50
+ report.gates.push({
+ metric: 'first_stateful_row_ms_vs_before',
+ before: beforeP50,
+ after: afterP50,
+ passed: afterP50 < beforeP50,
})
}
report.gates.push({
@@ -966,6 +1151,10 @@ if (
fs.writeFileSync(output, JSON.stringify({ ...report, catalogs }, null, 2), { mode: 0o600 })
}
}
+ // O: three gates per Host; L: two; W: two; both phases: round validity.
+ expect(report.gates.length, 'Desktop A/B gate count').toBe(
+ phase === 'OL' ? HOSTS.length * 3 + 3 : 3
+ )
expect(
report.gates.every(gate => gate.passed),
`Desktop A/B gates: ${JSON.stringify(report.gates)}; evidence: ${output}`
diff --git a/desktop-app/test/e2e-playwright/global-setup.ts b/desktop-app/test/e2e-playwright/global-setup.ts
index 07479223a..74ba4b267 100644
--- a/desktop-app/test/e2e-playwright/global-setup.ts
+++ b/desktop-app/test/e2e-playwright/global-setup.ts
@@ -6,12 +6,12 @@ import { E2E_TEST_EMAIL } from '../../../tests/e2e/testUser'
const configuredAllowedContexts = process.env.E2E_ALLOWED_CONTEXTS?.split(',')
.map(context => context.trim())
.filter(Boolean)
-const ALLOWED_CONTEXTS = new Set(
+export const ALLOWED_CONTEXTS: ReadonlySet<string> = new Set(
configuredAllowedContexts?.length ? configuredAllowedContexts : ['clerum-test']
)
const PROD_CONTEXT = process.env.E2E_PROD_CONTEXT?.trim() || ''
-function isProductionContext(context: string): boolean {
+export function isProductionContext(context: string): boolean {
return (
(PROD_CONTEXT !== '' && context === PROD_CONTEXT) ||
/(^|[-_])(prod|production)([-_]|$)/i.test(context)
@@ -27,7 +27,7 @@ function isBranchProfileContext(context: string): boolean {
return /^clerum-.+-[0-9a-f]{7,8}$/.test(context)
}
-function isLocalPortForwardContext(context: string): boolean {
+export function isLocalPortForwardContext(context: string): boolean {
return (
context === 'clerum-test' ||
/^clerum-(codex|detached)-.+-[0-9a-f]{8}$/.test(context) ||
@@ -35,6 +35,21 @@ function isLocalPortForwardContext(context: string): boolean {
)
}
+/** Refuses production and any context that is neither a local port-forward profile nor allow-listed. */
+export function assertAllowedKubeContext(context: string, source: string): void {
+ if (isProductionContext(context)) {
+ throw new Error(
+ `[E2E-GUARD] Production context "${context}" is hard-blocked. E2E against prod is never allowed.`
+ )
+ }
+ if (!isLocalPortForwardContext(context) && !ALLOWED_CONTEXTS.has(context)) {
+ throw new Error(
+ `[E2E-GUARD] ${source}="${context}" is not in the allow-list. ` +
+ `Allowed: ${[...ALLOWED_CONTEXTS].join(', ')}.`
+ )
+ }
+}
+
function parseBooleanEnv(value: string | undefined): boolean {
// Keep this narrow on purpose so only the documented values ("1"/"true")
// enable dev port-forward mode in CI/E2E runs.
@@ -174,17 +189,7 @@ async function globalSetup(): Promise<void> {
process.env.K8S_CONTEXT ||
'clerum-test'
- if (isProductionContext(expected)) {
- throw new Error(
- `[E2E-GUARD] Production context "${expected}" is hard-blocked. E2E against prod is never allowed.`
- )
- }
- if (!isLocalPortForwardContext(expected) && !ALLOWED_CONTEXTS.has(expected)) {
- throw new Error(
- `[E2E-GUARD] E2E_K8S_CONTEXT="${expected}" is not in the allow-list. ` +
- `Allowed: ${[...ALLOWED_CONTEXTS].join(', ')}.`
- )
- }
+ assertAllowedKubeContext(expected, 'E2E_K8S_CONTEXT')
let current: string
try {
diff --git a/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjs b/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjs
new file mode 100644
index 000000000..34a3c09bc
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/helpers/chatCacheFixture.mjs
@@ -0,0 +1,125 @@
+import assert from 'node:assert/strict'
+import { promises as fs } from 'node:fs'
+import { join } from 'node:path'
+
+// A synthetic chat cache written through ChatStore: shared by the chat-open benchmark
+// (scripts/bench-chat-open) and the Desktop A/B spec, which uses it as the large cache.
+export const AUTHORITY_SCOPE = Object.freeze({
+ environmentKey: 'benchmark-environment',
+ userId: 'benchmark-user',
+ teamId: 'benchmark-team',
+})
+export const TARGET_AGENT = 'agent-000'
+export const FULL_CHAT = 'chat-0000'
+export const PARTIAL_CHAT = 'chat-0001'
+export const PAGE_SIZE = 100
+export const PARTIAL_SIZE = 37
+
+function messages(count) {
+ return Array.from({ length: count }, (_, index) => ({
+ id: `message-${index}`,
+ role: index % 2 === 0 ? 'user' : 'assistant',
+ content: 'Synthetic benchmark message.',
+ timestamp: 1_700_000_000_000 + index,
+ }))
+}
+
+export const FULL_MESSAGES = messages(PAGE_SIZE)
+export const PARTIAL_MESSAGES = messages(PARTIAL_SIZE)
+
+export async function createFixture({
+ runDir,
+ adapter,
+ ChatStore,
+ agents,
+ chatsPerAgent,
+ pendingCleanupsPerAgent,
+}) {
+ const root = await fs.mkdtemp(join(runDir, 'data-'))
+ const agentNames = Array.from(
+ { length: agents },
+ (_, index) => `agent-${String(index).padStart(3, '0')}`
+ )
+ const tombstones = Array.from({ length: pendingCleanupsPerAgent }, (_, index) => ({
+ chatId: `deleted-${index}`,
+ authorityScope: { ...AUTHORITY_SCOPE },
+ }))
+ const chats = Array.from({ length: chatsPerAgent }, (_, index) => ({
+ id: `chat-${String(index).padStart(4, '0')}`,
+ title: 'Synthetic benchmark chat',
+ createdAt: '2024-01-01T00:00:00.000Z',
+ updatedAt: '2024-01-01T00:00:00.000Z',
+ messageCount: index === 0 ? PAGE_SIZE : index === 1 ? PARTIAL_SIZE : 0,
+ errorCount: 0,
+ toolCallCount: 0,
+ }))
+ const index = {
+ version: 2,
+ lastActiveChatId: PARTIAL_CHAT,
+ onboardingDismissed: false,
+ chats,
+ ...(tombstones.length
+ ? { deletedChatTombstones: tombstones, pendingChatCleanup: tombstones }
+ : {}),
+ }
+ const catalog = JSON.stringify(index, null, 2)
+ const emptyChatCatalog = JSON.stringify(
+ {
+ ...index,
+ chats: chats.map(chat => ({ ...chat, messageCount: 0 })),
+ },
+ null,
+ 2
+ )
+ for (const agent of agentNames) {
+ const agentDir = join(root, agent)
+ await fs.mkdir(join(agentDir, '.corrupt'), { recursive: true, mode: 0o700 })
+ await fs.writeFile(
+ join(agentDir, 'index.json'),
+ agent === TARGET_AGENT ? catalog : emptyChatCatalog,
+ { mode: 0o600 }
+ )
+ for (const tombstone of tombstones) {
+ const chatDir = join(agentDir, 'chats', tombstone.chatId)
+ await fs.mkdir(chatDir, { recursive: true, mode: 0o700 })
+ await fs.writeFile(
+ join(chatDir, 'meta.json'),
+ JSON.stringify({
+ version: 1,
+ chatId: tombstone.chatId,
+ pageSize: PAGE_SIZE,
+ messageCount: 0,
+ localMessageCount: 0,
+ pages: [],
+ }),
+ { mode: 0o600 }
+ )
+ await fs.writeFile(
+ join(agentDir, `${tombstone.chatId}.json`),
+ JSON.stringify({
+ version: 2,
+ chatId: tombstone.chatId,
+ messages: [],
+ }),
+ { mode: 0o600 }
+ )
+ }
+ }
+ // Use each version's own writer for valid page/compatibility metadata. Other
+ // catalog entries are empty chats: their transcripts are not opened by this bench.
+ const seedStore = adapter.bind(ChatStore, root)
+ await seedStore.saveMessages(TARGET_AGENT, FULL_CHAT, FULL_MESSAGES)
+ await seedStore.saveMessages(TARGET_AGENT, PARTIAL_CHAT, PARTIAL_MESSAGES)
+ // Seeding updates activity/counts. Restore identical catalogs before a fresh bind.
+ await fs.writeFile(join(root, TARGET_AGENT, 'index.json'), catalog, { mode: 0o600 })
+ assert.equal(
+ JSON.parse(await fs.readFile(join(root, TARGET_AGENT, 'index.json'), 'utf8')).chats.length,
+ chatsPerAgent
+ )
+ if (tombstones.length) {
+ assert.ok(
+ (await fs.stat(join(root, TARGET_AGENT, 'chats', tombstones[0].chatId))).isDirectory()
+ )
+ }
+ return { root, agentNames, tombstones, catalogBytes: Buffer.byteLength(catalog) }
+}
diff --git a/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts b/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts
index ae076f638..f8e2cfa52 100644
--- a/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts
+++ b/desktop-app/test/e2e-playwright/helpers/hostSnapshot.ts
@@ -5,22 +5,43 @@ import { promisify } from 'node:util'
// E2E_GUARDIAN_IPC_FLOW: Host lifecycle is observed read-only through Kubernetes;
// Desktop RPC runs in the main process, outside renderer HTTP observation.
const execFileAsync = promisify(execFile)
-const STATELESS_NAMESPACE = 'mcp-host'
+const HOST_NAMESPACE = 'mcp-host'
-async function kubectl(context: string, args: string[]): Promise<string> {
+export async function kubectl(
+ context: string,
+ args: string[],
+ timeoutMs = 20_000
+): Promise<string> {
if (!context.trim() || context.startsWith('-'))
throw new Error('An explicit kube context is required.')
const { stdout } = await execFileAsync('kubectl', ['--context', context, ...args], {
- timeout: 20_000,
+ timeout: timeoutMs,
maxBuffer: 8 * 1024 * 1024,
})
return stdout.toString()
}
-async function jsonKubectl(context: string, resource: string): Promise<any> {
- return JSON.parse(
- await kubectl(context, ['-n', STATELESS_NAMESPACE, 'get', resource, '-o', 'json'])
- )
+export async function jsonKubectl<T>(
+ context: string,
+ namespace: string,
+ resource: string
+): Promise<T> {
+ return JSON.parse(await kubectl(context, ['-n', namespace, 'get', resource, '-o', 'json'])) as T
+}
+
+/** The fields of the Deployment, Host and Pod JSON that a snapshot reads. */
+type DeploymentJson = { spec?: { replicas?: number }; status?: { readyReplicas?: number } }
+type HostJson = {
+ status?: {
+ lifecycle?: { state?: string; reason?: string }
+ conditions?: Array<{ type?: string; status?: string; reason?: string }>
+ }
+}
+type PodListJson = {
+ items?: Array<{
+ metadata?: { name?: string }
+ status?: { containerStatuses?: Array<{ imageID?: string }> }
+ }>
}
export type HostSnapshot = {
@@ -39,39 +60,27 @@ export async function readHostSnapshot(
context: string
): Promise<HostSnapshot> {
const [deploymentJson, hostJson, podsJson] = await Promise.all([
- jsonKubectl(context, `deployment.apps/${deployment}`),
- jsonKubectl(context, `host/${hostName}`),
- kubectl(context, [
- '-n',
- STATELESS_NAMESPACE,
- 'get',
- 'pods',
- '-l',
- `app=${hostName}`,
- '-o',
- 'json',
- ]),
+ jsonKubectl<DeploymentJson>(context, HOST_NAMESPACE, `deployment.apps/${deployment}`),
+ jsonKubectl<HostJson>(context, HOST_NAMESPACE, `host/${hostName}`),
+ kubectl(context, ['-n', HOST_NAMESPACE, 'get', 'pods', '-l', `app=${hostName}`, '-o', 'json']),
])
- const pods = JSON.parse(podsJson).items ?? []
- const spec = deploymentJson.spec ?? {}
- const status = deploymentJson.status ?? {}
+ const pods = (JSON.parse(podsJson) as PodListJson).items ?? []
const lifecycle = hostJson.status?.lifecycle ?? {}
const pullPolicyRejection = (hostJson.status?.conditions ?? []).find(
- (condition: { type?: string; status?: string }) =>
- condition.type === 'StatelessPullPolicyRejected' && condition.status === 'True'
+ condition => condition.type === 'StatelessPullPolicyRejected' && condition.status === 'True'
)
return {
lifecycleState: String(lifecycle.state ?? ''),
lifecycleReason: String(lifecycle.reason ?? ''),
- replicas: Number(spec.replicas ?? -1),
- readyReplicas: Number(status.readyReplicas ?? 0),
+ replicas: Number(deploymentJson.spec?.replicas ?? -1),
+ readyReplicas: Number(deploymentJson.status?.readyReplicas ?? 0),
podNames: pods
- .map((pod: { metadata?: { name?: string } }) => pod.metadata?.name)
- .filter(Boolean)
+ .map(pod => pod.metadata?.name)
+ .filter((name): name is string => Boolean(name))
.sort(),
imageIds: [
...new Set<string>(
- pods.flatMap((pod: { status?: { containerStatuses?: Array<{ imageID?: string }> } }) =>
+ pods.flatMap(pod =>
(pod.status?.containerStatuses ?? [])
.map(container => container.imageID)
.filter((id): id is string => typeof id === 'string' && id.length > 0)
diff --git a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
index 085dab1a8..a1e44f03a 100644
--- a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
+++ b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
@@ -16,10 +16,10 @@
* `HUMAN_E2E_STATEFUL_HOST_RESOURCE`, `HUMAN_E2E_STATELESS_HOST_RESOURCE`.
*/
import { type ElectronApplication, type Locator, type Page, expect, test } from '@playwright/test'
-import { type ChildProcess, execFile } from 'node:child_process'
+import { type ChildProcess } from 'node:child_process'
import { writeFileSync } from 'node:fs'
-import { promisify } from 'node:util'
import {
+ jsonKubectl,
readHostSnapshot,
waitForStatelessReady,
waitForStatelessSuspended,
@@ -37,8 +37,6 @@ import {
screenshotAndLog,
} from './qa-recorder-helpers'
-const execFileAsync = promisify(execFile)
-
const KUBE_CONTEXT = process.env.HUMAN_E2E_KUBE_CONTEXT?.trim() || ''
const STATEFUL_HOST = process.env.HUMAN_E2E_STATEFUL_HOST?.trim() || 'chatLLM'
const STATELESS_HOST = process.env.HUMAN_E2E_STATELESS_HOST?.trim() || 'chatllm-stateless'
@@ -47,7 +45,6 @@ const STATELESS_HOST_RESOURCE =
process.env.HUMAN_E2E_STATELESS_HOST_RESOURCE?.trim() || 'chatllm-stateless'
const HCC_DEPLOYMENT = 'host-context-controller'
const HCC_NAMESPACE = 'control-plane'
-const STATELESS_NAMESPACE = 'mcp-host'
const STATELESS_DEPLOYMENT = 'chatllm-stateless'
const STATEFUL_DEPLOYMENT = 'chatllm'
const SUSPEND_TIMEOUT_MS = 240_000
@@ -103,28 +100,22 @@ type JourneyMetrics = {
}
}
-async function kubectl(args: string[], timeoutMs = 20_000): Promise<string> {
- const { stdout } = await execFileAsync('kubectl', ['--context', KUBE_CONTEXT, ...args], {
- timeout: timeoutMs,
- maxBuffer: 8 * 1024 * 1024,
- })
- return stdout.toString()
-}
-
-async function jsonKubectl(resource: string, namespace: string): Promise<any> {
- return JSON.parse(await kubectl(['-n', namespace, 'get', resource, '-o', 'json']))
-}
-
async function readHccCadences(): Promise<{
idleMinutes: number
idleFloorMinutes: number
drainGraceMs: number
heartbeatPollMs: number
}> {
- const deployment = await jsonKubectl(`deployment.apps/${HCC_DEPLOYMENT}`, HCC_NAMESPACE)
+ const deployment = await jsonKubectl<{
+ spec?: {
+ template?: {
+ spec?: { containers?: Array<{ env?: Array<{ name?: string; value?: string }> }> }
+ }
+ }
+ }>(KUBE_CONTEXT, HCC_NAMESPACE, `deployment.apps/${HCC_DEPLOYMENT}`)
const env = deployment.spec?.template?.spec?.containers?.[0]?.env ?? []
const value = (key: string, fallback: number): number => {
- const found = env.find((item: { name?: string }) => item.name === key)
+ const found = env.find(item => item.name === key)
return typeof found?.value === 'string' ? Number(found.value) : fallback
}
return {
diff --git a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
index cfb747bcb..b7209ce59 100644
--- a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
+++ b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
@@ -17,6 +17,10 @@ if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be
export default defineConfig({
...base,
+ // The shared global setup stays off: its preflight login spends an attempt from the password
+ // budget, it reads E2E_K8S_CONTEXT instead of CHAT_OPEN_AB_KUBE_CONTEXT, and its seed marker
+ // ties the profile to this worktree's HEAD while BEFORE is another build. The spec runs the
+ // same context guard (assertAllowedKubeContext) before its first kubectl call.
globalSetup: undefined,
projects: [{ name: 'chat-open-ab', testMatch: /chat-open-ab\.spec\.ts/ }],
// Paced opens, 65 s rate-limit windows between launches and stateless suspension waits.
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts b/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts
index ec2b47863..d4a1538a4 100644
--- a/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts
+++ b/desktop-app/test/e2e-playwright/qa-recorder-helpers.ts
@@ -131,11 +131,17 @@ export function configuredHostRef(): string {
* 1280x720 WebM. Returns the app handle and its first window page. Caller owns
* closing the app (use finalizeRecording in a finally block) so the video file
* is finalized on disk.
+ * `launch.executablePath` pins the Electron binary; `launch.beforeFirstWindow`
+ * receives the app before its first window opens (to attach output streams).
*/
export async function launchDesktopApp(
testInfo: TestInfo,
isolationTarget?: string,
- mainEntry: string = MAIN_ENTRY
+ mainEntry: string = MAIN_ENTRY,
+ launch: {
+ executablePath?: string
+ beforeFirstWindow?: (app: ElectronApplication) => void
+ } = {}
): Promise<{
app: ElectronApplication
page: Page
@@ -153,6 +159,7 @@ export async function launchDesktopApp(
const slowMo = Number(process.env.QA_RECORDER_SLOW_MO_MS ?? 75)
const app = await electron.launch({
+ executablePath: launch.executablePath,
args: [`--user-data-dir=${userDataDir}`, mainEntry],
env: {
...process.env,
@@ -188,6 +195,7 @@ export async function launchDesktopApp(
slowMo: Number.isFinite(slowMo) ? slowMo : 75,
})
+ launch.beforeFirstWindow?.(app)
const actualUserDataDir = await app.evaluate(({ app: electronApp }) =>
electronApp.getPath('userData')
)
@@ -281,14 +289,8 @@ export async function openAgentsPage(page: Page): Promise<void> {
/**
* Navigate to the exact agent named by hostRef and land on its chat composer.
* Uses exact identity (no first-available fallback), per the recorder contract.
- * The UI shows the agent's display name; pass it as `label` when it differs
- * from hostRef.
*/
-export async function openExactAgentChat(
- page: Page,
- hostRef: string,
- label: string = hostRef
-): Promise<void> {
+export async function openExactAgentChat(page: Page, hostRef: string): Promise<void> {
const settingsMenu = page.getByTestId('nav-settings-menu')
if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') {
await settingsMenu.click()
@@ -299,7 +301,7 @@ export async function openExactAgentChat(
await expect(agentsItem).toBeVisible({ timeout: 15_000 })
await agentsItem.click()
const chatInput = page.getByRole('textbox', { name: 'Agent message composer' })
- const exactAgent = page.locator('.agents-table-row-clickable', { hasText: label }).first()
+ const exactAgent = page.locator('.agents-table-row-clickable', { hasText: hostRef }).first()
const emptyState = page.getByText('No agents available')
await expect(chatInput.or(exactAgent).or(emptyState)).toBeVisible({ timeout: 20_000 })
@@ -317,16 +319,16 @@ export async function openExactAgentChat(
if (
!(await chatInput.isVisible().catch(() => false)) ||
!(await switchAgent
- .getByText(label, { exact: true })
+ .getByText(hostRef, { exact: true })
.isVisible()
.catch(() => false))
) {
await switchAgent.click()
- await page.getByRole('menuitem', { name: label, exact: true }).click()
+ await page.getByRole('menuitem', { name: hostRef, exact: true }).click()
}
await expect(chatInput).toBeVisible({ timeout: 20_000 })
- await expect(switchAgent).toContainText(label)
+ await expect(switchAgent).toContainText(hostRef)
}
/** Send a chat message and wait for a non-empty assistant response. */
diff --git a/desktop-app/test/globalSetupGuard.test.ts b/desktop-app/test/globalSetupGuard.test.ts
index 9b4c979a8..687f0001a 100644
--- a/desktop-app/test/globalSetupGuard.test.ts
+++ b/desktop-app/test/globalSetupGuard.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest'
-import { validateBaseUrls } from './e2e-playwright/global-setup'
+import {
+ ALLOWED_CONTEXTS,
+ assertAllowedKubeContext,
+ isLocalPortForwardContext,
+ isProductionContext,
+ validateBaseUrls,
+} from './e2e-playwright/global-setup'
const DEV_CONTEXT = 'gke_${GCP_PROJECT}_us-central1-a_example-dev'
@@ -118,3 +124,27 @@ describe('validateBaseUrls', () => {
).toThrow(/requires non-localhost URLs/)
})
})
+
+describe('assertAllowedKubeContext', () => {
+ it('accepts a branch profile and refuses production with the same call', () => {
+ // Positive control: the guard returns for the context the A/B benchmark actually uses.
+ expect(isLocalPortForwardContext('clerum-fix-issue-1091-chat-open-latency-d7ddc3ab')).toBe(true)
+ expect(() =>
+ assertAllowedKubeContext(
+ 'clerum-fix-issue-1091-chat-open-latency-d7ddc3ab',
+ 'CHAT_OPEN_AB_KUBE_CONTEXT'
+ )
+ ).not.toThrow()
+ expect(isProductionContext('gke_example_us-central1-a_clerum-prod')).toBe(true)
+ expect(() =>
+ assertAllowedKubeContext('gke_example_us-central1-a_clerum-prod', 'CHAT_OPEN_AB_KUBE_CONTEXT')
+ ).toThrow(/Production context .* is hard-blocked/)
+ })
+
+ it('refuses a context that is neither a local profile nor allow-listed', () => {
+ expect(ALLOWED_CONTEXTS.has(DEV_CONTEXT)).toBe(false)
+ expect(() => assertAllowedKubeContext(DEV_CONTEXT, 'CHAT_OPEN_AB_KUBE_CONTEXT')).toThrow(
+ /CHAT_OPEN_AB_KUBE_CONTEXT=.* is not in the allow-list/
+ )
+ })
+})