Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(desktop): split the chat-open A/B into OL and W phases

Alfredo Lopez4015629eaff0
diff --git a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
index ab9cb41ef..2a862cf81 100644
--- a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
+++ b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts
@@ -29,7 +29,10 @@ import {
// E2E_GUARDIAN_IPC_FLOW: server reads are preconditions through Desktop IPC;
// the measured journey clicks real rows and observes the visible transcript.
// Inputs: CHAT_OPEN_AB_BEFORE_MAIN, CHAT_OPEN_AB_AFTER_MAIN, CHAT_OPEN_AB_KUBE_CONTEXT,
-// CHAT_OPEN_AB_OUTPUT_JSON. Run only with playwright.chat-open-ab.config.ts.
+// CHAT_OPEN_AB_OUTPUT_JSON, CHAT_OPEN_AB_PHASE. Run only with playwright.chat-open-ab.config.ts.
+// Phase OL (cached opens, large-cache relaunch) needs the stateless Host to stay up for the
+// whole phase; phase W needs it to suspend within minutes. They run as two invocations so the
+// HCC idle cadence can differ between them.
type Build = 'before' | 'after'
type Chat = { id: string; count: number; lastText: string }
type Catalog = {
@@ -286,6 +289,8 @@ if (
const output = required('CHAT_OPEN_AB_OUTPUT_JSON')
if (!path.isAbsolute(output) || path.extname(output) !== '.json')
throw new Error('CHAT_OPEN_AB_OUTPUT_JSON must be an absolute JSON path.')
+ const phase = required('CHAT_OPEN_AB_PHASE')
+ if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')
const builds = Object.fromEntries(
Object.entries(entries).map(([build, entry]) => [
build,
@@ -310,6 +315,7 @@ if (
invalidated: boolean
}> = []
const report = {
+ phase,
builds,
node: process.version,
machine: {
@@ -572,19 +578,22 @@ if (
)
if (new Set(lastTexts).size !== lastTexts.length)
throw new Error('precondition: every measured chat must end with a unique last text.')
- // W empties the cache, so its sidebar preview is the server's latest sessions across Hosts.
- const previewHosts = new Set(
- report.catalogs
- .flatMap(catalog => catalog.latestActivity.map(at => ({ at, host: catalog.host })))
- .sort((a, b) => b.at.localeCompare(a.at))
- .slice(0, SIDEBAR_PREVIEW)
- .map(entry => entry.host)
- )
- for (const host of HOSTS)
- if (!previewHosts.has(host))
- throw new Error(
- `precondition: W needs a ${host} session among the server's ${SIDEBAR_PREVIEW} latest sessions.`
- )
+ if (phase === 'W') {
+ // W empties the cache, so its sidebar preview is the server's latest sessions across Hosts.
+ const previewHosts = new Set(
+ report.catalogs
+ .flatMap(catalog => catalog.latestActivity.map(at => ({ at, host: catalog.host })))
+ .sort((a, b) => b.at.localeCompare(a.at))
+ .slice(0, SIDEBAR_PREVIEW)
+ .map(entry => entry.host)
+ )
+ for (const host of HOSTS)
+ if (!previewHosts.has(host))
+ throw new Error(
+ `precondition: W needs a ${host} session among the server's ${SIDEBAR_PREVIEW} latest sessions.`
+ )
+ return
+ }
// The benchmark builder owns valid paged metadata; rebind its synthetic tombstones to the real session below.
const fixtureDir = isolatedPath(runDir, path.join(runDir, 'fixture'))
fs.mkdirSync(fixtureDir, { mode: 0o700 })
@@ -611,163 +620,173 @@ if (
}
report.fixtureRoot = fixture.root
})
- for (const build of ['before', 'after', 'after', 'before'] as const) {
- await ready()
- await test.step(`O: ${build} cached chat opens`, () =>
- round('O', build, async id => {
- await launch(build)
- await authenticated()
- await page.getByTestId('nav-chat').click()
- // Stateless first: the sidebar preview orders cached chats by their first load, and L needs stateful rows in it.
- for (const catalog of [...report.catalogs].reverse()) {
- const checked = new Set<string>()
- for (let pass = 1; pass <= (id === 1 ? 3 : 2); pass++) {
- const remote = id === 1 && pass === 1
- if (!remote)
- await test.step(`precondition: ${catalog.host} transcripts are cached`, async () => {
- await expect
- .poll(
- () =>
+ if (phase === 'OL') {
+ for (const build of ['before', 'after', 'after', 'before'] as const) {
+ await ready()
+ await test.step(`O: ${build} cached chat opens`, () =>
+ round('O', build, async id => {
+ await launch(build)
+ await authenticated()
+ await page.getByTestId('nav-chat').click()
+ // Stateless first: the sidebar preview orders cached chats by their first load, and L needs stateful rows in it.
+ for (const catalog of [...report.catalogs].reverse()) {
+ const checked = new Set<string>()
+ for (let pass = 1; pass <= (id === 1 ? 3 : 2); pass++) {
+ const remote = id === 1 && pass === 1
+ if (!remote)
+ await test.step(`precondition: ${catalog.host} transcripts are cached`, async () => {
+ await expect
+ .poll(
+ () =>
+ page.evaluate(
+ async ({ host, chats }) => {
+ const messages = await Promise.all(
+ chats.map(chat => window.clerum.chat.loadMessages(host, chat.id))
+ )
+ return messages.every(
+ (items, index) =>
+ items.length === chats[index]!.count &&
+ items[items.length - 1]?.content === chats[index]!.lastText
+ )
+ },
+ { host: catalog.host, chats: catalog.chats }
+ ),
+ { timeout: 30_000 }
+ )
+ .toBe(true)
+ })
+ for (const chat of catalog.chats) {
+ const ms = await openChat(page, catalog.host, hostLabels[catalog.host]!, chat)
+ samples.push({
+ round: id,
+ build,
+ host: catalog.host,
+ metric: remote ? 'open_remote_ms' : 'open_cached_ms',
+ pass,
+ chatId: chat.id,
+ ms,
+ })
+ if (!checked.has(chat.id)) {
+ await expect
+ .poll(() =>
page.evaluate(
- async ({ host, chats }) => {
- const messages = await Promise.all(
- chats.map(chat => window.clerum.chat.loadMessages(host, chat.id))
- )
- return messages.every(
- (items, index) =>
- items.length === chats[index]!.count &&
- items[items.length - 1]?.content === chats[index]!.lastText
- )
- },
- { host: catalog.host, chats: catalog.chats }
- ),
- { timeout: 30_000 }
- )
- .toBe(true)
- })
- for (const chat of catalog.chats) {
- const ms = await openChat(page, catalog.host, hostLabels[catalog.host]!, chat)
- samples.push({
- round: id,
- build,
- host: catalog.host,
- metric: remote ? 'open_remote_ms' : 'open_cached_ms',
- pass,
- chatId: chat.id,
- ms,
- })
- if (!checked.has(chat.id)) {
- await expect
- .poll(() =>
- page.evaluate(host => window.clerum.chat.getLastActive(host), catalog.host)
- )
- .toBe(chat.id)
- await expect(page.getByTestId('message-list').locator(MESSAGES)).toHaveCount(
- chat.count
- )
- // The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.
- // An empty preview would satisfy "none is active" without checking anything.
- await expect(
- page.locator('[data-testid="latest-session-row"]').first()
- ).toBeVisible()
- const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0
- await expect(
- page.locator('[data-testid="latest-session-row"][aria-current="true"]')
- ).toHaveCount(inSidebar ? 1 : 0)
- if (inSidebar)
- await expect(page.locator(row(catalog.host, chat.id))).toHaveAttribute(
- 'aria-current',
- 'true'
+ host => window.clerum.chat.getLastActive(host),
+ catalog.host
+ )
+ )
+ .toBe(chat.id)
+ await expect(page.getByTestId('message-list').locator(MESSAGES)).toHaveCount(
+ chat.count
)
- checked.add(chat.id)
+ // The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.
+ // An empty preview would satisfy "none is active" without checking anything.
+ await expect(
+ page.locator('[data-testid="latest-session-row"]').first()
+ ).toBeVisible()
+ const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0
+ await expect(
+ page.locator('[data-testid="latest-session-row"][aria-current="true"]')
+ ).toHaveCount(inSidebar ? 1 : 0)
+ if (inSidebar)
+ await expect(page.locator(row(catalog.host, chat.id))).toHaveAttribute(
+ 'aria-current',
+ 'true'
+ )
+ checked.add(chat.id)
+ }
}
}
}
- }
- }))
- }
- // A cached chat's updatedAt is set by its first remote load in round 1, so L times the
- // most recently loaded stateful chat, which the reversed host order puts in the preview.
- const statefulIndex = JSON.parse(
- fs.readFileSync(isolatedPath(runDir, path.join(cacheDir!, HOSTS[0], 'index.json')), 'utf8')
- ) as { chats: Array<{ id: string; updatedAt: string }> }
- const updatedAt = new Map(statefulIndex.chats.map(chat => [chat.id, chat.updatedAt]))
- const selected = report.catalogs[0]!.chats.reduce((latest, chat) => {
- const at = updatedAt.get(chat.id)
- if (!at) throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)
- return at > updatedAt.get(latest.id)! ? chat : latest
- })
- await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
- await expect
- .poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0]))
- .toBe(selected.id)
- // L times a sidebar row, which exists only for sessions inside the sidebar preview.
- await expect(
- page!.locator(row(HOSTS[0], selected.id)),
- `precondition: L needs ${HOSTS[0]}/${selected.id} inside the sidebar preview.`
- ).toBeVisible()
- await close()
- const savedCache = isolatedPath(runDir, path.join(runDir, 'cache-snapshot'))
- fs.cpSync(isolatedPath(runDir, cacheDir!), savedCache, { recursive: true })
- const fixtureRoot = report.fixtureRoot
- for (const agent of fs.readdirSync(fixtureRoot))
- fs.cpSync(
- isolatedPath(runDir, path.join(fixtureRoot, agent)),
- isolatedPath(runDir, path.join(savedCache, agent)),
- { recursive: true }
- )
- for (const build of ['before', 'after'] as const)
- for (let iteration = 0; iteration < 3; iteration++) {
- await ready()
- await close()
- fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })
- fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })
- await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>
- round('L', build, async id => {
- await launch(build)
- const [paint] = await Promise.all([
- relaunchPaint(page, [
- {
- host: HOSTS[0],
- metric: 'relaunch_ms',
- selector: row(HOSTS[0], selected.id),
- chat: selected,
- },
- ]),
- (async () => {
- await authenticated()
- await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()
- await page.getByTestId('nav-chat').click()
- await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
- })(),
- ])
- for (const sample of paint) samples.push({ round: id, build, ...sample })
- await authenticated()
}))
}
- for (const build of ['before', 'after'] as const)
- for (let iteration = 0; iteration < 3; iteration++) {
- await close()
- await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])
- // Empty only the isolated chat cache: both observed rows must come from the remote catalog.
- if (fs.existsSync(cacheDir!))
+ // A cached chat's updatedAt is set by its first remote load in round 1, so L times the
+ // most recently loaded stateful chat, which the reversed host order puts in the preview.
+ const statefulIndex = JSON.parse(
+ fs.readFileSync(
+ isolatedPath(runDir, path.join(cacheDir!, HOSTS[0], 'index.json')),
+ 'utf8'
+ )
+ ) as { chats: Array<{ id: string; updatedAt: string }> }
+ const updatedAt = new Map(statefulIndex.chats.map(chat => [chat.id, chat.updatedAt]))
+ const selected = report.catalogs[0]!.chats.reduce((latest, chat) => {
+ const at = updatedAt.get(chat.id)
+ if (!at)
+ throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)
+ return at > updatedAt.get(latest.id)! ? chat : latest
+ })
+ await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
+ await expect
+ .poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0]))
+ .toBe(selected.id)
+ // L times a sidebar row, which exists only for sessions inside the sidebar preview.
+ await expect(
+ page!.locator(row(HOSTS[0], selected.id)),
+ `precondition: L needs ${HOSTS[0]}/${selected.id} inside the sidebar preview.`
+ ).toBeVisible()
+ await close()
+ const savedCache = isolatedPath(runDir, path.join(runDir, 'cache-snapshot'))
+ fs.cpSync(isolatedPath(runDir, cacheDir!), savedCache, { recursive: true })
+ const fixtureRoot = report.fixtureRoot
+ for (const agent of fs.readdirSync(fixtureRoot))
+ fs.cpSync(
+ isolatedPath(runDir, path.join(fixtureRoot, agent)),
+ isolatedPath(runDir, path.join(savedCache, agent)),
+ { recursive: true }
+ )
+ for (const build of ['before', 'after'] as const)
+ for (let iteration = 0; iteration < 3; iteration++) {
+ await ready()
+ await close()
fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })
- await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>
- round('W', build, async id => {
- await launch(build)
- for (const sample of await relaunchPaint(
- page,
- HOSTS.map((host, index) => ({
- host,
- selector: rows(host),
- metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',
- }))
- ))
- samples.push({ round: id, build, ...sample })
- await authenticated()
- await ready()
- }))
- }
+ fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })
+ await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>
+ round('L', build, async id => {
+ await launch(build)
+ const [paint] = await Promise.all([
+ relaunchPaint(page, [
+ {
+ host: HOSTS[0],
+ metric: 'relaunch_ms',
+ selector: row(HOSTS[0], selected.id),
+ chat: selected,
+ },
+ ]),
+ (async () => {
+ await authenticated()
+ await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()
+ await page.getByTestId('nav-chat').click()
+ await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)
+ })(),
+ ])
+ for (const sample of paint) samples.push({ round: id, build, ...sample })
+ await authenticated()
+ }))
+ }
+ }
+ if (phase === 'W')
+ for (const build of ['before', 'after'] as const)
+ for (let iteration = 0; iteration < 3; iteration++) {
+ await close()
+ await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])
+ // Empty only the isolated chat cache: both observed rows must come from the remote catalog.
+ if (fs.existsSync(cacheDir!))
+ fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })
+ await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>
+ round('W', build, async id => {
+ await launch(build)
+ for (const sample of await relaunchPaint(
+ page,
+ HOSTS.map((host, index) => ({
+ host,
+ selector: rows(host),
+ metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',
+ }))
+ ))
+ samples.push({ round: id, build, ...sample })
+ await authenticated()
+ await ready()
+ }))
+ }
const values = (build: Build, host: string, metric: string, roundId?: number) =>
samples
.filter(
@@ -782,41 +801,44 @@ if (
report.cells[`${sample.build}/${sample.host}/${sample.metric}`] = quantiles(
values(sample.build, sample.host, sample.metric)
)
- for (const host of HOSTS) {
- const before = quantiles(values('before', host, 'open_cached_ms')).p50,
- after = quantiles(values('after', host, 'open_cached_ms')).p50
- const spread = Math.abs(
- quantiles(values('before', host, 'open_cached_ms', 1)).p50 -
- quantiles(values('before', host, 'open_cached_ms', 4)).p50
- )
- const budget = Math.max(spread, before * 0.1)
+ if (phase === 'OL') {
+ for (const host of HOSTS) {
+ const before = quantiles(values('before', host, 'open_cached_ms')).p50,
+ after = quantiles(values('after', host, 'open_cached_ms')).p50
+ const spread = Math.abs(
+ quantiles(values('before', host, 'open_cached_ms', 1)).p50 -
+ quantiles(values('before', host, 'open_cached_ms', 4)).p50
+ )
+ const budget = Math.max(spread, before * 0.1)
+ report.gates.push({
+ metric: 'open_cached_ms',
+ host,
+ before,
+ after,
+ aa_spread_ms: spread,
+ budget_ms: budget,
+ passed: after <= before + budget,
+ })
+ }
+ const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,
+ after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50
+ report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })
+ } else {
+ const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))
report.gates.push({
- metric: 'open_cached_ms',
- host,
- before,
- after,
- aa_spread_ms: spread,
- budget_ms: budget,
- passed: after <= before + budget,
+ metric: 'first_stateful_row_ms',
+ after_max_ms: wakeAfter,
+ budget_ms: 2000,
+ passed: wakeAfter < 2000,
})
}
- const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,
- after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50
- report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })
- const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))
- report.gates.push({
- metric: 'first_stateful_row_ms',
- after_max_ms: wakeAfter,
- budget_ms: 2000,
- passed: wakeAfter < 2000,
- })
report.gates.push({
metric: 'round_validity',
invalidated: rounds.filter(round => round.invalidated).length,
passed: rounds.every(round => !round.invalidated),
})
// Rounds compared within a scenario must run under comparable 1-minute load.
- for (const scenario of ['O', 'L', 'W']) {
+ for (const scenario of phase === 'OL' ? ['O', 'L'] : ['W']) {
const loads = rounds
.filter(round => round.scenario === scenario)
.map(round => round.loadavg[0]!)
diff --git a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
index c82fef83e..cfb747bcb 100644
--- a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
+++ b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts
@@ -12,6 +12,8 @@ const commonDir = execFileSync('git', ['rev-parse', '--git-common-dir'], {
encoding: 'utf8',
}).trim()
const repositoryRoot = path.resolve(__dirname, '../../..', commonDir, '..')
+const phase = process.env.CHAT_OPEN_AB_PHASE
+if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')
export default defineConfig({
...base,
@@ -23,6 +25,7 @@ export default defineConfig({
retries: 0,
workers: 1,
reporter: [['list']],
- outputDir: path.join(repositoryRoot, '.local-notes/issue1091-evidence/v2/C3/runs'),
+ // Playwright empties outputDir on every run; one directory per phase keeps the other's evidence.
+ outputDir: path.join(repositoryRoot, `.local-notes/issue1091-evidence/v2/C3/runs-${phase}`),
preserveOutput: 'always',
})