Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(desktop): split the chat-open A/B into OL and W phases
Alfredo Lopez4015629eaff0
diff --git a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts b/desktop-app/test/e2e-playwright/chat-open-ab.spec.tsindex ab9cb41ef..2a862cf81 100644--- a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts+++ b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts@@ -29,7 +29,10 @@ import {// E2E_GUARDIAN_IPC_FLOW: server reads are preconditions through Desktop IPC;// the measured journey clicks real rows and observes the visible transcript.// Inputs: CHAT_OPEN_AB_BEFORE_MAIN, CHAT_OPEN_AB_AFTER_MAIN, CHAT_OPEN_AB_KUBE_CONTEXT,-// CHAT_OPEN_AB_OUTPUT_JSON. Run only with playwright.chat-open-ab.config.ts.+// CHAT_OPEN_AB_OUTPUT_JSON, CHAT_OPEN_AB_PHASE. Run only with playwright.chat-open-ab.config.ts.+// Phase OL (cached opens, large-cache relaunch) needs the stateless Host to stay up for the+// whole phase; phase W needs it to suspend within minutes. They run as two invocations so the+// HCC idle cadence can differ between them.type Build = 'before' | 'after'type Chat = { id: string; count: number; lastText: string }type Catalog = {@@ -286,6 +289,8 @@ if (const output = required('CHAT_OPEN_AB_OUTPUT_JSON')if (!path.isAbsolute(output) || path.extname(output) !== '.json')throw new Error('CHAT_OPEN_AB_OUTPUT_JSON must be an absolute JSON path.')+ const phase = required('CHAT_OPEN_AB_PHASE')+ if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')const builds = Object.fromEntries(Object.entries(entries).map(([build, entry]) => [build,@@ -310,6 +315,7 @@ if (invalidated: boolean}> = []const report = {+ phase,builds,node: process.version,machine: {@@ -572,19 +578,22 @@ if ()if (new Set(lastTexts).size !== lastTexts.length)throw new Error('precondition: every measured chat must end with a unique last text.')- // W empties the cache, so its sidebar preview is the server's latest sessions across Hosts.- const previewHosts = new Set(- report.catalogs- .flatMap(catalog => catalog.latestActivity.map(at => ({ at, host: catalog.host })))- .sort((a, b) => b.at.localeCompare(a.at))- .slice(0, SIDEBAR_PREVIEW)- .map(entry => entry.host)- )- for (const host of HOSTS)- if (!previewHosts.has(host))- throw new Error(- `precondition: W needs a ${host} session among the server's ${SIDEBAR_PREVIEW} latest sessions.`- )+ if (phase === 'W') {+ // W empties the cache, so its sidebar preview is the server's latest sessions across Hosts.+ const previewHosts = new Set(+ report.catalogs+ .flatMap(catalog => catalog.latestActivity.map(at => ({ at, host: catalog.host })))+ .sort((a, b) => b.at.localeCompare(a.at))+ .slice(0, SIDEBAR_PREVIEW)+ .map(entry => entry.host)+ )+ for (const host of HOSTS)+ if (!previewHosts.has(host))+ throw new Error(+ `precondition: W needs a ${host} session among the server's ${SIDEBAR_PREVIEW} latest sessions.`+ )+ return+ }// The benchmark builder owns valid paged metadata; rebind its synthetic tombstones to the real session below.const fixtureDir = isolatedPath(runDir, path.join(runDir, 'fixture'))fs.mkdirSync(fixtureDir, { mode: 0o700 })@@ -611,163 +620,173 @@ if (}report.fixtureRoot = fixture.root})- for (const build of ['before', 'after', 'after', 'before'] as const) {- await ready()- await test.step(`O: ${build} cached chat opens`, () =>- round('O', build, async id => {- await launch(build)- await authenticated()- await page.getByTestId('nav-chat').click()- // Stateless first: the sidebar preview orders cached chats by their first load, and L needs stateful rows in it.- for (const catalog of [...report.catalogs].reverse()) {- const checked = new Set<string>()- for (let pass = 1; pass <= (id === 1 ? 3 : 2); pass++) {- const remote = id === 1 && pass === 1- if (!remote)- await test.step(`precondition: ${catalog.host} transcripts are cached`, async () => {- await expect- .poll(- () =>+ if (phase === 'OL') {+ for (const build of ['before', 'after', 'after', 'before'] as const) {+ await ready()+ await test.step(`O: ${build} cached chat opens`, () =>+ round('O', build, async id => {+ await launch(build)+ await authenticated()+ await page.getByTestId('nav-chat').click()+ // Stateless first: the sidebar preview orders cached chats by their first load, and L needs stateful rows in it.+ for (const catalog of [...report.catalogs].reverse()) {+ const checked = new Set<string>()+ for (let pass = 1; pass <= (id === 1 ? 3 : 2); pass++) {+ const remote = id === 1 && pass === 1+ if (!remote)+ await test.step(`precondition: ${catalog.host} transcripts are cached`, async () => {+ await expect+ .poll(+ () =>+ page.evaluate(+ async ({ host, chats }) => {+ const messages = await Promise.all(+ chats.map(chat => window.clerum.chat.loadMessages(host, chat.id))+ )+ return messages.every(+ (items, index) =>+ items.length === chats[index]!.count &&+ items[items.length - 1]?.content === chats[index]!.lastText+ )+ },+ { host: catalog.host, chats: catalog.chats }+ ),+ { timeout: 30_000 }+ )+ .toBe(true)+ })+ for (const chat of catalog.chats) {+ const ms = await openChat(page, catalog.host, hostLabels[catalog.host]!, chat)+ samples.push({+ round: id,+ build,+ host: catalog.host,+ metric: remote ? 'open_remote_ms' : 'open_cached_ms',+ pass,+ chatId: chat.id,+ ms,+ })+ if (!checked.has(chat.id)) {+ await expect+ .poll(() =>page.evaluate(- async ({ host, chats }) => {- const messages = await Promise.all(- chats.map(chat => window.clerum.chat.loadMessages(host, chat.id))- )- return messages.every(- (items, index) =>- items.length === chats[index]!.count &&- items[items.length - 1]?.content === chats[index]!.lastText- )- },- { host: catalog.host, chats: catalog.chats }- ),- { timeout: 30_000 }- )- .toBe(true)- })- for (const chat of catalog.chats) {- const ms = await openChat(page, catalog.host, hostLabels[catalog.host]!, chat)- samples.push({- round: id,- build,- host: catalog.host,- metric: remote ? 'open_remote_ms' : 'open_cached_ms',- pass,- chatId: chat.id,- ms,- })- if (!checked.has(chat.id)) {- await expect- .poll(() =>- page.evaluate(host => window.clerum.chat.getLastActive(host), catalog.host)- )- .toBe(chat.id)- await expect(page.getByTestId('message-list').locator(MESSAGES)).toHaveCount(- chat.count- )- // The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.- // An empty preview would satisfy "none is active" without checking anything.- await expect(- page.locator('[data-testid="latest-session-row"]').first()- ).toBeVisible()- const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0- await expect(- page.locator('[data-testid="latest-session-row"][aria-current="true"]')- ).toHaveCount(inSidebar ? 1 : 0)- if (inSidebar)- await expect(page.locator(row(catalog.host, chat.id))).toHaveAttribute(- 'aria-current',- 'true'+ host => window.clerum.chat.getLastActive(host),+ catalog.host+ )+ )+ .toBe(chat.id)+ await expect(page.getByTestId('message-list').locator(MESSAGES)).toHaveCount(+ chat.count)- checked.add(chat.id)+ // The sidebar previews only the latest sessions: when this chat has a row it is the active one, otherwise none is.+ // An empty preview would satisfy "none is active" without checking anything.+ await expect(+ page.locator('[data-testid="latest-session-row"]').first()+ ).toBeVisible()+ const inSidebar = (await page.locator(row(catalog.host, chat.id)).count()) > 0+ await expect(+ page.locator('[data-testid="latest-session-row"][aria-current="true"]')+ ).toHaveCount(inSidebar ? 1 : 0)+ if (inSidebar)+ await expect(page.locator(row(catalog.host, chat.id))).toHaveAttribute(+ 'aria-current',+ 'true'+ )+ checked.add(chat.id)+ }}}}- }- }))- }- // A cached chat's updatedAt is set by its first remote load in round 1, so L times the- // most recently loaded stateful chat, which the reversed host order puts in the preview.- const statefulIndex = JSON.parse(- fs.readFileSync(isolatedPath(runDir, path.join(cacheDir!, HOSTS[0], 'index.json')), 'utf8')- ) as { chats: Array<{ id: string; updatedAt: string }> }- const updatedAt = new Map(statefulIndex.chats.map(chat => [chat.id, chat.updatedAt]))- const selected = report.catalogs[0]!.chats.reduce((latest, chat) => {- const at = updatedAt.get(chat.id)- if (!at) throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)- return at > updatedAt.get(latest.id)! ? chat : latest- })- await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)- await expect- .poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0]))- .toBe(selected.id)- // L times a sidebar row, which exists only for sessions inside the sidebar preview.- await expect(- page!.locator(row(HOSTS[0], selected.id)),- `precondition: L needs ${HOSTS[0]}/${selected.id} inside the sidebar preview.`- ).toBeVisible()- await close()- const savedCache = isolatedPath(runDir, path.join(runDir, 'cache-snapshot'))- fs.cpSync(isolatedPath(runDir, cacheDir!), savedCache, { recursive: true })- const fixtureRoot = report.fixtureRoot- for (const agent of fs.readdirSync(fixtureRoot))- fs.cpSync(- isolatedPath(runDir, path.join(fixtureRoot, agent)),- isolatedPath(runDir, path.join(savedCache, agent)),- { recursive: true }- )- for (const build of ['before', 'after'] as const)- for (let iteration = 0; iteration < 3; iteration++) {- await ready()- await close()- fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })- fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })- await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>- round('L', build, async id => {- await launch(build)- const [paint] = await Promise.all([- relaunchPaint(page, [- {- host: HOSTS[0],- metric: 'relaunch_ms',- selector: row(HOSTS[0], selected.id),- chat: selected,- },- ]),- (async () => {- await authenticated()- await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()- await page.getByTestId('nav-chat').click()- await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)- })(),- ])- for (const sample of paint) samples.push({ round: id, build, ...sample })- await authenticated()}))}- for (const build of ['before', 'after'] as const)- for (let iteration = 0; iteration < 3; iteration++) {- await close()- await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])- // Empty only the isolated chat cache: both observed rows must come from the remote catalog.- if (fs.existsSync(cacheDir!))+ // A cached chat's updatedAt is set by its first remote load in round 1, so L times the+ // most recently loaded stateful chat, which the reversed host order puts in the preview.+ const statefulIndex = JSON.parse(+ fs.readFileSync(+ isolatedPath(runDir, path.join(cacheDir!, HOSTS[0], 'index.json')),+ 'utf8'+ )+ ) as { chats: Array<{ id: string; updatedAt: string }> }+ const updatedAt = new Map(statefulIndex.chats.map(chat => [chat.id, chat.updatedAt]))+ const selected = report.catalogs[0]!.chats.reduce((latest, chat) => {+ const at = updatedAt.get(chat.id)+ if (!at)+ throw new Error(`precondition: ${HOSTS[0]}/${chat.id} is missing from the cache.`)+ return at > updatedAt.get(latest.id)! ? chat : latest+ })+ await openChat(page!, HOSTS[0], hostLabels[HOSTS[0]]!, selected)+ await expect+ .poll(() => page!.evaluate(host => window.clerum.chat.getLastActive(host), HOSTS[0]))+ .toBe(selected.id)+ // L times a sidebar row, which exists only for sessions inside the sidebar preview.+ await expect(+ page!.locator(row(HOSTS[0], selected.id)),+ `precondition: L needs ${HOSTS[0]}/${selected.id} inside the sidebar preview.`+ ).toBeVisible()+ await close()+ const savedCache = isolatedPath(runDir, path.join(runDir, 'cache-snapshot'))+ fs.cpSync(isolatedPath(runDir, cacheDir!), savedCache, { recursive: true })+ const fixtureRoot = report.fixtureRoot+ for (const agent of fs.readdirSync(fixtureRoot))+ fs.cpSync(+ isolatedPath(runDir, path.join(fixtureRoot, agent)),+ isolatedPath(runDir, path.join(savedCache, agent)),+ { recursive: true }+ )+ for (const build of ['before', 'after'] as const)+ for (let iteration = 0; iteration < 3; iteration++) {+ await ready()+ await close()fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })- await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>- round('W', build, async id => {- await launch(build)- for (const sample of await relaunchPaint(- page,- HOSTS.map((host, index) => ({- host,- selector: rows(host),- metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',- }))- ))- samples.push({ round: id, build, ...sample })- await authenticated()- await ready()- }))- }+ fs.cpSync(savedCache, isolatedPath(runDir, cacheDir!), { recursive: true })+ await test.step(`L: ${build} large-cache relaunch ${iteration + 1}`, () =>+ round('L', build, async id => {+ await launch(build)+ const [paint] = await Promise.all([+ relaunchPaint(page, [+ {+ host: HOSTS[0],+ metric: 'relaunch_ms',+ selector: row(HOSTS[0], selected.id),+ chat: selected,+ },+ ]),+ (async () => {+ await authenticated()+ await expect(page.locator(row(HOSTS[0], selected.id))).toBeVisible()+ await page.getByTestId('nav-chat').click()+ await openChat(page, HOSTS[0], hostLabels[HOSTS[0]]!, selected)+ })(),+ ])+ for (const sample of paint) samples.push({ round: id, build, ...sample })+ await authenticated()+ }))+ }+ }+ if (phase === 'W')+ for (const build of ['before', 'after'] as const)+ for (let iteration = 0; iteration < 3; iteration++) {+ await close()+ await waitForStatelessSuspended(240_000, context, HOSTS[1], HOSTS[1])+ // Empty only the isolated chat cache: both observed rows must come from the remote catalog.+ if (fs.existsSync(cacheDir!))+ fs.rmSync(isolatedPath(runDir, cacheDir!), { recursive: true })+ await test.step(`W: ${build} suspended-stateless relaunch ${iteration + 1}`, () =>+ round('W', build, async id => {+ await launch(build)+ for (const sample of await relaunchPaint(+ page,+ HOSTS.map((host, index) => ({+ host,+ selector: rows(host),+ metric: index === 0 ? 'first_stateful_row_ms' : 'stateless_rows_ms',+ }))+ ))+ samples.push({ round: id, build, ...sample })+ await authenticated()+ await ready()+ }))+ }const values = (build: Build, host: string, metric: string, roundId?: number) =>samples.filter(@@ -782,41 +801,44 @@ if (report.cells[`${sample.build}/${sample.host}/${sample.metric}`] = quantiles(values(sample.build, sample.host, sample.metric))- for (const host of HOSTS) {- const before = quantiles(values('before', host, 'open_cached_ms')).p50,- after = quantiles(values('after', host, 'open_cached_ms')).p50- const spread = Math.abs(- quantiles(values('before', host, 'open_cached_ms', 1)).p50 -- quantiles(values('before', host, 'open_cached_ms', 4)).p50- )- const budget = Math.max(spread, before * 0.1)+ if (phase === 'OL') {+ for (const host of HOSTS) {+ const before = quantiles(values('before', host, 'open_cached_ms')).p50,+ after = quantiles(values('after', host, 'open_cached_ms')).p50+ const spread = Math.abs(+ quantiles(values('before', host, 'open_cached_ms', 1)).p50 -+ quantiles(values('before', host, 'open_cached_ms', 4)).p50+ )+ const budget = Math.max(spread, before * 0.1)+ report.gates.push({+ metric: 'open_cached_ms',+ host,+ before,+ after,+ aa_spread_ms: spread,+ budget_ms: budget,+ passed: after <= before + budget,+ })+ }+ const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,+ after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50+ report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })+ } else {+ const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))report.gates.push({- metric: 'open_cached_ms',- host,- before,- after,- aa_spread_ms: spread,- budget_ms: budget,- passed: after <= before + budget,+ metric: 'first_stateful_row_ms',+ after_max_ms: wakeAfter,+ budget_ms: 2000,+ passed: wakeAfter < 2000,})}- const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,- after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50- report.gates.push({ metric: 'relaunch_ms', before, after, passed: after < before })- const wakeAfter = Math.max(...values('after', HOSTS[0], 'first_stateful_row_ms'))- report.gates.push({- metric: 'first_stateful_row_ms',- after_max_ms: wakeAfter,- budget_ms: 2000,- passed: wakeAfter < 2000,- })report.gates.push({metric: 'round_validity',invalidated: rounds.filter(round => round.invalidated).length,passed: rounds.every(round => !round.invalidated),})// Rounds compared within a scenario must run under comparable 1-minute load.- for (const scenario of ['O', 'L', 'W']) {+ for (const scenario of phase === 'OL' ? ['O', 'L'] : ['W']) {const loads = rounds.filter(round => round.scenario === scenario).map(round => round.loadavg[0]!)diff --git a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.tsindex c82fef83e..cfb747bcb 100644--- a/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts+++ b/desktop-app/test/e2e-playwright/playwright.chat-open-ab.config.ts@@ -12,6 +12,8 @@ const commonDir = execFileSync('git', ['rev-parse', '--git-common-dir'], {encoding: 'utf8',}).trim()const repositoryRoot = path.resolve(__dirname, '../../..', commonDir, '..')+const phase = process.env.CHAT_OPEN_AB_PHASE+if (phase !== 'OL' && phase !== 'W') throw new Error('CHAT_OPEN_AB_PHASE must be OL or W.')export default defineConfig({...base,@@ -23,6 +25,7 @@ export default defineConfig({retries: 0,workers: 1,reporter: [['list']],- outputDir: path.join(repositoryRoot, '.local-notes/issue1091-evidence/v2/C3/runs'),+ // Playwright empties outputDir on every run; one directory per phase keeps the other's evidence.+ outputDir: path.join(repositoryRoot, `.local-notes/issue1091-evidence/v2/C3/runs-${phase}`),preserveOutput: 'always',})