Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(desktop): split chat-open A/B into first open and reopen
Alfredo Lopez8348a10fae8a
diff --git a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts b/desktop-app/test/e2e-playwright/chat-open-ab.spec.tsindex 2a862cf81..79d5cb62e 100644--- a/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts+++ b/desktop-app/test/e2e-playwright/chat-open-ab.spec.ts@@ -33,6 +33,8 @@ import {// Phase OL (cached opens, large-cache relaunch) needs the stateless Host to stay up for the// whole phase; phase W needs it to suspend within minutes. They run as two invocations so the// HCC idle cadence can differ between them.+// O measures each chat's first open after launch (open_cached_ms, read from disk) apart from+// its reopen in the same session (open_memo_ms), which the AFTER build serves from memory.type Build = 'before' | 'after'type Chat = { id: string; count: number; lastText: string }type Catalog = {@@ -53,8 +55,12 @@ type Sample = {chatId?: stringobserved_at_install?: booleanobserver_installed_ms?: number+ /** The transcript was missing or empty at some point between the click and the measurement. */+ blank?: boolean}const HOSTS = ['chatllm', 'chatllm-stateless'] as const+// The Desktop remembers the last 8 transcripts it showed (`transcriptMemo`); reopens stay within it.+const REMEMBERED_CHATS = 8const MESSAGES = '[data-testid="user-message"], [data-testid="agent-response"]'const LIMIT_WINDOW_MS = 65_000const RATE_LIMITED = /\b429\b|Too Many Requests/g@@ -92,7 +98,12 @@ function isolatedPath(runDir: string, target: string): string {// per-session limit; spacing opens keeps a launch under that limit. The wait is outside the timing.const OPEN_PACE_MS = 4_000let lastOpenAt = 0-async function openChat(page: Page, host: string, label: string, chat: Chat): Promise<number> {+async function openChat(+ page: Page,+ host: string,+ label: string,+ chat: Chat+): Promise<{ ms: number; blank: boolean }> {const pace = lastOpenAt + OPEN_PACE_MS - Date.now()if (pace > 0) await page.waitForTimeout(pace)lastOpenAt = Date.now()@@ -130,11 +141,13 @@ async function openChat(page: Page, host: string, label: string, chat: Chat): Pr.toBe(true)return page.evaluate(({ selector, chat, messages }) =>- new Promise<number>((resolve, reject) => {+ new Promise<{ ms: number; blank: boolean }>((resolve, reject) => {const button = document.querySelector<HTMLButtonElement>(`${selector} button`)if (!button || button.disabled)throw new Error(`Chat row has no enabled open button: ${selector}`)let started: number+ let clicked = false+ let blank = false// The sidebar previews only the latest sessions, so the active-row witness is checked outside the timing.const finish = () => {const transcripts = document.querySelectorAll('[data-testid="message-list"]')@@ -146,6 +159,8 @@ async function openChat(page: Page, host: string, label: string, chat: Chat): Pr}const transcript = transcripts[0]const bubbles = transcript?.querySelectorAll(messages)+ // React commits a click's state synchronously, so every check after the click sees the new chat's frames.+ if (clicked && !bubbles?.length) blank = trueif (bubbles?.length === chat.count &&bubbles[chat.count - 1]?.textContent?.includes(chat.lastText) &&@@ -154,7 +169,7 @@ async function openChat(page: Page, host: string, label: string, chat: Chat): Pr) {observer.disconnect()clearTimeout(deadline)- resolve(performance.now() - started)+ resolve({ ms: performance.now() - started, blank })}}const observer = new MutationObserver(finish)@@ -170,6 +185,7 @@ async function openChat(page: Page, host: string, label: string, chat: Chat): Pr}, 120_000)started = performance.now()button.click()+ clicked = truefinish()}),{@@ -631,8 +647,22 @@ if (// Stateless first: the sidebar preview orders cached chats by their first load, and L needs stateful rows in it.for (const catalog of [...report.catalogs].reverse()) {const checked = new Set<string>()- for (let pass = 1; pass <= (id === 1 ? 3 : 2); pass++) {+ // Round 1 is always BEFORE, so its first cached pass cannot be served from memory.+ const firstCachedPass = id === 1 ? 2 : 1+ // Reopens stay inside the remembered window: the last chats opened, newest first,+ // skipping the active chat (a click on it changes nothing) until the end.+ const remembered = catalog.chats.slice(+ -Math.min(catalog.chats.length, REMEMBERED_CHATS)+ )+ if (remembered.length < 2)+ throw new Error(`precondition: ${catalog.host} needs two chats to reopen.`)+ const reopenOrder = [+ ...remembered.slice(0, -1).reverse(),+ remembered[remembered.length - 1]!,+ ]+ for (let pass = 1; pass <= firstCachedPass + 1; pass++) {const remote = id === 1 && pass === 1+ const reopen = pass > firstCachedPassif (!remote)await test.step(`precondition: ${catalog.host} transcripts are cached`, async () => {await expect@@ -655,16 +685,26 @@ if ().toBe(true)})- for (const chat of catalog.chats) {- const ms = await openChat(page, catalog.host, hostLabels[catalog.host]!, chat)+ for (const chat of reopen ? reopenOrder : catalog.chats) {+ const { ms, blank } = await openChat(+ page,+ catalog.host,+ hostLabels[catalog.host]!,+ chat+ )samples.push({round: id,build,host: catalog.host,- metric: remote ? 'open_remote_ms' : 'open_cached_ms',+ metric: remote+ ? 'open_remote_ms'+ : reopen+ ? 'open_memo_ms'+ : 'open_cached_ms',pass,chatId: chat.id,ms,+ blank,})if (!checked.has(chat.id)) {await expect@@ -819,6 +859,40 @@ if (budget_ms: budget,passed: after <= before + budget,})+ const memoBefore = quantiles(values('before', host, 'open_memo_ms')).p50,+ memoAfter = quantiles(values('after', host, 'open_memo_ms')).p50+ report.gates.push({+ metric: 'open_memo_ms',+ host,+ before: memoBefore,+ after: memoAfter,+ passed: memoAfter < memoBefore,+ })+ // AFTER reopens never show an empty transcript. The witnesses prove the observer does see+ // one where it happens: on AFTER first opens (disk path) and on every BEFORE reopen.+ const blanks = (build: Build, metric: string) =>+ samples.filter(+ sample =>+ sample.build === build &&+ sample.host === host &&+ sample.metric === metric &&+ sample.blank+ ).length+ const witness = {+ after_reopen_blank: blanks('after', 'open_memo_ms'),+ after_first_open_blank: blanks('after', 'open_cached_ms'),+ before_reopen_blank: blanks('before', 'open_memo_ms'),+ }+ report.gates.push({+ metric: 'reopen_blank_witness',+ host,+ ...witness,+ passed:+ values('after', host, 'open_memo_ms').length > 0 &&+ witness.after_reopen_blank === 0 &&+ witness.after_first_open_blank > 0 &&+ witness.before_reopen_blank > 0,+ })}const before = quantiles(values('before', HOSTS[0], 'relaunch_ms')).p50,after = quantiles(values('after', HOSTS[0], 'relaunch_ms')).p50