The Open Framework for Building Agents
fix(eve): deflake web chat browser tests, rename to test-webchat (#4468)
Ben Pankowac6d499e60c6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7c140116f..29dd0000c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -187,7 +187,7 @@ jobs:- name: Build framework fixtures against the workspace eve distrun: pnpm test:framework-fixtures- test-browser:+ test-webchat:runs-on: ubuntu-lateststeps:@@ -211,8 +211,8 @@ jobs:- name: Install Chromiumrun: pnpm --filter eve exec playwright-core install --with-deps chromium-headless-shell- - name: Run browser tests- run: pnpm test:browser+ - name: Run web chat tests+ run: pnpm test:webchattest-templates:runs-on: ubuntu-latestdiff --git a/AGENTS.md b/AGENTS.mdindex 65605afd7..837da3116 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -59,7 +59,7 @@ pnpm test:unit # unit tests (<3s)pnpm test:integration # integration tests (several minutes; leave the full run to CI)pnpm test:scenario # scenario tests (2–5 min; requires pnpm build first)pnpm test:framework-fixtures # apps/frameworks smoke builds (requires pnpm build first)-pnpm test:browser # client templates in headless Chromium (needs playwright-core's chromium-headless-shell)+pnpm test:webchat # web chat template in headless Chromium (needs playwright-core's chromium-headless-shell)pnpm test:e2e # fixture-owned eve eval suites (CI only)pnpm test:tui # TUI smoke scripts (not e2e)```diff --git a/package.json b/package.jsonindex 6fb3419e6..a8ecd504c 100644--- a/package.json+++ b/package.json@@ -23,7 +23,7 @@"release": "pnpm build && node ./scripts/assert-changeset-publish-packages.mjs && changeset publish","test": "pnpm test:unit && pnpm test:integration","test:e2e": "turbo run test:e2e --concurrency=1",- "test:browser": "pnpm --filter eve run test:browser",+ "test:webchat": "pnpm --filter eve run test:webchat","test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures","test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"","test:scenario": "turbo run test:scenario",diff --git a/packages/eve/package.json b/packages/eve/package.jsonindex 89b53aca9..661b1e926 100644--- a/packages/eve/package.json+++ b/packages/eve/package.json@@ -527,7 +527,7 @@"dev": "pnpm run build:compiled && node ./scripts/copy-runtime-assets.mjs && tsc -p tsconfig.dev.json --watch --preserveWatchOutput","prepack": "pnpm run build && node ../../scripts/copy-package-license.mjs .","test": "pnpm run test:unit && pnpm run test:integration",- "test:browser": "pnpm run build:js && vitest run --config vitest.browser.config.ts",+ "test:webchat": "pnpm run build:js && vitest run --config vitest.browser.config.ts","test:framework-fixtures": "pnpm run build:compiled && vitest run --config vitest.framework-fixtures.config.ts","test:integration": "pnpm run build:js && vitest run --config vitest.integration.config.ts","test:scenario": "pnpm run build:compiled && vitest run --config vitest.scenario.config.ts",diff --git a/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts b/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.tsindex 6cc2eb511..7ca0f6c00 100644--- a/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts@@ -108,7 +108,13 @@ export function webChatDriver(): ClientDriver {`one of the questions ${JSON.stringify(prompts)}`,async () => {await look();- for (const prompt of prompts) {+ // The page lists requests in the order the session asked them, which is the+ // order a typed reply answers them in.+ const texts = (await page.locator("p").allInnerTexts()).map((text) => text.trim());+ const shown = prompts+ .filter((prompt) => texts.includes(prompt))+ .sort((a, b) => texts.lastIndexOf(a) - texts.lastIndexOf(b));+ for (const prompt of shown) {const options = await promptOptions(page, prompt).catch(() => undefined);if (options !== undefined) return { options, prompt };}@@ -143,7 +149,9 @@ export function webChatDriver(): ClientDriver {for (const message of await page.locator(".is-assistant").all()) {const links: string[] = [];for (const anchor of await message.locator("a[href]").all()) {- links.push((await anchor.getAttribute("href"))!);+ // A re-render can drop a link after it was listed; the next look reads it again.+ const href = await anchor.getAttribute("href", { timeout: 100 }).catch(() => null);+ if (href !== null) links.push(href);}const text = [await message.innerText(), ...links].join("\n");const options = (await readButtons(message.getByRole("button"))) ?? [];@@ -167,12 +175,18 @@ async function promptOptions(page: Page, prompt: string): Promise<RenderedOption);}-/** An open-ended `ask_question` card: the prompt above a text field while it's open. */+/** An open-ended `ask_question` card: the prompt above a text field and no options while it's open. */async function openEndedQuestion(page: Page, prompt: string): Promise<[] | undefined> {const label = page.locator("p").getByText(prompt, { exact: true }).last();if ((await label.count()) === 0) return undefined;- const field = label.locator("xpath=..").getByRole("textbox", { name: "Answer" });- return (await field.count()) > 0 && (await field.isEditable()) ? [] : undefined;+ const card = label.locator("xpath=..");+ const field = card.getByRole("textbox", { name: "Answer" });+ if ((await field.count()) === 0 || !(await field.isEditable().catch(() => false))) {+ return undefined;+ }+ // A question with options also takes free text. Its card can mount after `questionOptions`+ // looked, so check for options only once the field is known to be there.+ return (await card.getByRole("radio").count()) === 0 ? [] : undefined;}/** An `ask_question` card: one radio per option while the question is open. */