Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): deflake web chat browser tests, rename to test-webchat (#4468)

Ben Pankowac6d499e60c6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7c140116f..29dd0000c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -187,7 +187,7 @@ jobs:
- name: Build framework fixtures against the workspace eve dist
run: pnpm test:framework-fixtures
- test-browser:
+ test-webchat:
runs-on: ubuntu-latest
steps:
@@ -211,8 +211,8 @@ jobs:
- name: Install Chromium
run: pnpm --filter eve exec playwright-core install --with-deps chromium-headless-shell
- - name: Run browser tests
- run: pnpm test:browser
+ - name: Run web chat tests
+ run: pnpm test:webchat
test-templates:
runs-on: ubuntu-latest
diff --git a/AGENTS.md b/AGENTS.md
index 65605afd7..837da3116 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -59,7 +59,7 @@ pnpm test:unit # unit tests (<3s)
pnpm test:integration # integration tests (several minutes; leave the full run to CI)
pnpm test:scenario # scenario tests (2–5 min; requires pnpm build first)
pnpm test:framework-fixtures # apps/frameworks smoke builds (requires pnpm build first)
-pnpm test:browser # client templates in headless Chromium (needs playwright-core's chromium-headless-shell)
+pnpm test:webchat # web chat template in headless Chromium (needs playwright-core's chromium-headless-shell)
pnpm test:e2e # fixture-owned eve eval suites (CI only)
pnpm test:tui # TUI smoke scripts (not e2e)
```
diff --git a/package.json b/package.json
index 6fb3419e6..a8ecd504c 100644
--- a/package.json
+++ b/package.json
@@ -23,7 +23,7 @@
"release": "pnpm build && node ./scripts/assert-changeset-publish-packages.mjs && changeset publish",
"test": "pnpm test:unit && pnpm test:integration",
"test:e2e": "turbo run test:e2e --concurrency=1",
- "test:browser": "pnpm --filter eve run test:browser",
+ "test:webchat": "pnpm --filter eve run test:webchat",
"test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures",
"test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"",
"test:scenario": "turbo run test:scenario",
diff --git a/packages/eve/package.json b/packages/eve/package.json
index 89b53aca9..661b1e926 100644
--- a/packages/eve/package.json
+++ b/packages/eve/package.json
@@ -527,7 +527,7 @@
"dev": "pnpm run build:compiled && node ./scripts/copy-runtime-assets.mjs && tsc -p tsconfig.dev.json --watch --preserveWatchOutput",
"prepack": "pnpm run build && node ../../scripts/copy-package-license.mjs .",
"test": "pnpm run test:unit && pnpm run test:integration",
- "test:browser": "pnpm run build:js && vitest run --config vitest.browser.config.ts",
+ "test:webchat": "pnpm run build:js && vitest run --config vitest.browser.config.ts",
"test:framework-fixtures": "pnpm run build:compiled && vitest run --config vitest.framework-fixtures.config.ts",
"test:integration": "pnpm run build:js && vitest run --config vitest.integration.config.ts",
"test:scenario": "pnpm run build:compiled && vitest run --config vitest.scenario.config.ts",
diff --git a/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts b/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts
index 6cc2eb511..7ca0f6c00 100644
--- a/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/web-chat-driver.ts
@@ -108,7 +108,13 @@ export function webChatDriver(): ClientDriver {
`one of the questions ${JSON.stringify(prompts)}`,
async () => {
await look();
- for (const prompt of prompts) {
+ // The page lists requests in the order the session asked them, which is the
+ // order a typed reply answers them in.
+ const texts = (await page.locator("p").allInnerTexts()).map((text) => text.trim());
+ const shown = prompts
+ .filter((prompt) => texts.includes(prompt))
+ .sort((a, b) => texts.lastIndexOf(a) - texts.lastIndexOf(b));
+ for (const prompt of shown) {
const options = await promptOptions(page, prompt).catch(() => undefined);
if (options !== undefined) return { options, prompt };
}
@@ -143,7 +149,9 @@ export function webChatDriver(): ClientDriver {
for (const message of await page.locator(".is-assistant").all()) {
const links: string[] = [];
for (const anchor of await message.locator("a[href]").all()) {
- links.push((await anchor.getAttribute("href"))!);
+ // A re-render can drop a link after it was listed; the next look reads it again.
+ const href = await anchor.getAttribute("href", { timeout: 100 }).catch(() => null);
+ if (href !== null) links.push(href);
}
const text = [await message.innerText(), ...links].join("\n");
const options = (await readButtons(message.getByRole("button"))) ?? [];
@@ -167,12 +175,18 @@ async function promptOptions(page: Page, prompt: string): Promise<RenderedOption
);
}
-/** An open-ended `ask_question` card: the prompt above a text field while it's open. */
+/** An open-ended `ask_question` card: the prompt above a text field and no options while it's open. */
async function openEndedQuestion(page: Page, prompt: string): Promise<[] | undefined> {
const label = page.locator("p").getByText(prompt, { exact: true }).last();
if ((await label.count()) === 0) return undefined;
- const field = label.locator("xpath=..").getByRole("textbox", { name: "Answer" });
- return (await field.count()) > 0 && (await field.isEditable()) ? [] : undefined;
+ const card = label.locator("xpath=..");
+ const field = card.getByRole("textbox", { name: "Answer" });
+ if ((await field.count()) === 0 || !(await field.isEditable().catch(() => false))) {
+ return undefined;
+ }
+ // A question with options also takes free text. Its card can mount after `questionOptions`
+ // looked, so check for options only once the field is known to be there.
+ return (await card.getByRole("radio").count()) === 0 ? [] : undefined;
}
/** An `ask_question` card: one radio per option while the question is open. */