π OpenHands: AI-Driven Development
test(e2e): bound and diagnose sidebar modified-click tab waits (#18364)
Engel Nystaf6b1f3cf691
diff --git a/.agents/skills/e2e-testing/references/guide.md b/.agents/skills/e2e-testing/references/guide.mdindex d2b098a261..2f6137aa58 100644--- a/.agents/skills/e2e-testing/references/guide.md+++ b/.agents/skills/e2e-testing/references/guide.md@@ -52,6 +52,7 @@- `regressions/` β CSS isolation, event pagination, workspace persistence (`mock-llm-ui-regressions.spec.ts`). Always included in selective runs.- **Selective test resolver**: `tests/e2e/mock-llm/test-mapping.json` maps source paths to test subdirectories, and `tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs` can resolve a changed-file list for local investigation. Post-merge CI and `workflow_dispatch` intentionally run the full suite, so this resolver is not part of the workflow path.- Tests run serially (`workers: 1`, `mode: "serial"` per describe block). Each spec is self-contained (configures its own LLM profile, resets mock LLM in `afterEach`). The `afterEach` hook resets the mock LLM to its default trajectory so subsequent specs start fresh even when a preceding test fails.+- Cross-connect's sidebar modified click uses a five-second new-tab wait, independent of its slow backend setup budget. `modified-click-diagnostics.json` records the actual target/anchor, modifiers and final `defaultPrevented` state on both success and failure. Do not retry a lost click: inspect that attachment before attributing the failure to the app or browser harness. `mock-llm-modified-click-tab.spec.ts` exercises native link success, a deliberately consumed click, and an obscured link without a backend.- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs the full suite after pushes reach `main` and on manual dispatch; it does not run on pull-request events. The workflow builds the frontend, starts the mock LLM server, runs the tests, uploads artifacts, and writes the rendered report to the workflow summary. Npm-path Mock-LLM CI uses `MOCK_LLM_GLOBAL_TIMEOUT_MS` (default 20 min) and the workflow deadline adds a 60-second teardown buffer; keep `playwright.mock-llm.config.ts` and `.github/workflows/mock-llm-e2e.yml` in sync if that timeout changes.- The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process.diff --git a/.pr/18123/browser-only.config.ts b/.pr/18123/browser-only.config.tsnew file mode 100644index 0000000000..05245810e7--- /dev/null+++ b/.pr/18123/browser-only.config.ts@@ -0,0 +1,13 @@+// Replays the focused browser checks without starting unrelated stack services.+import { defineConfig, devices } from "@playwright/test";+export default defineConfig({+ testDir: "../../tests/e2e/mock-llm/backends",+ testMatch: "mock-llm-modified-click-tab.spec.ts",+ retries: 0,+ workers: 1,+ timeout: 20_000,+ reporter: [["list"], ["json", { outputFile: "../../.tmp/18123-native-results.json" }]],+ outputDir: "../../.tmp/18123-native-results",+ use: { trace: "off", video: "off", screenshot: "only-on-failure" },+ projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],+});diff --git a/.pr/18123/native-click-diagnostics.json b/.pr/18123/native-click-diagnostics.jsonnew file mode 100644index 0000000000..ae4224f5f7--- /dev/null+++ b/.pr/18123/native-click-diagnostics.json@@ -0,0 +1,121 @@+[+ {+ "check": "a modified link click opens one tab and records the delivered event",+ "before": {+ "sourceUrl": "about:blank",+ "expectedHref": "https://example.test/conversations/123?backend=A",+ "modifiers": [+ "ControlOrMeta"+ ],+ "pages": [+ "about:blank"+ ],+ "timeoutMs": 5000+ },+ "observed": {+ "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36",+ "events": [+ {+ "target": {+ "tag": "SPAN",+ "id": "title",+ "testId": null+ },+ "anchor": {+ "href": "https://example.test/conversations/123?backend=A",+ "target": null+ },+ "type": "click",+ "button": 0,+ "ctrlKey": false,+ "metaKey": true,+ "shiftKey": false,+ "altKey": false,+ "isTrusted": true,+ "defaultPrevented": false+ }+ ]+ },+ "after": {+ "sourceUrl": "about:blank",+ "pages": [+ "about:blank",+ "https://example.test/conversations/123?backend=A"+ ],+ "openedTab": true,+ "clickFailed": false+ }+ },+ {+ "check": "a consumed click fails quickly and records its actual target and final prevention",+ "before": {+ "sourceUrl": "about:blank",+ "expectedHref": "https://example.test/conversations/123?backend=A",+ "modifiers": [+ "ControlOrMeta"+ ],+ "pages": [+ "about:blank"+ ],+ "timeoutMs": 5000+ },+ "observed": {+ "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36",+ "events": [+ {+ "target": {+ "tag": "SPAN",+ "id": "title",+ "testId": null+ },+ "anchor": {+ "href": "https://example.test/conversations/123?backend=A",+ "target": null+ },+ "type": "click",+ "button": 0,+ "ctrlKey": false,+ "metaKey": true,+ "shiftKey": false,+ "altKey": false,+ "isTrusted": true,+ "defaultPrevented": true+ }+ ]+ },+ "after": {+ "sourceUrl": "about:blank",+ "pages": [+ "about:blank"+ ],+ "openedTab": false,+ "clickFailed": false+ }+ },+ {+ "check": "an obscured link reports click failure and handles its pending page waiter",+ "before": {+ "sourceUrl": "about:blank",+ "expectedHref": "https://example.test/conversations/123?backend=A",+ "modifiers": [+ "ControlOrMeta"+ ],+ "pages": [+ "about:blank"+ ],+ "timeoutMs": 300+ },+ "observed": {+ "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36",+ "events": []+ },+ "after": {+ "sourceUrl": "about:blank",+ "pages": [+ "about:blank"+ ],+ "openedTab": false,+ "clickFailed": true+ }+ }+]diff --git a/.pr/18123/verification-report.json b/.pr/18123/verification-report.jsonnew file mode 100644index 0000000000..5c6bd0e37f--- /dev/null+++ b/.pr/18123/verification-report.json@@ -0,0 +1,123 @@+{+ "attribution": "I am an AI agent helping Engel Nyst (@enyst) with project maintenance.",+ "issue": "https://github.com/OpenHands/OpenHands/issues/18123",+ "sourceCommit": "d5b5cca9d85f9bdc08f68f88aae40be22d46e3e7",+ "helperSha256": "4478659ee79db977e23e460ceb224b2cfec2624a03f298aa3ecf46f979734ecb",+ "scope": "Cross-connect modified-click harness timeout and diagnostic observer. No production UI changes.",+ "browser": "Project-pinned Playwright 1.63.0 Chromium runner; one worker; no retries; native setContent fixtures; no app/backend stack for focused cases.",+ "positiveRun": {+ "startTime": "2026-10-13T04:25:22.920Z",+ "durationMs": 5992.856000000001,+ "passed": 3,+ "failed": 0,+ "flaky": 0,+ "checks": [+ {+ "name": "a modified link click opens one tab and records the delivered event",+ "status": "passed",+ "durationMs": 136+ },+ {+ "name": "a consumed click fails quickly and records its actual target and final prevention",+ "status": "passed",+ "durationMs": 5082+ },+ {+ "name": "an obscured link reports click failure and handles its pending page waiter",+ "status": "passed",+ "durationMs": 384+ }+ ]+ },+ "negativeControl": {+ "description": "Same consumed native click using the original unbounded context.waitForEvent(page) pattern extracted into a temporary helper. Forced no-tab path reached the 15-second overall test timeout instead of the expected five-second step error.",+ "startTime": "2026-10-13T04:24:29.272Z",+ "durationMs": 15417.512,+ "failedAsExpected": true,+ "actualError": "browserContext.waitForEvent: Test timeout of 15000ms exceeded.",+ "expectedError": "cmd-click opened no new tab within 5000 ms"+ },+ "staticChecks": [+ "Strict isolated TypeScript for helper and native browser spec passed.",+ "Prettier for helper, native spec and modified cross-connect spec passed.",+ "git diff --check passed."+ ],+ "notRun": [+ "Separate Docker suite.",+ "Original intermittent Linux CI lost-click was not reproduced locally or in this CI run."+ ],+ "teardown": "Configured Playwright runner exited; native fixtures/browser contexts closed. No product stack or user service was started for focused tests.",+ "replayCommand": "npx playwright test --config=.pr/18123/browser-only.config.ts",+ "configurationNote": "The original run used the same settings from .tmp/18123-browser.config.ts; the retained replay config adjusts only relative directory depth for its artifact location.",+ "fullStackCiRun": {+ "workflow": "mock-llm-e2e.yml",+ "url": "https://github.com/OpenHands/OpenHands/actions/runs/38347341282",+ "testedHead": "6b87b2a054cec3f02fc99af8717aa6799ae2777a",+ "startTime": "2026-10-13T04:39:22.262Z",+ "durationMs": 683385.902,+ "counts": {+ "passed": 74,+ "failed": 2,+ "didNotRun": 2,+ "flaky": 0,+ "total": 78+ },+ "relevantChecks": [+ {+ "file": "backends/mock-llm-cross-connect.spec.ts",+ "name": "frontend-only connects to a separate backend-only instance",+ "status": "passed",+ "durationMs": 15838+ },+ {+ "file": "backends/mock-llm-cross-connect.spec.ts",+ "name": "connects to two separate backends and switches between them",+ "status": "passed",+ "durationMs": 19755+ },+ {+ "file": "backends/mock-llm-cross-connect.spec.ts",+ "name": "cmd-clicking a sidebar conversation opens it on the owning backend",+ "status": "passed",+ "durationMs": 29949+ },+ {+ "file": "backends/mock-llm-modified-click-tab.spec.ts",+ "name": "a modified link click opens one tab and records the delivered event",+ "status": "passed",+ "durationMs": 239+ },+ {+ "file": "backends/mock-llm-modified-click-tab.spec.ts",+ "name": "a consumed click fails quickly and records its actual target and final prevention",+ "status": "passed",+ "durationMs": 5264+ },+ {+ "file": "backends/mock-llm-modified-click-tab.spec.ts",+ "name": "an obscured link reports click failure and handles its pending page waiter",+ "status": "passed",+ "durationMs": 419+ }+ ],+ "unrelatedFailures": [+ {+ "file": "tests/e2e/mock-llm/conversations/mock-llm-conversation.spec.ts",+ "name": "step 3: run a conversation with the mock LLM",+ "error": "No successful bash execution after 30000ms. 1 BashOutput events",+ "scope": "The API returned one BashOutput event that did not satisfy the existing exit_code/stdout predicate. The retained report does not contain that event body, so its non-qualifying field and underlying cause are unconfirmed. The screenshot shows the scripted agent reply; the test stopped at the Bash assertion before its later reply assertions."+ },+ {+ "file": "tests/e2e/mock-llm/regressions/mock-llm-ui-regressions.spec.ts",+ "name": "home warning icon follows the host warning color",+ "error": "home-llm-not-configured-banner was absent after 5000ms",+ "scope": "The page snapshot showed the mock-llm model selected. This test does not arrange empty LLM settings and stopped before testing icon color. A configured-model precondition mismatch is inferred from the snapshot and test source."+ }+ ],+ "workflowExit": {+ "code": 124,+ "explanation": "The wrapper observed the completed-failed marker at 04:50:44 UTC; the runner printed its normal summary at 04:50:45 UTC; the wrapper printed code 124 at 04:50:49 UTC. No runtime kill/hang warning appeared. Its already-running-process branch sleeps five seconds and then unconditionally assigns 124 even if the process exits normally during that grace period. This supports a wrapper exit-code misclassification, not a global deadline or confirmed teardown hang."+ },+ "claimLimit": "All three cross-connect stack cases and all three native helper cases passed. The full suite failed. The original intermittent Linux lost-click was not reproduced, and its application interception cause remains unidentified."+ }+}diff --git a/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts b/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.tsindex ffb00f1b53..5f087e9f47 100644--- a/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts+++ b/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts@@ -29,6 +29,7 @@ import { existsSync, mkdtempSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";+import { openModifiedClickTab } from "../utils/modified-click-tab";// ββ Paths ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββconst PROJECT_ROOT = resolve(@@ -665,7 +666,7 @@ test.describe("cross-connect: sidebar links pin their backend", () => {test("cmd-clicking a sidebar conversation opens it on the owning backend", async ({page,context,- }) => {+ }, testInfo) => {// Two backend-only instances via uvx β very slowtest.setTimeout(360_000);@@ -815,9 +816,7 @@ test.describe("cross-connect: sidebar links pin their backend", () => {const href = await card.getAttribute("href");- const newTabPromise = context.waitForEvent("page");- await card.click({ modifiers: ["ControlOrMeta"] });- const newTab = await newTabPromise;+ const newTab = await openModifiedClickTab(card, testInfo);// ββ 7. The new tab must resolve the conversation on backend A βββββ// A freshly opened tab starts at about:blank, so wait for the realdiff --git a/tests/e2e/mock-llm/backends/mock-llm-modified-click-tab.spec.ts b/tests/e2e/mock-llm/backends/mock-llm-modified-click-tab.spec.tsnew file mode 100644index 0000000000..f1eb8218cc--- /dev/null+++ b/tests/e2e/mock-llm/backends/mock-llm-modified-click-tab.spec.ts@@ -0,0 +1,103 @@+import { expect, test } from "@playwright/test";+import { openModifiedClickTab } from "../utils/modified-click-tab";++test("a modified link click opens one tab and records the delivered event", async ({+ page,+}, testInfo) => {+ // Arrange: use the browser's native link behavior, without an app/backend.+ await page.setContent(+ '<a href="https://example.test/conversations/123?backend=A"><span id="title">Conversation</span></a>',+ );+ await page+ .context()+ .route("https://example.test/**", (route) =>+ route.fulfill({ body: "Owning backend A" }),+ );++ // Act: the same helper used by the full cross-connect regression.+ const newTab = await openModifiedClickTab(page.locator("a"), testInfo);++ // Assert: browser navigation and its diagnostic attachment stay intact.+ await expect(newTab).toHaveURL(+ "https://example.test/conversations/123?backend=A",+ );+ const attachment = testInfo.attachments.find(+ (item) => item.name === "modified-click-diagnostics.json",+ );+ const diagnostics = JSON.parse(attachment!.body!.toString());+ expect(diagnostics.observed.events).toEqual([+ expect.objectContaining({+ anchor: {+ href: "https://example.test/conversations/123?backend=A",+ target: null,+ },+ isTrusted: true,+ defaultPrevented: false,+ }),+ ]);+});++test("a consumed click fails quickly and records its actual target and final prevention", async ({+ page,+}, testInfo) => {+ test.setTimeout(15_000);+ // Arrange: deliberately prevent a new tab, including bubble propagation.+ await page.setContent(+ '<a href="https://example.test/conversations/123?backend=A"><span id="title">Conversation</span></a>',+ );+ await page.locator("#title").evaluate((title) => {+ title.addEventListener("click", (event) => {+ event.preventDefault();+ event.stopPropagation();+ });+ });++ // Act / Assert: bound only this new-tab step, preserving the suite budget.+ const startedAt = Date.now();+ await expect(+ openModifiedClickTab(page.locator("a"), testInfo),+ ).rejects.toThrow("cmd-click opened no new tab within 5000 ms");+ expect(Date.now() - startedAt).toBeLessThan(10_000);+ const attachment = testInfo.attachments.find(+ (item) => item.name === "modified-click-diagnostics.json",+ );+ const diagnostics = JSON.parse(attachment!.body!.toString());+ expect(diagnostics.observed.events).toEqual([+ expect.objectContaining({+ target: { tag: "SPAN", id: "title", testId: null },+ anchor: {+ href: "https://example.test/conversations/123?backend=A",+ target: null,+ },+ defaultPrevented: true,+ }),+ ]);+ expect(diagnostics.after.openedTab).toBe(false);+});++test("an obscured link reports click failure and handles its pending page waiter", async ({+ page,+}, testInfo) => {+ test.setTimeout(5_000);+ // Arrange: a real overlay prevents Playwright from delivering the click.+ await page.setContent(`+ <a href="https://example.test/conversations/123?backend=A">Conversation</a>+ <div style="position:fixed;inset:0;background:white">Blocking overlay</div>+ `);++ // Act: both the actionability failure and page wait remain bounded.+ await expect(+ openModifiedClickTab(page.locator("a"), testInfo, 300),+ ).rejects.toThrow("cmd-click failed before opening a new tab");++ // Assert: diagnostics distinguish non-delivery from prevented dispatch.+ const attachment = testInfo.attachments.find(+ (item) => item.name === "modified-click-diagnostics.json",+ );+ const diagnostics = JSON.parse(attachment!.body!.toString());+ expect(diagnostics.observed.events).toEqual([]);+ expect(diagnostics.after).toMatchObject({+ openedTab: false,+ clickFailed: true,+ });+});diff --git a/tests/e2e/mock-llm/utils/modified-click-tab.ts b/tests/e2e/mock-llm/utils/modified-click-tab.tsnew file mode 100644index 0000000000..2b151bc36b--- /dev/null+++ b/tests/e2e/mock-llm/utils/modified-click-tab.ts@@ -0,0 +1,140 @@+import type { Locator, Page, TestInfo } from "@playwright/test";++/** Observe one real modified click without changing its browser behavior. */+export async function openModifiedClickTab(+ link: Locator,+ testInfo: Pick<TestInfo, "attach">,+ timeoutMs = 5_000,+): Promise<Page> {+ const page = link.page();+ const context = page.context();+ const before = {+ sourceUrl: page.url(),+ expectedHref: await link.getAttribute("href"),+ modifiers: ["ControlOrMeta"],+ pages: context.pages().map((tab) => tab.url()),+ timeoutMs,+ };++ await page.evaluate(() => {+ const state = {+ events: [] as {+ event: MouseEvent;+ target: { tag: string; id: string; testId: string | null } | null;+ anchor: { href: string | null; target: string | null } | null;+ }[],+ listener: (event: MouseEvent) => {+ const target = event.target instanceof Element ? event.target : null;+ const anchor = target?.closest("a");+ state.events.push({+ event,+ target: target+ ? {+ tag: target.tagName,+ id: target.id,+ testId: target.getAttribute("data-testid"),+ }+ : null,+ anchor: anchor+ ? {+ href: anchor.getAttribute("href"),+ target: anchor.getAttribute("target"),+ }+ : null,+ });+ },+ };+ // Keep the event itself until dispatch finishes, so defaultPrevented is+ // accurate even when a descendant stops propagation before window bubbles.+ const observedWindow = window as typeof window & {+ __openhandsModifiedClick?: typeof state;+ };+ observedWindow.__openhandsModifiedClick = state;+ window.addEventListener("click", state.listener, true);+ });++ // Handle rejection immediately: a failed click must not leave an unhandled+ // page-event timeout behind it. There is one click and no retry.+ const newTabPromise = context+ .waitForEvent("page", { timeout: timeoutMs })+ .then(+ (newTab) => ({ newTab }),+ (error: unknown) => ({ error }),+ );+ let clickError: unknown;+ try {+ await link.click({ modifiers: ["ControlOrMeta"], timeout: timeoutMs });+ } catch (error) {+ clickError = error;+ }+ const outcome = await newTabPromise;++ let observed: unknown;+ try {+ observed = await page.evaluate(() => {+ const observedWindow = window as typeof window & {+ __openhandsModifiedClick?: {+ events: {+ event: MouseEvent;+ target: unknown;+ anchor: unknown;+ }[];+ listener: (event: MouseEvent) => void;+ };+ };+ const state = observedWindow.__openhandsModifiedClick;+ if (!state)+ return { error: "click observer unavailable after navigation" };+ window.removeEventListener("click", state.listener, true);+ delete observedWindow.__openhandsModifiedClick;+ return {+ userAgent: navigator.userAgent,+ events: state.events.map(({ event, target, anchor }) => ({+ target,+ anchor,+ type: event.type,+ button: event.button,+ ctrlKey: event.ctrlKey,+ metaKey: event.metaKey,+ shiftKey: event.shiftKey,+ altKey: event.altKey,+ isTrusted: event.isTrusted,+ defaultPrevented: event.defaultPrevented,+ })),+ };+ });+ } catch {+ observed = { error: "click observer unavailable because the page closed" };+ }+ await testInfo.attach("modified-click-diagnostics.json", {+ body: Buffer.from(+ JSON.stringify(+ {+ before,+ observed,+ after: {+ sourceUrl: page.url(),+ pages: context.pages().map((tab) => tab.url()),+ openedTab: "newTab" in outcome,+ clickFailed: clickError !== undefined,+ },+ },+ null,+ 2,+ ),+ ),+ contentType: "application/json",+ });++ if (clickError !== undefined) {+ throw new Error("cmd-click failed before opening a new tab", {+ cause: clickError,+ });+ }+ if ("error" in outcome) {+ throw new Error(`cmd-click opened no new tab within ${timeoutMs} ms`, {+ cause: outcome.error,+ });+ }+ return outcome.newTab;+}