Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: relax the visibility test for ai snapshots (#35898)

Pavel Feldman263453a7b2ce
diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts
index 6782814970..771f4e8327 100644
--- a/packages/injected/src/ariaSnapshot.ts
+++ b/packages/injected/src/ariaSnapshot.ts
@@ -16,7 +16,7 @@
import { escapeRegExp, longestCommonSubstring, normalizeWhiteSpace } from '@isomorphic/stringUtils';
-import { box, getElementComputedStyle, getGlobalOptions } from './domUtils';
+import { box, getElementComputedStyle, getGlobalOptions, isElementVisible } from './domUtils';
import * as roleUtils from './roleUtils';
import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from './yaml';
@@ -75,7 +75,10 @@ export function generateAriaTree(rootElement: Element, generation: number, optio
return;
const element = node as Element;
- if (roleUtils.isElementHiddenForAria(element))
+ let isVisible = !roleUtils.isElementHiddenForAria(element);
+ if (options?.forAI)
+ isVisible = isVisible || isElementVisible(element);
+ if (!isVisible)
return;
const ariaChildren: Element[] = [];
@@ -197,7 +200,9 @@ function normalizeGenericRoles(node: AriaNode) {
const normalized = normalizeChildren(child);
result.push(...normalized);
}
- const removeSelf = node.role === 'generic' && result.every(c => typeof c !== 'string' && canRef(c));
+
+ // Only remove generic that encloses one element, logical grouping still makes sense, even if it is not ref-able.
+ const removeSelf = node.role === 'generic' && result.length <= 1 && result.every(c => typeof c !== 'string' && receivesPointerEvents(c));
if (removeSelf)
return result;
node.children = result;
@@ -402,10 +407,11 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'r
key += ` [pressed]`;
if (ariaNode.selected === true)
key += ` [selected]`;
- if (options?.forAI && canRef(ariaNode)) {
+ if (options?.forAI && receivesPointerEvents(ariaNode)) {
const id = ariaSnapshot.ids.get(ariaNode.element);
+ const cursor = hasPointerCursor(ariaNode) ? ' [cursor=pointer]' : '';
if (id)
- key += ` [ref=s${ariaSnapshot.generation}e${id}]`;
+ key += ` [ref=s${ariaSnapshot.generation}e${id}]${cursor}`;
}
const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(key);
@@ -496,6 +502,10 @@ function textContributesInfo(node: AriaNode, text: string): boolean {
return filtered.trim().length / text.length > 0.1;
}
-function canRef(ariaNode: AriaNode): boolean {
+function receivesPointerEvents(ariaNode: AriaNode): boolean {
return ariaNode.box.visible && ariaNode.receivesPointerEvents;
}
+
+function hasPointerCursor(ariaNode: AriaNode): boolean {
+ return ariaNode.box.style?.cursor === 'pointer';
+}
diff --git a/tests/page/page-aria-snapshot-ai.spec.ts b/tests/page/page-aria-snapshot-ai.spec.ts
new file mode 100644
index 0000000000..dde1ebff51
--- /dev/null
+++ b/tests/page/page-aria-snapshot-ai.spec.ts
@@ -0,0 +1,223 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { FrameLocator, Page } from '@playwright/test';
+import { test as it, expect } from './pageTest';
+
+const forAI = { _forAI: true } as any;
+
+it('should generate refs', async ({ page }) => {
+ await page.setContent(`
+ <button>One</button>
+ <button>Two</button>
+ <button>Three</button>
+ `);
+
+ const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot1).toContain('- button "One" [ref=s1e3]');
+ expect(snapshot1).toContain('- button "Two" [ref=s1e4]');
+ expect(snapshot1).toContain('- button "Three" [ref=s1e5]');
+
+ await expect(page.locator('aria-ref=s1e3')).toHaveText('One');
+ await expect(page.locator('aria-ref=s1e4')).toHaveText('Two');
+ await expect(page.locator('aria-ref=s1e5')).toHaveText('Three');
+
+ const snapshot2 = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot2).toContain('- button "One" [ref=s2e3]');
+ await expect(page.locator('aria-ref=s2e3')).toHaveText('One');
+
+ const e = await expect(page.locator('aria-ref=s1e3')).toHaveText('One').catch(e => e);
+ expect(e.message).toContain('Error: Stale aria-ref, expected s2e{number}, got s1e3');
+});
+
+it('should list iframes', async ({ page }) => {
+ await page.setContent(`
+ <h1>Hello</h1>
+ <iframe name="foo" src="data:text/html,<h1>World</h1>">
+ `);
+
+ const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot1).toContain('- iframe');
+
+ const frameSnapshot = await page.frameLocator(`iframe`).locator('body').ariaSnapshot();
+ expect(frameSnapshot).toEqual('- heading "World" [level=1]');
+});
+
+it('ref mode can be used to stitch all frame snapshots', async ({ page, server }) => {
+ await page.goto(server.PREFIX + '/frames/nested-frames.html');
+
+ async function allFrameSnapshot(frame: Page | FrameLocator): Promise<string> {
+ const snapshot = await frame.locator('body').ariaSnapshot(forAI);
+ const lines = snapshot.split('\n');
+ const result = [];
+ for (const line of lines) {
+ const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);
+ if (!match) {
+ result.push(line);
+ continue;
+ }
+
+ const leadingSpace = match[1];
+ const ref = match[2];
+ const childFrame = frame.frameLocator(`aria-ref=${ref}`);
+ const childSnapshot = await allFrameSnapshot(childFrame);
+ result.push(line + ':', childSnapshot.split('\n').map(l => leadingSpace + ' ' + l).join('\n'));
+ }
+ return result.join('\n');
+ }
+
+ expect(await allFrameSnapshot(page)).toContainYaml(`
+ - generic [ref=s1e2]:
+ - iframe [ref=s1e3]:
+ - generic [ref=s1e2]:
+ - iframe [ref=s1e3]:
+ - generic [ref=s1e3]: Hi, I'm frame
+ - iframe [ref=s1e4]:
+ - generic [ref=s1e3]: Hi, I'm frame
+ - iframe [ref=s1e4]:
+ - generic [ref=s1e3]: Hi, I'm frame
+ `);
+});
+
+it('should not generate refs for hidden elements', async ({ page }) => {
+ await page.setContent(`
+ <button>One</button>
+ <button style="width: 0; height: 0; appearance: none; border: 0; padding: 0;">Two</button>
+ <button>Three</button>
+ `);
+
+ const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot).toContainYaml(`
+ - generic [ref=s1e2]:
+ - button "One" [ref=s1e3]
+ - button "Two"
+ - button "Three" [ref=s1e5]
+ `);
+});
+
+it('should not generate refs for elements with pointer-events:none', async ({ page }) => {
+ await page.setContent(`
+ <button style="pointer-events: none">no-ref</button>
+ <div style="pointer-events: none">
+ <button style="pointer-events: auto">with-ref</button>
+ </div>
+ <div style="pointer-events: none">
+ <div style="pointer-events: initial">
+ <button>with-ref</button>
+ </div>
+ </div>
+ <div style="pointer-events: none">
+ <div style="pointer-events: auto">
+ <button>with-ref</button>
+ </div>
+ </div>
+ <div style="pointer-events: auto">
+ <div style="pointer-events: none">
+ <button>no-ref</button>
+ </div>
+ </div>
+ `);
+
+ const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot).toContainYaml(`
+ - generic [ref=s1e2]:
+ - button "no-ref"
+ - button "with-ref" [ref=s1e5]
+ - button "with-ref" [ref=s1e8]
+ - button "with-ref" [ref=s1e11]
+ - generic [ref=s1e12]:
+ - generic:
+ - button "no-ref"
+ `);
+});
+
+it('emit generic roles for nodes w/o roles', async ({ page }) => {
+ await page.setContent(`
+ <style>
+ input {
+ width: 0;
+ height: 0;
+ opacity: 0;
+ }
+ </style>
+ <div>
+ <label>
+ <span>
+ <input type="radio" value="Apple" checked="">
+ </span>
+ <span>Apple</span>
+ </label>
+ <label>
+ <span>
+ <input type="radio" value="Pear">
+ </span>
+ <span>Pear</span>
+ </label>
+ <label>
+ <span>
+ <input type="radio" value="Orange">
+ </span>
+ <span>Orange</span>
+ </label>
+ </div>
+ `);
+
+ const snapshot = await page.locator('body').ariaSnapshot(forAI);
+
+ expect(snapshot).toContainYaml(`
+ - generic [ref=s1e3]:
+ - generic [ref=s1e4]:
+ - generic [ref=s1e5]:
+ - radio "Apple" [checked]
+ - generic [ref=s1e7]: Apple
+ - generic [ref=s1e8]:
+ - generic [ref=s1e9]:
+ - radio "Pear"
+ - generic [ref=s1e11]: Pear
+ - generic [ref=s1e12]:
+ - generic [ref=s1e13]:
+ - radio "Orange"
+ - generic [ref=s1e15]: Orange
+ `);
+});
+
+it('should collapse generic nodes', async ({ page }) => {
+ await page.setContent(`
+ <div>
+ <div>
+ <div>
+ <button>Button</button>
+ </div>
+ </div>
+ </div>
+ `);
+
+ const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot).toContainYaml(`
+ - button \"Button\" [ref=s1e6]
+ `);
+});
+
+it('should include cursor pointer hint', async ({ page }) => {
+ await page.setContent(`
+ <button style="cursor: pointer">Button</button>
+ `);
+
+ const snapshot = await page.locator('body').ariaSnapshot(forAI);
+ expect(snapshot).toContainYaml(`
+ - button \"Button\" [ref=s1e3] [cursor=pointer]
+ `);
+});
diff --git a/tests/page/page-aria-snapshot.spec.ts b/tests/page/page-aria-snapshot.spec.ts
index 87cc91796d..ce19ea4351 100644
--- a/tests/page/page-aria-snapshot.spec.ts
+++ b/tests/page/page-aria-snapshot.spec.ts
@@ -1,12 +1,11 @@
/**
- * Copyright 2018 Google Inc. All rights reserved.
- * Modifications copyright (c) Microsoft Corporation.
+ * Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,33 +14,8 @@
* limitations under the License.
*/
-import type { Locator, FrameLocator, Page } from '@playwright/test';
-import { test as it, expect as baseExpect } from './pageTest';
-
-const expect = baseExpect.extend({
- toContainYaml(received: string, expected: string) {
- const trimmed = expected.split('\n').filter(a => !!a.trim());
- const maxPrefixLength = Math.min(...trimmed.map(line => line.match(/^\s*/)[0].length));
- const trimmedExpected = trimmed.map(line => line.substring(maxPrefixLength)).join('\n');
- try {
- if (this.isNot)
- expect(received).not.toContain(trimmedExpected);
- else
- expect(received).toContain(trimmedExpected);
- return {
- pass: !this.isNot,
- message: () => '',
- };
- } catch (e) {
- return {
- pass: this.isNot,
- message: () => e.message,
- };
- }
- }
-});
-
-const forAI = { _forAI: true } as any;
+import type { Locator } from '@playwright/test';
+import { test as it, expect } from './pageTest';
function unshift(snapshot: string): string {
const lines = snapshot.split('\n');
@@ -684,189 +658,3 @@ it('should not report textarea textContent', async ({ page }) => {
- textbox: After
`);
});
-
-it('should generate refs', async ({ page }) => {
- await page.setContent(`
- <button>One</button>
- <button>Two</button>
- <button>Three</button>
- `);
-
- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot1).toContain('- button "One" [ref=s1e3]');
- expect(snapshot1).toContain('- button "Two" [ref=s1e4]');
- expect(snapshot1).toContain('- button "Three" [ref=s1e5]');
-
- await expect(page.locator('aria-ref=s1e3')).toHaveText('One');
- await expect(page.locator('aria-ref=s1e4')).toHaveText('Two');
- await expect(page.locator('aria-ref=s1e5')).toHaveText('Three');
-
- const snapshot2 = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot2).toContain('- button "One" [ref=s2e3]');
- await expect(page.locator('aria-ref=s2e3')).toHaveText('One');
-
- const e = await expect(page.locator('aria-ref=s1e3')).toHaveText('One').catch(e => e);
- expect(e.message).toContain('Error: Stale aria-ref, expected s2e{number}, got s1e3');
-});
-
-it('should list iframes', async ({ page }) => {
- await page.setContent(`
- <h1>Hello</h1>
- <iframe name="foo" src="data:text/html,<h1>World</h1>">
- `);
-
- const snapshot1 = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot1).toContain('- iframe');
-
- const frameSnapshot = await page.frameLocator(`iframe`).locator('body').ariaSnapshot();
- expect(frameSnapshot).toEqual('- heading "World" [level=1]');
-});
-
-it('ref mode can be used to stitch all frame snapshots', async ({ page, server }) => {
- await page.goto(server.PREFIX + '/frames/nested-frames.html');
-
- async function allFrameSnapshot(frame: Page | FrameLocator): Promise<string> {
- const snapshot = await frame.locator('body').ariaSnapshot(forAI);
- const lines = snapshot.split('\n');
- const result = [];
- for (const line of lines) {
- const match = line.match(/^(\s*)- iframe \[ref=(.*)\]/);
- if (!match) {
- result.push(line);
- continue;
- }
-
- const leadingSpace = match[1];
- const ref = match[2];
- const childFrame = frame.frameLocator(`aria-ref=${ref}`);
- const childSnapshot = await allFrameSnapshot(childFrame);
- result.push(line + ':', childSnapshot.split('\n').map(l => leadingSpace + ' ' + l).join('\n'));
- }
- return result.join('\n');
- }
-
- expect(await allFrameSnapshot(page)).toContainYaml(`
- - iframe [ref=s1e3]:
- - iframe [ref=s1e3]:
- - generic [ref=s1e3]: Hi, I'm frame
- - iframe [ref=s1e4]:
- - generic [ref=s1e3]: Hi, I'm frame
- - iframe [ref=s1e4]:
- - generic [ref=s1e3]: Hi, I'm frame
- `);
-});
-
-it('should not generate refs for hidden elements', async ({ page }) => {
- await page.setContent(`
- <button>One</button>
- <button style="width: 0; height: 0; appearance: none; border: 0; padding: 0;">Two</button>
- <button>Three</button>
- `);
-
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot).toContainYaml(`
- - generic [ref=s1e2]:
- - button "One" [ref=s1e3]
- - button "Two"
- - button "Three" [ref=s1e5]
- `);
-});
-
-it('should not generate refs for elements with pointer-events:none', async ({ page }) => {
- await page.setContent(`
- <button style="pointer-events: none">no-ref</button>
- <div style="pointer-events: none">
- <button style="pointer-events: auto">with-ref</button>
- </div>
- <div style="pointer-events: none">
- <div style="pointer-events: initial">
- <button>with-ref</button>
- </div>
- </div>
- <div style="pointer-events: none">
- <div style="pointer-events: auto">
- <button>with-ref</button>
- </div>
- </div>
- <div style="pointer-events: auto">
- <div style="pointer-events: none">
- <button>no-ref</button>
- </div>
- </div>
- `);
-
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot).toContainYaml(`
- - generic [ref=s1e2]:
- - button "no-ref"
- - button "with-ref" [ref=s1e5]
- - button "with-ref" [ref=s1e8]
- - button "with-ref" [ref=s1e11]
- - generic [ref=s1e12]:
- - generic:
- - button "no-ref"
- `);
-});
-
-it('emit generic roles for nodes w/o roles', async ({ page }) => {
- await page.setContent(`
- <style>
- input {
- width: 0;
- height: 0;
- opacity: 0;
- }
- </style>
- <div>
- <label>
- <span>
- <input type="radio" value="Apple" checked="">
- </span>
- <span>Apple</span>
- </label>
- <label>
- <span>
- <input type="radio" value="Pear">
- </span>
- <span>Pear</span>
- </label>
- <label>
- <span>
- <input type="radio" value="Orange">
- </span>
- <span>Orange</span>
- </label>
- </div>
- `);
-
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
-
- expect(snapshot).toContainYaml(`
- - generic [ref=s1e5]:
- - radio "Apple" [checked]
- - generic [ref=s1e7]: Apple
- - generic [ref=s1e9]:
- - radio "Pear"
- - generic [ref=s1e11]: Pear
- - generic [ref=s1e13]:
- - radio "Orange"
- - generic [ref=s1e15]: Orange
- `);
-});
-
-it('should collapse generic nodes', async ({ page }) => {
- await page.setContent(`
- <div>
- <div>
- <div>
- <button>Button</button>
- </div>
- </div>
- </div>
- `);
-
- const snapshot = await page.locator('body').ariaSnapshot(forAI);
- expect(snapshot).toContainYaml(`
- - button \"Button\" [ref=s1e6]
- `);
-});
diff --git a/tests/page/pageTest.ts b/tests/page/pageTest.ts
index 0bf4d469ea..f664756388 100644
--- a/tests/page/pageTest.ts
+++ b/tests/page/pageTest.ts
@@ -23,7 +23,7 @@ import { electronTest } from '../electron/electronTest';
import { webView2Test } from '../webview2/webView2Test';
import type { PageTestFixtures, PageWorkerFixtures } from './pageTestApi';
import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtures';
-export { expect } from '@playwright/test';
+import { expect as baseExpect } from '@playwright/test';
let impl: TestType<PageTestFixtures & ServerFixtures & TestModeTestFixtures, PageWorkerFixtures & PlatformWorkerFixtures & TestModeWorkerFixtures & TestModeWorkerOptions & ServerWorkerOptions> = browserTest;
export type BoundingBox = Awaited<ReturnType<Locator['boundingBox']>>;
@@ -53,3 +53,26 @@ export function roundBox(box: BoundingBox): BoundingBox {
height: Math.round(box.height),
};
}
+
+export const expect = baseExpect.extend({
+ toContainYaml(received: string, expected: string) {
+ const trimmed = expected.split('\n').filter(a => !!a.trim());
+ const maxPrefixLength = Math.min(...trimmed.map(line => line.match(/^\s*/)[0].length));
+ const trimmedExpected = trimmed.map(line => line.substring(maxPrefixLength)).join('\n');
+ try {
+ if (this.isNot)
+ expect(received).not.toContain(trimmedExpected);
+ else
+ expect(received).toContain(trimmedExpected);
+ return {
+ pass: !this.isNot,
+ message: () => '',
+ };
+ } catch (e) {
+ return {
+ pass: this.isNot,
+ message: () => e.message,
+ };
+ }
+ }
+});